rbbt-util 5.21.56 → 5.21.57

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c9f3400dbfe429698356deea0dcd2f0a8232662a
-  data.tar.gz: 7d05e712c84d010cd1d4ca076f0036d505b2d8e0
+  metadata.gz: 92a11a2f42a6597852eb74111dc2355f6704ac1f
+  data.tar.gz: 15716a971a9ea69dc92a86ea9d9adfb76bea6d44
 SHA512:
-  metadata.gz: a29f6431499cdd7a4c72361ab8c1d1f6c6d17300e9cdd9f3f67ae113d8bc2bdb20f22fc2161da7f27a602b5d44be7b5073fb7814cb5f679040b3ee7db785fe6e
-  data.tar.gz: dda6fe83af93525aeff78ed47b00c36664accf5ee6be95214dbad5b115ccffb72dfd0e0a1f5e2830757267c9980710464602fe6df3ae4f8601cd111757a8cf92
+  metadata.gz: da503c49cffbf5dd1c1f438e51ff196c0167c2718a7e2856267b84117e042065fef188666ea9e60c9c14da8e3773452941fbd65207fdfb190eb577595142a568
+  data.tar.gz: 499e01ccd183f9993386d4c4ba129b015189512a4f33551b651f0e4c20ef2ab80e5cfb5a5ab05e825d17b08e62ecdbfdae5584d0d16e108257a302e530761f59

data/lib/rbbt/tsv/excel.rb

@@ -1,14 +1,42 @@
 require 'spreadsheet'
+require 'rubyXL'
+
 module TSV
-  def self.excel(tsv, filename, options ={})
+
+  def self._remove_link(value)
+    if value =~ /<([\w]+)[^>]*>(.*?)<\/\1>/
+      $2
+    else
+      value
+    end
+  end
+
+  def self._clean_float(v)
+    case v
+    when Float
+      v.to_s.sub(/e(-?\d+)$/,'E\1')
+    when String
+      if v =~ /^-?[\d\.]+e(-?\d+)$/
+        v.sub(/e(-?\d+)$/,'E\1') 
+      else
+        v
+      end
+    else
+      v
+    end
+  end
+
+
+  def self._excel_data(tsv, options ={})
+    options = Misc.add_defaults options, :sep2 => ', '
+
     name = Misc.process_options options, :name
+    sep2 = Misc.process_options options, :sep2
+    unmerge = Misc.process_options options, :unmerge
     sort_by = Misc.process_options options, :sort_by
     sort_by_cast = Misc.process_options options, :sort_by_cast
-    fields = Misc.process_options(options, :fields) || tsv.all_fields
+    remove_links = Misc.process_options options, :remove_links
 
-    book = Spreadsheet::Workbook.new
-    sheet1 = book.create_worksheet 
-    sheet1.row(0).concat fields
     i = 1
     if sort_by
       if sort_by_cast
@@ -26,7 +54,8 @@ module TSV
       data = tsv
     end
 
-    data.each do |key, values|
+    rows = []
+    data.through do |key, values|
       cells = []
       cells.push((name and key.respond_to?(:name)) ?  key.name || key : key )
 
@@ -34,86 +63,173 @@ module TSV
       values.each do |value|
         v = (name and value.respond_to?(:name)) ?  value.name || value : value 
         if Array === v
-          cells.push v * ", "
+          v = v.collect{|_v| _remove_link(_v)} if remove_links
+          v = v.collect{|_v| _clean_float(_v)} 
+          if unmerge
+            cells.push v
+          else
+            cells.push v * sep2
+          end
         else
+          v = remove_link(v) if remove_links
           cells.push v
         end
       end
 
-      sheet1.row(i).concat cells
+      rows << cells
       i += 1
     end
-    book.write filename
+    if unmerge
+      new_rows = []
+      rows.each do |row|
+        header = row.shift
+        Misc.zip_fields(row).each do |values|
+          new_rows << [header] + values
+        end
+      end
+      rows = new_rows
+    end
+    [tsv.all_fields, rows]
   end
 
-  def remove_link(value)
-    if value =~ /<([\w]+)[^>]*>(.*?)<\/\1>/
-      $2
-    else
-      value
+  module XLS
+    def self.read(file, options = {})
+      options = Misc.add_defaults options, :sep2 => /[,|]\s?/
+      sheet = Misc.process_options options, :sheet
+      header = Misc.process_options options, :header
+
+      header = true unless header == false
+      sheet ||= 0
+      TmpFile.with_file do |filename|
+        workbook = Spreadsheet.open Open.open(file)
+        sheet    = workbook.worksheet sheet
+
+        rows = []
+
+        sheet.each do |row|
+          rows << row.values_at(0..(row.size - 1))
+        end
+
+        File.open(filename, 'w') do |f|
+          if header
+            header = rows.shift
+            f.puts "#" + header * "\t"
+          end
+
+          rows.each do |row| f.puts row * "\t" end
+        end
+
+        TSV.open(filename, options)
+      end
+    end
+
+    def self.write(tsv, file, options = {})
+      options = Misc.add_defaults options, :sheet => "Sheet1"
+      sheet = Misc.process_options options, :sheet
+      fields, rows = TSV._excel_data(tsv, options)
+
+      book = Spreadsheet::Workbook.new
+      sheet1 = book.create_worksheet 
+      sheet1.name = sheet if sheet
+
+      sheet1.row(0).concat fields
+
+      rows.each_with_index do |cells,i|
+        sheet1.row(i+1).concat cells
+      end
+
+      book.write file
     end
   end
 
-  def excel(filename, options ={})
-    name = Misc.process_options options, :name
-    sort_by = Misc.process_options options, :sort_by
-    sort_by_cast = Misc.process_options options, :sort_by_cast
-    remove_links = Misc.process_options options, :remove_links
+  module XLSX
+    def self.read(file, options = {})
+      options = Misc.add_defaults options, :sep2 => /[,|]\s?/
+      sheet = Misc.process_options options, :sheet
+      header = Misc.process_options options, :header
 
-    book = Spreadsheet::Workbook.new
-    sheet1 = book.create_worksheet 
-    sheet1.row(0).concat all_fields
-    i = 1
-    if sort_by
-      if sort_by_cast
-        data = self.sort_by sort_by do |k, v| 
-          if Array === v
-            v.first.send(sort_by_cast)
-          else
-            v.send(sort_by_cast)
+      header = true unless header == false
+      TmpFile.with_file do |filename|
+        workbook = RubyXL::Parser.parse file
+        sheet    = sheet ? workbook[sheet] : workbook.worksheets.first
+
+        rows = []
+
+        sheet.each do |row|
+          rows << row.cells.collect{|c| c.value}
+        end
+
+        File.open(filename, 'w') do |f|
+          if header
+            header = rows.shift
+            f.puts "#" + header * "\t"
           end
+
+          rows.each do |row| f.puts row * "\t" end
         end
-      else
-        data = self.sort_by sort_by
+
+        TSV.open(filename, options)
       end
-    else
-      data = self
     end
 
-    data.through do |key, values|
-      cells = []
-      cells.push((name and key.respond_to?(:name)) ?  key.name || key : key )
+    def self.write(tsv, file, options = {})
+      sheet = Misc.process_options options, :sheet
 
-      values = [values] unless Array === values
-      values.each do |value|
-        v = (name and value.respond_to?(:name)) ?  value.name || value : value 
-        if Array === v
-          v = v.collect{|_v| remove_link(_v)} if remove_links
-          cells.push v * ", "
-        else
-          v = remove_link(v) if remove_links
-          cells.push v
-        end
+      fields, rows = TSV._excel_data(tsv, options)
+
+      book = RubyXL::Workbook.new
+      sheet1 = book.worksheets.first
+      sheet1.sheet_name = sheet if sheet
+
+      fields.each_with_index do |e,i|
+        sheet1.add_cell(0, i, e)
       end
 
-      cells = cells.collect do |v| 
-        case v
-        when Float
-          v.to_s.sub(/e(-?\d+)$/,'E\1')
-        when String
-          if v =~ /^-?[\d\.]+e(-?\d+)$/
-            v.sub(/e(-?\d+)$/,'E\1') 
-          else
-            v
-          end
-        else
-          v
+      rows.each_with_index do |cells,i|
+        cells.each_with_index do |e,j|
+          sheet1.add_cell(i+1, j, e)
         end
       end
 
-      sheet1.row(i).concat cells
-      i += 1
+      book.write file
+    end
+  end
+
+  def xls(filename, options ={})
+    TSV::XLS.write(self, filename, options)
+  end
+
+  def xlsx(filename, options ={})
+    TSV::XLSX.write(self, filename, options)
+  end
+
+  def excel(filename, options ={})
+    if filename =~ /\.xlsx$/
+      xlsx(filename, options)
+    else
+      xls(filename, options)
     end
-    book.write filename
   end
+
+
+  def self.xls(filename, options ={})
+    TSV::XLS.read(filename, options)
+  end
+
+  def self.xlsx(filename, options ={})
+    TSV::XLSX.read(filename, options)
+  end
+
+  def self.excel(filename, options = {})
+    if filename =~ /\.xlsx$/
+      xlsx(filename, options)
+    else
+      xls(filename, options)
+    end
+  end
+
+  def self.excel2tsv(filename, options ={})
+    excel(filename, options={})
+  end
+
 end

data/lib/rbbt/util/excel2tsv.rb

@@ -1,33 +1,2 @@
-require 'spreadsheet'
-require 'rbbt/tsv'
-require 'rbbt/util/tmpfile'
-
-module TSV
-  def self.excel2tsv(file, options = {})
-    sheet = options.delete :sheet
-    header = options.delete :header
-    header = true unless header == false
-    sheet ||= 0
-    TmpFile.with_file do |filename|
-      workbook = Spreadsheet.open Open.open(file)
-      sheet    = workbook.worksheet sheet
-
-      rows = []
-
-      sheet.each do |row|
-        rows << row.values_at(0..(row.size - 1))
-      end
-
-      File.open(filename, 'w') do |f|
-        if header
-          header = rows.shift
-          f.puts "#" + header * "\t"
-        end
-
-        rows.each do |row| f.puts row * "\t" end
-      end
-
-      TSV.open(filename, options)
-    end
-  end
-end
+Log.warn "File #{__FILE__} is deprecated, please require 'rbbt/tsv/excel' instead"
+require 'rbbt/tsv/excel'

data/lib/rbbt/util/open.rb

@@ -208,6 +208,7 @@ module Open
     if (dir_sub_path = find_repo_dir(file))
       stream = get_stream_from_repo(*dir_sub_path)
     else
+      file = file.find if Path === file
       stream =  File.open(file, mode)
     end
 

data/share/rbbt_commands/tsv/read

@@ -13,7 +13,7 @@ Use - to read from STDIN
 -k--key_field* Key field
 -f--fields* Fields
 -t--type* Type
--m--merge* Merge from multiple rows
+-m--merge Merge from multiple rows
 -h--help Print this help
 
 EOF

data/share/rbbt_commands/tsv/read_excel

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description
+
+$ rbbt tsv read_excel [options] <filename.xls|filename.xlsx>
+
+Use - to read from STDIN
+
+-k--key_field* Key field
+-f--fields* Fields
+-t--type* Type
+-m--merge Merge from multiple rows
+-s2--sep2* Second level separator. Default /[,|]\\s?/
+-h--help Print this help
+
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+excelfile = ARGV.first
+
+raise ParameterException, "No excel file given" if excelfile.nil?
+
+require 'rbbt/tsv/excel'
+puts TSV.excel(excelfile, options).to_s
+

data/share/rbbt_commands/tsv/write_excel

@@ -0,0 +1,62 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description
+
+$ rbbt tsv write_excel [options] <filename.tsv|-> <filename.xls|filename.xlsx>
+
+Use - to read from STDIN
+
+-k--key_field* Key field
+-f--fields* Fields
+-t--type* Type
+-m--merge Merge from multiple rows
+-um--unmerge Write the results unmerged
+-W--workflows* Load a list of workflows
+-R--requires* Require a list of files
+-n--name Transform entities to human readible names
+-h--help Print this help
+
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+if options[:workflows]
+  require 'rbbt/workflow'
+  workflows = options[:workflows].split(',')
+  workflows.each do |workflow|
+    workflow.strip!
+    Workflow.require_workflow workflow
+  end
+end
+
+if options[:requires]
+  requires = options[:requires].split(',')
+  requires.each do |req|
+    req.strip!
+    require req
+  end
+end
+
+tsvfile, excelfile = ARGV
+
+excelfile, tsvfile = tsvfile, nil if excelfile.nil? and tsvfile =~ /\.(xls|xlsx)$/
+
+raise ParameterException, "No files given" if tsvfile.nil? and excelfile.nil?
+raise ParameterException, "No excel file given" if excelfile.nil?
+
+require 'rbbt/tsv/excel'
+TSV.open(tsvfile, options).excel(excelfile, options)
+

data/test/rbbt/tsv/test_excel.rb

@@ -0,0 +1,137 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/tsv/excel'
+
+class TestExcel < Test::Unit::TestCase
+  def _test_xls
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/)
+      TmpFile.with_file(nil, true, :extension => 'xlsx') do |excelfile|
+        tsv.xls(excelfile)
+        new = TSV.xls(excelfile)
+        assert_equal %w(row1 row2), new.keys.sort
+      end
+    end
+  end
+
+  def _test_xlsx
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/)
+      TmpFile.with_file(nil, true, :extension => 'xlsx') do |excelfile|
+        tsv.xlsx(excelfile, :sheet => "Sheet1")
+        new = TSV.xlsx(excelfile, :sheet => "Sheet1")
+        assert_equal %w(row1 row2), new.keys.sort
+      end
+    end
+  end
+
+  def _test_excel
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/)
+      TmpFile.with_file(nil, true, :extension => 'xlsx') do |excelfile|
+        tsv.xlsx(excelfile, :sheet => "Sheet1")
+        new = TSV.excel(excelfile, :sheet => "Sheet1")
+        assert_equal %w(row1 row2), new.keys.sort
+        new = TSV.xlsx(excelfile, :sheet => "Sheet1")
+        assert_equal %w(row1 row2), new.keys.sort
+      end
+
+      TmpFile.with_file(nil, true, :extension => 'xls') do |excelfile|
+        tsv.xls(excelfile, :sheet => "Sheet1")
+        new = TSV.excel(excelfile, :sheet => "Sheet1")
+        assert_equal %w(row1 row2), new.keys.sort
+        new = TSV.xls(excelfile, :sheet => "Sheet1")
+        assert_equal %w(row1 row2), new.keys.sort
+      end
+    end
+  end
+  
+  def _test_excel_sheets
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/)
+
+      TmpFile.with_file(nil, false, :extension => 'xlsx') do |excelfile|
+        tsv.xlsx(excelfile)
+        new = TSV.excel(excelfile)
+        assert_equal %w(row1 row2), new.keys.sort
+
+        tsv.xlsx(excelfile, :sheet => "Test")
+        new = TSV.excel(excelfile, :sheet => "Test")
+        assert_equal %w(row1 row2), new.keys.sort
+
+        tsv.xlsx(excelfile, :sheet => "Test")
+        new = TSV.excel(excelfile)
+        assert_equal %w(row1 row2), new.keys.sort
+
+        tsv.xlsx(excelfile)
+        new = TSV.excel(excelfile, :sheet => "Sheet1")
+        assert_equal %w(row1 row2), new.keys.sort
+      end
+
+
+      TmpFile.with_file(nil, false, :extension => 'xls') do |excelfile|
+        tsv.xls(excelfile)
+        new = TSV.excel(excelfile)
+        assert_equal %w(row1 row2), new.keys.sort
+
+        tsv.xls(excelfile, :sheet => "Test")
+        new = TSV.excel(excelfile, :sheet => "Test")
+        assert_equal %w(row1 row2), new.keys.sort
+
+        tsv.xls(excelfile, :sheet => "Test")
+        new = TSV.excel(excelfile)
+        assert_equal %w(row1 row2), new.keys.sort
+
+        tsv.xls(excelfile)
+        new = TSV.excel(excelfile, :sheet => "Sheet1")
+        assert_equal %w(row1 row2), new.keys.sort
+      end
+    end
+  end
+
+  def test_excel_unmerge
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa    b|bb    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/)
+
+      TmpFile.with_file(nil, false, :extension => 'xlsx') do |excelfile|
+        tsv.xlsx(excelfile, :unmerge => true)
+
+        new = TSV.excel(excelfile, :merge => false)
+        assert_equal %w(a), new["row1"]["ValueA"]
+
+        new = TSV.excel(excelfile, :merge => true)
+        assert_equal %w(a aa), new["row1"]["ValueA"]
+      end
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.56
+  version: 5.21.57
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-24 00:00:00.000000000 Z
+date: 2016-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -360,11 +360,13 @@ files:
 - share/rbbt_commands/tsv/json
 - share/rbbt_commands/tsv/query
 - share/rbbt_commands/tsv/read
+- share/rbbt_commands/tsv/read_excel
 - share/rbbt_commands/tsv/slice
 - share/rbbt_commands/tsv/sort
 - share/rbbt_commands/tsv/subset
 - share/rbbt_commands/tsv/unzip
 - share/rbbt_commands/tsv/values
+- share/rbbt_commands/tsv/write_excel
 - share/rbbt_commands/tsv/zip
 - share/rbbt_commands/watch
 - share/rbbt_commands/workflow/cmd
@@ -421,6 +423,7 @@ files:
 - test/rbbt/tsv/test_accessor.rb
 - test/rbbt/tsv/test_attach.rb
 - test/rbbt/tsv/test_change_id.rb
+- test/rbbt/tsv/test_excel.rb
 - test/rbbt/tsv/test_field_index.rb
 - test/rbbt/tsv/test_filter.rb
 - test/rbbt/tsv/test_index.rb
@@ -559,6 +562,7 @@ test_files:
 - test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_stream.rb
 - test/rbbt/tsv/test_util.rb
+- test/rbbt/tsv/test_excel.rb
 - test/rbbt/tsv/test_accessor.rb
 - test/rbbt/tsv/test_filter.rb
 - test/rbbt/tsv/test_attach.rb