data_list_converter 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 7862bf6ab71eb4187b03ba25e7bd199ea9d3eb51
-  data.tar.gz: f095138e55e9cc72c1691628c782a9c7ec5a203d
+SHA256:
+  metadata.gz: 5acc292029d72d6edf08f6b588b4625bec532d8c9b3be03376c69c732ae4512f
+  data.tar.gz: 61850de2e9b3d1da73ecb42e638d4c473a93442dd12d1012bcec1065ad672bb9
 SHA512:
-  metadata.gz: dfaa0989bafc3ae7e8d8675827b5728dd0476e1f62f79021b121407af2fa05ee4b47456f1db22aa46eb3aadfb4549a4ca3d0cf0c85bd0f8e6334c5105b36f1aa
-  data.tar.gz: f9056d920a5164dc109490c809fd8d57174ffcc30d6851f52cb676937befce297e42ff89c702b60b2d2b825c3edc55b8b744538744296bb6c83c36e4a1bcf9ad
+  metadata.gz: 355008ec8cf509d617a023779a6ae2824e87c9b4b41d0ff636a1b1f1d2557ab624b9a02a01e15eb9aff731cd755e2e5c63fbb379c9e056455d59fd7836a18021
+  data.tar.gz: 8f1449e216cf16ff1a49198556880039806dd60adbf49c9edbfd638a6240c8c324297d5486dff4ab64f4360a93e8a81f90df933ef304e430fabf08997c764a64

data/README.md

@@ -71,7 +71,7 @@ Please see [test examples](https://github.com/halida/data_list_converter/blob/ma
 - **save_to_file**: Save data to file, it will use `get_file_format` to find proper file format.  `DataListConverter.save_to_file(filename, data, data_format=:item_data)`
 - **load_from_file**: Get data from file, it will use `get_file_format` to find proper file format. `DataListConverter.load_from_file(filename, data_format=:item_data)`
 - **unify_item_data_keys**: Sometimes in the `item_data` list, each data keys don't exactly same, so use this function to fix it, example: `DataListConverter.unify_item_data_keys([{a: 12}, {b: 11}]) == [{a: 12, b: nil}, {a: nil, b: 11}]`.
-- **flatten**: Flatten multi level item_data list into one level, example: `DataListConverter.flatten({a: {b: 12}, c: {d: {e: 11}}}) == {:"a:b"=>12, :"c:d:e"=>11}`, can change seperator: `DataListConverter.flatten(data, '_')`
+- **flatten**: Flatten multi level item_data list into one level, example: `DataListConverter.flatten({a: {b: 12}, c: {d: {e: 11}}}) == {:"a:b"=>12, :"c:d:e"=>11}`, can change seperator: `DataListConverter.flatten(data, '_')`, set max level: `DataListConverter.flatten(data, '_', 2)`
 
 ## Extend
 

data/changelog.md

@@ -2,6 +2,9 @@
 
 Add parameter check, type check, change record type API.
 
+- 0.4.1 add max_level to flatten helper
+- 0.4.2 fix value cannot save if value is symbol
+
 ## 0.3
 
 Refactor.

data/data_list_converter.gemspec

@@ -19,6 +19,8 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'pry', '~> 0.10.1'
   s.add_development_dependency 'sqlite3', '~> 1.3'
   s.add_development_dependency 'activerecord', '~> 4.2'
+  s.add_development_dependency 'xlsxtream'
+  s.add_development_dependency 'creek'
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/data_list_converter/base.rb

@@ -45,7 +45,7 @@ class DataListConverter
     def convert(from_type, to_type, from_value, options={})
       methods = []
       add_filter = lambda { |type|
-        filters = (options[type] || {}).delete(:filter)
+        filters = (options[type] || {})[:filter]
         return unless filters
         methods += normalize_filters(type, filters)
       }
@@ -58,6 +58,7 @@ class DataListConverter
         from_type, to_type = route[i], route[i+1]
         method = CONVERTERS[[from_type, to_type]]
         raise "cannot find converter #{from_type} -> #{to_type}" unless method
+        self.log "#{from_type} -> #{to_type} options: #{options[to_type]}"
         methods.push([method, options[to_type] || {}])
         add_filter.call(to_type)
       end

data/lib/data_list_converter/helper.rb

@@ -1,14 +1,14 @@
 class DataListConverter
-  def self.save_to_file(filename, data, data_format=:item_data)
+  def self.save_to_file(filename, data, data_format=:item_data, options={})
     file_format = self.get_file_format(filename)
-    DataListConverter.convert(data_format, file_format, data,
-                              file_format => {filename: filename})
+    options[file_format] = {filename: filename}
+    DataListConverter.convert(data_format, file_format, data, options)
   end
 
-  def self.load_from_file(filename, data_format=:item_data)
+  def self.load_from_file(filename, data_format=:item_data, options={})
     file_format = self.get_file_format(filename)
-    DataListConverter.convert(file_format, data_format,
-                              {filename: filename})
+    options[:filename] = filename
+    DataListConverter.convert(file_format, data_format, options)
   end
 
   def self.get_file_format(filename)
@@ -55,18 +55,17 @@ class DataListConverter
   #   {a: {b: 12}, c: {d: {e: 11}}}
   #   =>
   #   {:"a:b"=>12, :"c:d:e"=>11}
-  def self.flatten(data, sep=':')
+  def self.flatten(data, sep=':', max_level=nil)
     out = {}
-    recursive_flatten(out, data, nil, sep)
+    recursive_flatten(out, data, nil, sep, 1, max_level)
     out
   end
 
-  def self.recursive_flatten(out, data, header, sep)
+  def self.recursive_flatten(out, data, header, sep, level, max_level)
     data.each do |k, v|
       k = header ? :"#{header}#{sep}#{k}" : k
-      case v
-      when Hash
-        recursive_flatten(out, v, k, sep)
+      if v.kind_of?(Hash) and (!max_level or level <= max_level)
+        recursive_flatten(out, v, k, sep, level+1, max_level)
       else
         out[k] = v
       end

data/lib/data_list_converter/types/fast_xlsx_file.rb

@@ -0,0 +1,58 @@
+require 'xlsxtream'
+require 'creek'
+
+class DataListConverter
+
+  self.register_converter(:xlsx_file, :table_iterator) do |input, options|
+    lambda { |&block|
+      filename = self.parameter(input, :filename, :input)
+
+      creek = Creek::Book.new filename
+      sheet = creek.sheets[input[:sheet] || 0]
+      sheet.rows.each do |row|
+        block.call(row.values)
+      end
+    }
+  end
+
+  self.register_converter(:xlsx_file, :multi_sheet_table_iterator) do |input, options|
+    filename = self.parameter(input, :filename, :input)
+
+    creek = Creek::Book.new filename
+    creek.sheets.map do |sheet|
+      iterator = lambda { |&block|
+        sheet.rows.each do |row|
+          block.call(row.values)
+        end
+      }
+      [sheet.name.to_sym, iterator]
+    end.to_h
+  end
+
+  self.register_converter(:table_iterator, :xlsx_file) do |proc, options|
+    filename = self.parameter(options, :filename, :xlsx_file)
+    Xlsxtream::Workbook.open(filename) do |xlsx|
+      xlsx.write_worksheet (options[:sheet] || "Sheet1") do |sheet|
+        proc.call do |row|
+          sheet << row
+        end
+      end
+    end
+    filename
+  end
+
+  self.register_converter(:multi_sheet_table_iterator, :xlsx_file) do |data, options|
+    filename = self.parameter(options, :filename, :xlsx_file)
+    Xlsxtream::Workbook.open(filename) do |xlsx|
+      data.each do |name, table_iterator|
+        xlsx.write_worksheet(name.to_s) do |sheet|
+          table_iterator.call do |row|
+            sheet << row
+          end
+        end
+      end
+    end
+    filename
+  end
+
+end

data/lib/data_list_converter/types/multi_sheet.rb

@@ -19,8 +19,9 @@ class DataListConverter
       :"multi_sheet_#{from_type}",
       :"multi_sheet_#{to_type}",
     ) do |data, options|
+      self.log("multi_sheet #{from_type} -> #{to_type} with options: #{options}")
       data.map do |sheet, from_data|
-        to_data = self.convert(from_type, to_type, from_data)
+        to_data = self.convert(from_type, to_type, from_data, options)
         [sheet, to_data]
       end.to_h
     end

data/lib/data_list_converter/types/xls_file.rb

@@ -19,6 +19,7 @@ class DataListConverter
     sheet = book.create_worksheet(name: (options[:sheet] || "Sheet1"))
     i = 0
     proc.call do |row|
+      row = row.map(&:to_s)
       sheet.row(i).push *row
       i += 1
     end
@@ -33,6 +34,7 @@ class DataListConverter
       sheet = book.create_worksheet(name: name.to_s)
       i = 0
       table_iterator.call do |row|
+        row = row.map(&:to_s)
         sheet.row(i).concat(row)
         i += 1
       end

data/lib/data_list_converter/types/xlsx_file.rb

@@ -23,7 +23,7 @@ class DataListConverter
     i = 0
     proc.call do |row|
       row.each_with_index do |v, j|
-        sheet.add_cell(i, j, v)
+        sheet.add_cell(i, j, v.to_s)
       end
       i += 1
     end
@@ -42,14 +42,14 @@ class DataListConverter
         row.each_with_index do |v, j|
           if v.kind_of?(Hash)
             # custom cell format
-            cell = sheet.add_cell(i, j, v[:text])
+            cell = sheet.add_cell(i, j, v[:text].to_s)
             v.each do |k, v|
               next if k == :text
               cell.send(k, v)
             end
             cell.change_fill(v[:fill_color]) if v[:fill_color]
           else
-            cell = sheet.add_cell(i, j, v)
+            cell = sheet.add_cell(i, j, v.to_s)
           end
 
         end

data/lib/data_list_converter/version.rb

@@ -1,3 +1,3 @@
 class DataListConverter
-  VERSION = "0.4.0".freeze
+  VERSION = "0.5.0".freeze
 end

data/test/filters_test.rb

@@ -26,12 +26,30 @@ describe DataListConverter do
       string = StringIO.new
       filter = {count: {size: 4000,
                         out: string,
-                        msg: "%{percent}%"}}
+                        msg: "%{percent}%%"}}
       result = DataListConverter.convert(
         :item_iterator, :table_data, iter,
         item_iterator: {filter: filter})
       string.string.split("\n").must_equal ['total: 10000', '40.0%', '80.0%']
     end
   end
+
+  describe :remove_debug do
+    specify do
+      filter = {remove_debug: true}
+      item_data = [{name: "james", debug: "", a: 12}] * 2
+      result = DataListConverter.convert(
+        :item_data, :table_data, item_data,
+        table_iterator: {filter: filter})
+      result.must_equal [["name"], ["james"], ["james"]]
+
+      # check on multi_sheet
+      item_data = {a: [{name: "james", debug: "", a: 12}] * 2, b: [{name: 'cc', debug: "", b: 3}]*3}
+      result = DataListConverter.convert(
+        :multi_sheet_item_data, :multi_sheet_table_data, item_data,
+        multi_sheet_table_iterator: {table_iterator: {filter: {remove_debug: true}}})
+      result.must_equal(a: [["name"], ["james"], ["james"]], b: [["name"], ["cc"], ["cc"], ["cc"]])
+    end
+  end
   
 end

data/test/helper_test.rb

@@ -79,6 +79,12 @@ describe DataListConverter do
       # change sep
       data = {a: {b: 12}, c: {d: {e: 11}}}
       @c.flatten(data, '_').must_equal({:"a_b"=>12, :"c_d_e"=>11})
+
+      # set max level
+      data = {a: {b: 12}, c: {d: {e: {f: 11}}}}
+      @c.flatten(data, ':', 1).must_equal({:"a:b"=>12, :"c:d"=>{e: {f: 11}}})
+      data = {a: {b: 12}, c: {d: {e: {f: 11}}}}
+      @c.flatten(data, ':', 2).must_equal({:"a:b"=>12, :"c:d:e"=>{f: 11}})
     end
   end
 end

data/test/types_test.rb

@@ -79,6 +79,27 @@ describe DataListConverter do
     end
   end
 
+  require 'data_list_converter/types/fast_xlsx_file'
+
+  describe :xlsx_file do
+    specify do
+      filename = 'test.xlsx'
+      begin
+        @c.convert(:item_data, :xlsx_file, ITEM_DATA, xlsx_file: {filename: filename})
+        @c.convert(:xlsx_file, :item_data, {filename: filename}).must_equal ITEM_DATA
+
+        @c.convert(:multi_sheet_table_data, :xlsx_file, MULTI_SHEET_TABLE_DATA,
+                   xlsx_file: {filename: filename})
+        @c.convert(:xlsx_file, :multi_sheet_table_data,
+                   {filename: filename},
+                  ).must_equal(MULTI_SHEET_TABLE_DATA)
+      ensure
+        FileUtils.rm_f(filename)
+      end
+    end
+
+  end
+
   describe :marshal do
     specify do
       filename = 'test.marshal'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_list_converter
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - linjunhalida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-02 00:00:00.000000000 Z
+date: 2020-12-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -108,6 +108,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: xlsxtream
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: creek
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Data List Converter is a tool to convert data between different formats.
 email:
 - linjunhalida@gmail.com
@@ -130,6 +158,7 @@ files:
 - lib/data_list_converter/helper.rb
 - lib/data_list_converter/types/basic.rb
 - lib/data_list_converter/types/csv_file.rb
+- lib/data_list_converter/types/fast_xlsx_file.rb
 - lib/data_list_converter/types/marshal.rb
 - lib/data_list_converter/types/multi_sheet.rb
 - lib/data_list_converter/types/records.rb
@@ -160,14 +189,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: convert data between different formats
-test_files:
-- test/base_test.rb
-- test/filters_test.rb
-- test/helper_test.rb
-- test/testdata.rb
-- test/types_test.rb
+test_files: []