data_list_converter 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 7862bf6ab71eb4187b03ba25e7bd199ea9d3eb51
4
- data.tar.gz: f095138e55e9cc72c1691628c782a9c7ec5a203d
2
+ SHA256:
3
+ metadata.gz: 5acc292029d72d6edf08f6b588b4625bec532d8c9b3be03376c69c732ae4512f
4
+ data.tar.gz: 61850de2e9b3d1da73ecb42e638d4c473a93442dd12d1012bcec1065ad672bb9
5
5
  SHA512:
6
- metadata.gz: dfaa0989bafc3ae7e8d8675827b5728dd0476e1f62f79021b121407af2fa05ee4b47456f1db22aa46eb3aadfb4549a4ca3d0cf0c85bd0f8e6334c5105b36f1aa
7
- data.tar.gz: f9056d920a5164dc109490c809fd8d57174ffcc30d6851f52cb676937befce297e42ff89c702b60b2d2b825c3edc55b8b744538744296bb6c83c36e4a1bcf9ad
6
+ metadata.gz: 355008ec8cf509d617a023779a6ae2824e87c9b4b41d0ff636a1b1f1d2557ab624b9a02a01e15eb9aff731cd755e2e5c63fbb379c9e056455d59fd7836a18021
7
+ data.tar.gz: 8f1449e216cf16ff1a49198556880039806dd60adbf49c9edbfd638a6240c8c324297d5486dff4ab64f4360a93e8a81f90df933ef304e430fabf08997c764a64
data/README.md CHANGED
@@ -71,7 +71,7 @@ Please see [test examples](https://github.com/halida/data_list_converter/blob/ma
71
71
  - **save_to_file**: Save data to file, it will use `get_file_format` to find proper file format. `DataListConverter.save_to_file(filename, data, data_format=:item_data)`
72
72
  - **load_from_file**: Get data from file, it will use `get_file_format` to find proper file format. `DataListConverter.load_from_file(filename, data_format=:item_data)`
73
73
  - **unify_item_data_keys**: Sometimes in the `item_data` list, each data keys don't exactly same, so use this function to fix it, example: `DataListConverter.unify_item_data_keys([{a: 12}, {b: 11}]) == [{a: 12, b: nil}, {a: nil, b: 11}]`.
74
- - **flatten**: Flatten multi level item_data list into one level, example: `DataListConverter.flatten({a: {b: 12}, c: {d: {e: 11}}}) == {:"a:b"=>12, :"c:d:e"=>11}`, can change seperator: `DataListConverter.flatten(data, '_')`
74
+ - **flatten**: Flatten multi level item_data list into one level, example: `DataListConverter.flatten({a: {b: 12}, c: {d: {e: 11}}}) == {:"a:b"=>12, :"c:d:e"=>11}`, can change seperator: `DataListConverter.flatten(data, '_')`, set max level: `DataListConverter.flatten(data, '_', 2)`
75
75
 
76
76
  ## Extend
77
77
 
@@ -2,6 +2,9 @@
2
2
 
3
3
  Add parameter check, type check, change record type API.
4
4
 
5
+ - 0.4.1 add max_level to flatten helper
6
+ - 0.4.2 fix value cannot save if value is symbol
7
+
5
8
  ## 0.3
6
9
 
7
10
  Refactor.
@@ -19,6 +19,8 @@ Gem::Specification.new do |s|
19
19
  s.add_development_dependency 'pry', '~> 0.10.1'
20
20
  s.add_development_dependency 'sqlite3', '~> 1.3'
21
21
  s.add_development_dependency 'activerecord', '~> 4.2'
22
+ s.add_development_dependency 'xlsxtream'
23
+ s.add_development_dependency 'creek'
22
24
 
23
25
  s.files = `git ls-files`.split("\n")
24
26
  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
@@ -45,7 +45,7 @@ class DataListConverter
45
45
  def convert(from_type, to_type, from_value, options={})
46
46
  methods = []
47
47
  add_filter = lambda { |type|
48
- filters = (options[type] || {}).delete(:filter)
48
+ filters = (options[type] || {})[:filter]
49
49
  return unless filters
50
50
  methods += normalize_filters(type, filters)
51
51
  }
@@ -58,6 +58,7 @@ class DataListConverter
58
58
  from_type, to_type = route[i], route[i+1]
59
59
  method = CONVERTERS[[from_type, to_type]]
60
60
  raise "cannot find converter #{from_type} -> #{to_type}" unless method
61
+ self.log "#{from_type} -> #{to_type} options: #{options[to_type]}"
61
62
  methods.push([method, options[to_type] || {}])
62
63
  add_filter.call(to_type)
63
64
  end
@@ -1,14 +1,14 @@
1
1
  class DataListConverter
2
- def self.save_to_file(filename, data, data_format=:item_data)
2
+ def self.save_to_file(filename, data, data_format=:item_data, options={})
3
3
  file_format = self.get_file_format(filename)
4
- DataListConverter.convert(data_format, file_format, data,
5
- file_format => {filename: filename})
4
+ options[file_format] = {filename: filename}
5
+ DataListConverter.convert(data_format, file_format, data, options)
6
6
  end
7
7
 
8
- def self.load_from_file(filename, data_format=:item_data)
8
+ def self.load_from_file(filename, data_format=:item_data, options={})
9
9
  file_format = self.get_file_format(filename)
10
- DataListConverter.convert(file_format, data_format,
11
- {filename: filename})
10
+ options[:filename] = filename
11
+ DataListConverter.convert(file_format, data_format, options)
12
12
  end
13
13
 
14
14
  def self.get_file_format(filename)
@@ -55,18 +55,17 @@ class DataListConverter
55
55
  # {a: {b: 12}, c: {d: {e: 11}}}
56
56
  # =>
57
57
  # {:"a:b"=>12, :"c:d:e"=>11}
58
- def self.flatten(data, sep=':')
58
+ def self.flatten(data, sep=':', max_level=nil)
59
59
  out = {}
60
- recursive_flatten(out, data, nil, sep)
60
+ recursive_flatten(out, data, nil, sep, 1, max_level)
61
61
  out
62
62
  end
63
63
 
64
- def self.recursive_flatten(out, data, header, sep)
64
+ def self.recursive_flatten(out, data, header, sep, level, max_level)
65
65
  data.each do |k, v|
66
66
  k = header ? :"#{header}#{sep}#{k}" : k
67
- case v
68
- when Hash
69
- recursive_flatten(out, v, k, sep)
67
+ if v.kind_of?(Hash) and (!max_level or level <= max_level)
68
+ recursive_flatten(out, v, k, sep, level+1, max_level)
70
69
  else
71
70
  out[k] = v
72
71
  end
@@ -0,0 +1,58 @@
1
+ require 'xlsxtream'
2
+ require 'creek'
3
+
4
+ class DataListConverter
5
+
6
+ self.register_converter(:xlsx_file, :table_iterator) do |input, options|
7
+ lambda { |&block|
8
+ filename = self.parameter(input, :filename, :input)
9
+
10
+ creek = Creek::Book.new filename
11
+ sheet = creek.sheets[input[:sheet] || 0]
12
+ sheet.rows.each do |row|
13
+ block.call(row.values)
14
+ end
15
+ }
16
+ end
17
+
18
+ self.register_converter(:xlsx_file, :multi_sheet_table_iterator) do |input, options|
19
+ filename = self.parameter(input, :filename, :input)
20
+
21
+ creek = Creek::Book.new filename
22
+ creek.sheets.map do |sheet|
23
+ iterator = lambda { |&block|
24
+ sheet.rows.each do |row|
25
+ block.call(row.values)
26
+ end
27
+ }
28
+ [sheet.name.to_sym, iterator]
29
+ end.to_h
30
+ end
31
+
32
+ self.register_converter(:table_iterator, :xlsx_file) do |proc, options|
33
+ filename = self.parameter(options, :filename, :xlsx_file)
34
+ Xlsxtream::Workbook.open(filename) do |xlsx|
35
+ xlsx.write_worksheet (options[:sheet] || "Sheet1") do |sheet|
36
+ proc.call do |row|
37
+ sheet << row
38
+ end
39
+ end
40
+ end
41
+ filename
42
+ end
43
+
44
+ self.register_converter(:multi_sheet_table_iterator, :xlsx_file) do |data, options|
45
+ filename = self.parameter(options, :filename, :xlsx_file)
46
+ Xlsxtream::Workbook.open(filename) do |xlsx|
47
+ data.each do |name, table_iterator|
48
+ xlsx.write_worksheet(name.to_s) do |sheet|
49
+ table_iterator.call do |row|
50
+ sheet << row
51
+ end
52
+ end
53
+ end
54
+ end
55
+ filename
56
+ end
57
+
58
+ end
@@ -19,8 +19,9 @@ class DataListConverter
19
19
  :"multi_sheet_#{from_type}",
20
20
  :"multi_sheet_#{to_type}",
21
21
  ) do |data, options|
22
+ self.log("multi_sheet #{from_type} -> #{to_type} with options: #{options}")
22
23
  data.map do |sheet, from_data|
23
- to_data = self.convert(from_type, to_type, from_data)
24
+ to_data = self.convert(from_type, to_type, from_data, options)
24
25
  [sheet, to_data]
25
26
  end.to_h
26
27
  end
@@ -19,6 +19,7 @@ class DataListConverter
19
19
  sheet = book.create_worksheet(name: (options[:sheet] || "Sheet1"))
20
20
  i = 0
21
21
  proc.call do |row|
22
+ row = row.map(&:to_s)
22
23
  sheet.row(i).push *row
23
24
  i += 1
24
25
  end
@@ -33,6 +34,7 @@ class DataListConverter
33
34
  sheet = book.create_worksheet(name: name.to_s)
34
35
  i = 0
35
36
  table_iterator.call do |row|
37
+ row = row.map(&:to_s)
36
38
  sheet.row(i).concat(row)
37
39
  i += 1
38
40
  end
@@ -23,7 +23,7 @@ class DataListConverter
23
23
  i = 0
24
24
  proc.call do |row|
25
25
  row.each_with_index do |v, j|
26
- sheet.add_cell(i, j, v)
26
+ sheet.add_cell(i, j, v.to_s)
27
27
  end
28
28
  i += 1
29
29
  end
@@ -42,14 +42,14 @@ class DataListConverter
42
42
  row.each_with_index do |v, j|
43
43
  if v.kind_of?(Hash)
44
44
  # custom cell format
45
- cell = sheet.add_cell(i, j, v[:text])
45
+ cell = sheet.add_cell(i, j, v[:text].to_s)
46
46
  v.each do |k, v|
47
47
  next if k == :text
48
48
  cell.send(k, v)
49
49
  end
50
50
  cell.change_fill(v[:fill_color]) if v[:fill_color]
51
51
  else
52
- cell = sheet.add_cell(i, j, v)
52
+ cell = sheet.add_cell(i, j, v.to_s)
53
53
  end
54
54
 
55
55
  end
@@ -1,3 +1,3 @@
1
1
  class DataListConverter
2
- VERSION = "0.4.0".freeze
2
+ VERSION = "0.5.0".freeze
3
3
  end
@@ -26,12 +26,30 @@ describe DataListConverter do
26
26
  string = StringIO.new
27
27
  filter = {count: {size: 4000,
28
28
  out: string,
29
- msg: "%{percent}%"}}
29
+ msg: "%{percent}%%"}}
30
30
  result = DataListConverter.convert(
31
31
  :item_iterator, :table_data, iter,
32
32
  item_iterator: {filter: filter})
33
33
  string.string.split("\n").must_equal ['total: 10000', '40.0%', '80.0%']
34
34
  end
35
35
  end
36
+
37
+ describe :remove_debug do
38
+ specify do
39
+ filter = {remove_debug: true}
40
+ item_data = [{name: "james", debug: "", a: 12}] * 2
41
+ result = DataListConverter.convert(
42
+ :item_data, :table_data, item_data,
43
+ table_iterator: {filter: filter})
44
+ result.must_equal [["name"], ["james"], ["james"]]
45
+
46
+ # check on multi_sheet
47
+ item_data = {a: [{name: "james", debug: "", a: 12}] * 2, b: [{name: 'cc', debug: "", b: 3}]*3}
48
+ result = DataListConverter.convert(
49
+ :multi_sheet_item_data, :multi_sheet_table_data, item_data,
50
+ multi_sheet_table_iterator: {table_iterator: {filter: {remove_debug: true}}})
51
+ result.must_equal(a: [["name"], ["james"], ["james"]], b: [["name"], ["cc"], ["cc"], ["cc"]])
52
+ end
53
+ end
36
54
 
37
55
  end
@@ -79,6 +79,12 @@ describe DataListConverter do
79
79
  # change sep
80
80
  data = {a: {b: 12}, c: {d: {e: 11}}}
81
81
  @c.flatten(data, '_').must_equal({:"a_b"=>12, :"c_d_e"=>11})
82
+
83
+ # set max level
84
+ data = {a: {b: 12}, c: {d: {e: {f: 11}}}}
85
+ @c.flatten(data, ':', 1).must_equal({:"a:b"=>12, :"c:d"=>{e: {f: 11}}})
86
+ data = {a: {b: 12}, c: {d: {e: {f: 11}}}}
87
+ @c.flatten(data, ':', 2).must_equal({:"a:b"=>12, :"c:d:e"=>{f: 11}})
82
88
  end
83
89
  end
84
90
  end
@@ -79,6 +79,27 @@ describe DataListConverter do
79
79
  end
80
80
  end
81
81
 
82
+ require 'data_list_converter/types/fast_xlsx_file'
83
+
84
+ describe :xlsx_file do
85
+ specify do
86
+ filename = 'test.xlsx'
87
+ begin
88
+ @c.convert(:item_data, :xlsx_file, ITEM_DATA, xlsx_file: {filename: filename})
89
+ @c.convert(:xlsx_file, :item_data, {filename: filename}).must_equal ITEM_DATA
90
+
91
+ @c.convert(:multi_sheet_table_data, :xlsx_file, MULTI_SHEET_TABLE_DATA,
92
+ xlsx_file: {filename: filename})
93
+ @c.convert(:xlsx_file, :multi_sheet_table_data,
94
+ {filename: filename},
95
+ ).must_equal(MULTI_SHEET_TABLE_DATA)
96
+ ensure
97
+ FileUtils.rm_f(filename)
98
+ end
99
+ end
100
+
101
+ end
102
+
82
103
  describe :marshal do
83
104
  specify do
84
105
  filename = 'test.marshal'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: data_list_converter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.0
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - linjunhalida
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-02 00:00:00.000000000 Z
11
+ date: 2020-12-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -108,6 +108,34 @@ dependencies:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
110
  version: '4.2'
111
+ - !ruby/object:Gem::Dependency
112
+ name: xlsxtream
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: creek
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
111
139
  description: Data List Converter is a tool to convert data between different formats.
112
140
  email:
113
141
  - linjunhalida@gmail.com
@@ -130,6 +158,7 @@ files:
130
158
  - lib/data_list_converter/helper.rb
131
159
  - lib/data_list_converter/types/basic.rb
132
160
  - lib/data_list_converter/types/csv_file.rb
161
+ - lib/data_list_converter/types/fast_xlsx_file.rb
133
162
  - lib/data_list_converter/types/marshal.rb
134
163
  - lib/data_list_converter/types/multi_sheet.rb
135
164
  - lib/data_list_converter/types/records.rb
@@ -160,14 +189,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
160
189
  - !ruby/object:Gem::Version
161
190
  version: '0'
162
191
  requirements: []
163
- rubyforge_project:
164
- rubygems_version: 2.4.8
192
+ rubygems_version: 3.0.8
165
193
  signing_key:
166
194
  specification_version: 4
167
195
  summary: convert data between different formats
168
- test_files:
169
- - test/base_test.rb
170
- - test/filters_test.rb
171
- - test/helper_test.rb
172
- - test/testdata.rb
173
- - test/types_test.rb
196
+ test_files: []