data_list_converter 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +1 -1
- data/changelog.md +3 -0
- data/data_list_converter.gemspec +2 -0
- data/lib/data_list_converter/base.rb +2 -1
- data/lib/data_list_converter/helper.rb +11 -12
- data/lib/data_list_converter/types/fast_xlsx_file.rb +58 -0
- data/lib/data_list_converter/types/multi_sheet.rb +2 -1
- data/lib/data_list_converter/types/xls_file.rb +2 -0
- data/lib/data_list_converter/types/xlsx_file.rb +3 -3
- data/lib/data_list_converter/version.rb +1 -1
- data/test/filters_test.rb +19 -1
- data/test/helper_test.rb +6 -0
- data/test/types_test.rb +21 -0
- metadata +33 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 5acc292029d72d6edf08f6b588b4625bec532d8c9b3be03376c69c732ae4512f
|
4
|
+
data.tar.gz: 61850de2e9b3d1da73ecb42e638d4c473a93442dd12d1012bcec1065ad672bb9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 355008ec8cf509d617a023779a6ae2824e87c9b4b41d0ff636a1b1f1d2557ab624b9a02a01e15eb9aff731cd755e2e5c63fbb379c9e056455d59fd7836a18021
|
7
|
+
data.tar.gz: 8f1449e216cf16ff1a49198556880039806dd60adbf49c9edbfd638a6240c8c324297d5486dff4ab64f4360a93e8a81f90df933ef304e430fabf08997c764a64
|
data/README.md
CHANGED
@@ -71,7 +71,7 @@ Please see [test examples](https://github.com/halida/data_list_converter/blob/ma
|
|
71
71
|
- **save_to_file**: Save data to file, it will use `get_file_format` to find proper file format. `DataListConverter.save_to_file(filename, data, data_format=:item_data)`
|
72
72
|
- **load_from_file**: Get data from file, it will use `get_file_format` to find proper file format. `DataListConverter.load_from_file(filename, data_format=:item_data)`
|
73
73
|
- **unify_item_data_keys**: Sometimes in the `item_data` list, each data keys don't exactly same, so use this function to fix it, example: `DataListConverter.unify_item_data_keys([{a: 12}, {b: 11}]) == [{a: 12, b: nil}, {a: nil, b: 11}]`.
|
74
|
-
- **flatten**: Flatten multi level item_data list into one level, example: `DataListConverter.flatten({a: {b: 12}, c: {d: {e: 11}}}) == {:"a:b"=>12, :"c:d:e"=>11}`, can change seperator: `DataListConverter.flatten(data, '_')`
|
74
|
+
- **flatten**: Flatten multi level item_data list into one level, example: `DataListConverter.flatten({a: {b: 12}, c: {d: {e: 11}}}) == {:"a:b"=>12, :"c:d:e"=>11}`, can change seperator: `DataListConverter.flatten(data, '_')`, set max level: `DataListConverter.flatten(data, '_', 2)`
|
75
75
|
|
76
76
|
## Extend
|
77
77
|
|
data/changelog.md
CHANGED
data/data_list_converter.gemspec
CHANGED
@@ -19,6 +19,8 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.add_development_dependency 'pry', '~> 0.10.1'
|
20
20
|
s.add_development_dependency 'sqlite3', '~> 1.3'
|
21
21
|
s.add_development_dependency 'activerecord', '~> 4.2'
|
22
|
+
s.add_development_dependency 'xlsxtream'
|
23
|
+
s.add_development_dependency 'creek'
|
22
24
|
|
23
25
|
s.files = `git ls-files`.split("\n")
|
24
26
|
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
@@ -45,7 +45,7 @@ class DataListConverter
|
|
45
45
|
def convert(from_type, to_type, from_value, options={})
|
46
46
|
methods = []
|
47
47
|
add_filter = lambda { |type|
|
48
|
-
filters = (options[type] || {})
|
48
|
+
filters = (options[type] || {})[:filter]
|
49
49
|
return unless filters
|
50
50
|
methods += normalize_filters(type, filters)
|
51
51
|
}
|
@@ -58,6 +58,7 @@ class DataListConverter
|
|
58
58
|
from_type, to_type = route[i], route[i+1]
|
59
59
|
method = CONVERTERS[[from_type, to_type]]
|
60
60
|
raise "cannot find converter #{from_type} -> #{to_type}" unless method
|
61
|
+
self.log "#{from_type} -> #{to_type} options: #{options[to_type]}"
|
61
62
|
methods.push([method, options[to_type] || {}])
|
62
63
|
add_filter.call(to_type)
|
63
64
|
end
|
@@ -1,14 +1,14 @@
|
|
1
1
|
class DataListConverter
|
2
|
-
def self.save_to_file(filename, data, data_format=:item_data)
|
2
|
+
def self.save_to_file(filename, data, data_format=:item_data, options={})
|
3
3
|
file_format = self.get_file_format(filename)
|
4
|
-
|
5
|
-
|
4
|
+
options[file_format] = {filename: filename}
|
5
|
+
DataListConverter.convert(data_format, file_format, data, options)
|
6
6
|
end
|
7
7
|
|
8
|
-
def self.load_from_file(filename, data_format=:item_data)
|
8
|
+
def self.load_from_file(filename, data_format=:item_data, options={})
|
9
9
|
file_format = self.get_file_format(filename)
|
10
|
-
|
11
|
-
|
10
|
+
options[:filename] = filename
|
11
|
+
DataListConverter.convert(file_format, data_format, options)
|
12
12
|
end
|
13
13
|
|
14
14
|
def self.get_file_format(filename)
|
@@ -55,18 +55,17 @@ class DataListConverter
|
|
55
55
|
# {a: {b: 12}, c: {d: {e: 11}}}
|
56
56
|
# =>
|
57
57
|
# {:"a:b"=>12, :"c:d:e"=>11}
|
58
|
-
def self.flatten(data, sep=':')
|
58
|
+
def self.flatten(data, sep=':', max_level=nil)
|
59
59
|
out = {}
|
60
|
-
recursive_flatten(out, data, nil, sep)
|
60
|
+
recursive_flatten(out, data, nil, sep, 1, max_level)
|
61
61
|
out
|
62
62
|
end
|
63
63
|
|
64
|
-
def self.recursive_flatten(out, data, header, sep)
|
64
|
+
def self.recursive_flatten(out, data, header, sep, level, max_level)
|
65
65
|
data.each do |k, v|
|
66
66
|
k = header ? :"#{header}#{sep}#{k}" : k
|
67
|
-
|
68
|
-
|
69
|
-
recursive_flatten(out, v, k, sep)
|
67
|
+
if v.kind_of?(Hash) and (!max_level or level <= max_level)
|
68
|
+
recursive_flatten(out, v, k, sep, level+1, max_level)
|
70
69
|
else
|
71
70
|
out[k] = v
|
72
71
|
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'xlsxtream'
|
2
|
+
require 'creek'
|
3
|
+
|
4
|
+
class DataListConverter
|
5
|
+
|
6
|
+
self.register_converter(:xlsx_file, :table_iterator) do |input, options|
|
7
|
+
lambda { |&block|
|
8
|
+
filename = self.parameter(input, :filename, :input)
|
9
|
+
|
10
|
+
creek = Creek::Book.new filename
|
11
|
+
sheet = creek.sheets[input[:sheet] || 0]
|
12
|
+
sheet.rows.each do |row|
|
13
|
+
block.call(row.values)
|
14
|
+
end
|
15
|
+
}
|
16
|
+
end
|
17
|
+
|
18
|
+
self.register_converter(:xlsx_file, :multi_sheet_table_iterator) do |input, options|
|
19
|
+
filename = self.parameter(input, :filename, :input)
|
20
|
+
|
21
|
+
creek = Creek::Book.new filename
|
22
|
+
creek.sheets.map do |sheet|
|
23
|
+
iterator = lambda { |&block|
|
24
|
+
sheet.rows.each do |row|
|
25
|
+
block.call(row.values)
|
26
|
+
end
|
27
|
+
}
|
28
|
+
[sheet.name.to_sym, iterator]
|
29
|
+
end.to_h
|
30
|
+
end
|
31
|
+
|
32
|
+
self.register_converter(:table_iterator, :xlsx_file) do |proc, options|
|
33
|
+
filename = self.parameter(options, :filename, :xlsx_file)
|
34
|
+
Xlsxtream::Workbook.open(filename) do |xlsx|
|
35
|
+
xlsx.write_worksheet (options[:sheet] || "Sheet1") do |sheet|
|
36
|
+
proc.call do |row|
|
37
|
+
sheet << row
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
filename
|
42
|
+
end
|
43
|
+
|
44
|
+
self.register_converter(:multi_sheet_table_iterator, :xlsx_file) do |data, options|
|
45
|
+
filename = self.parameter(options, :filename, :xlsx_file)
|
46
|
+
Xlsxtream::Workbook.open(filename) do |xlsx|
|
47
|
+
data.each do |name, table_iterator|
|
48
|
+
xlsx.write_worksheet(name.to_s) do |sheet|
|
49
|
+
table_iterator.call do |row|
|
50
|
+
sheet << row
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
filename
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
@@ -19,8 +19,9 @@ class DataListConverter
|
|
19
19
|
:"multi_sheet_#{from_type}",
|
20
20
|
:"multi_sheet_#{to_type}",
|
21
21
|
) do |data, options|
|
22
|
+
self.log("multi_sheet #{from_type} -> #{to_type} with options: #{options}")
|
22
23
|
data.map do |sheet, from_data|
|
23
|
-
to_data = self.convert(from_type, to_type, from_data)
|
24
|
+
to_data = self.convert(from_type, to_type, from_data, options)
|
24
25
|
[sheet, to_data]
|
25
26
|
end.to_h
|
26
27
|
end
|
@@ -19,6 +19,7 @@ class DataListConverter
|
|
19
19
|
sheet = book.create_worksheet(name: (options[:sheet] || "Sheet1"))
|
20
20
|
i = 0
|
21
21
|
proc.call do |row|
|
22
|
+
row = row.map(&:to_s)
|
22
23
|
sheet.row(i).push *row
|
23
24
|
i += 1
|
24
25
|
end
|
@@ -33,6 +34,7 @@ class DataListConverter
|
|
33
34
|
sheet = book.create_worksheet(name: name.to_s)
|
34
35
|
i = 0
|
35
36
|
table_iterator.call do |row|
|
37
|
+
row = row.map(&:to_s)
|
36
38
|
sheet.row(i).concat(row)
|
37
39
|
i += 1
|
38
40
|
end
|
@@ -23,7 +23,7 @@ class DataListConverter
|
|
23
23
|
i = 0
|
24
24
|
proc.call do |row|
|
25
25
|
row.each_with_index do |v, j|
|
26
|
-
sheet.add_cell(i, j, v)
|
26
|
+
sheet.add_cell(i, j, v.to_s)
|
27
27
|
end
|
28
28
|
i += 1
|
29
29
|
end
|
@@ -42,14 +42,14 @@ class DataListConverter
|
|
42
42
|
row.each_with_index do |v, j|
|
43
43
|
if v.kind_of?(Hash)
|
44
44
|
# custom cell format
|
45
|
-
cell = sheet.add_cell(i, j, v[:text])
|
45
|
+
cell = sheet.add_cell(i, j, v[:text].to_s)
|
46
46
|
v.each do |k, v|
|
47
47
|
next if k == :text
|
48
48
|
cell.send(k, v)
|
49
49
|
end
|
50
50
|
cell.change_fill(v[:fill_color]) if v[:fill_color]
|
51
51
|
else
|
52
|
-
cell = sheet.add_cell(i, j, v)
|
52
|
+
cell = sheet.add_cell(i, j, v.to_s)
|
53
53
|
end
|
54
54
|
|
55
55
|
end
|
data/test/filters_test.rb
CHANGED
@@ -26,12 +26,30 @@ describe DataListConverter do
|
|
26
26
|
string = StringIO.new
|
27
27
|
filter = {count: {size: 4000,
|
28
28
|
out: string,
|
29
|
-
msg: "%{percent}
|
29
|
+
msg: "%{percent}%%"}}
|
30
30
|
result = DataListConverter.convert(
|
31
31
|
:item_iterator, :table_data, iter,
|
32
32
|
item_iterator: {filter: filter})
|
33
33
|
string.string.split("\n").must_equal ['total: 10000', '40.0%', '80.0%']
|
34
34
|
end
|
35
35
|
end
|
36
|
+
|
37
|
+
describe :remove_debug do
|
38
|
+
specify do
|
39
|
+
filter = {remove_debug: true}
|
40
|
+
item_data = [{name: "james", debug: "", a: 12}] * 2
|
41
|
+
result = DataListConverter.convert(
|
42
|
+
:item_data, :table_data, item_data,
|
43
|
+
table_iterator: {filter: filter})
|
44
|
+
result.must_equal [["name"], ["james"], ["james"]]
|
45
|
+
|
46
|
+
# check on multi_sheet
|
47
|
+
item_data = {a: [{name: "james", debug: "", a: 12}] * 2, b: [{name: 'cc', debug: "", b: 3}]*3}
|
48
|
+
result = DataListConverter.convert(
|
49
|
+
:multi_sheet_item_data, :multi_sheet_table_data, item_data,
|
50
|
+
multi_sheet_table_iterator: {table_iterator: {filter: {remove_debug: true}}})
|
51
|
+
result.must_equal(a: [["name"], ["james"], ["james"]], b: [["name"], ["cc"], ["cc"], ["cc"]])
|
52
|
+
end
|
53
|
+
end
|
36
54
|
|
37
55
|
end
|
data/test/helper_test.rb
CHANGED
@@ -79,6 +79,12 @@ describe DataListConverter do
|
|
79
79
|
# change sep
|
80
80
|
data = {a: {b: 12}, c: {d: {e: 11}}}
|
81
81
|
@c.flatten(data, '_').must_equal({:"a_b"=>12, :"c_d_e"=>11})
|
82
|
+
|
83
|
+
# set max level
|
84
|
+
data = {a: {b: 12}, c: {d: {e: {f: 11}}}}
|
85
|
+
@c.flatten(data, ':', 1).must_equal({:"a:b"=>12, :"c:d"=>{e: {f: 11}}})
|
86
|
+
data = {a: {b: 12}, c: {d: {e: {f: 11}}}}
|
87
|
+
@c.flatten(data, ':', 2).must_equal({:"a:b"=>12, :"c:d:e"=>{f: 11}})
|
82
88
|
end
|
83
89
|
end
|
84
90
|
end
|
data/test/types_test.rb
CHANGED
@@ -79,6 +79,27 @@ describe DataListConverter do
|
|
79
79
|
end
|
80
80
|
end
|
81
81
|
|
82
|
+
require 'data_list_converter/types/fast_xlsx_file'
|
83
|
+
|
84
|
+
describe :xlsx_file do
|
85
|
+
specify do
|
86
|
+
filename = 'test.xlsx'
|
87
|
+
begin
|
88
|
+
@c.convert(:item_data, :xlsx_file, ITEM_DATA, xlsx_file: {filename: filename})
|
89
|
+
@c.convert(:xlsx_file, :item_data, {filename: filename}).must_equal ITEM_DATA
|
90
|
+
|
91
|
+
@c.convert(:multi_sheet_table_data, :xlsx_file, MULTI_SHEET_TABLE_DATA,
|
92
|
+
xlsx_file: {filename: filename})
|
93
|
+
@c.convert(:xlsx_file, :multi_sheet_table_data,
|
94
|
+
{filename: filename},
|
95
|
+
).must_equal(MULTI_SHEET_TABLE_DATA)
|
96
|
+
ensure
|
97
|
+
FileUtils.rm_f(filename)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
end
|
102
|
+
|
82
103
|
describe :marshal do
|
83
104
|
specify do
|
84
105
|
filename = 'test.marshal'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: data_list_converter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- linjunhalida
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-12-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -108,6 +108,34 @@ dependencies:
|
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '4.2'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: xlsxtream
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: creek
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
111
139
|
description: Data List Converter is a tool to convert data between different formats.
|
112
140
|
email:
|
113
141
|
- linjunhalida@gmail.com
|
@@ -130,6 +158,7 @@ files:
|
|
130
158
|
- lib/data_list_converter/helper.rb
|
131
159
|
- lib/data_list_converter/types/basic.rb
|
132
160
|
- lib/data_list_converter/types/csv_file.rb
|
161
|
+
- lib/data_list_converter/types/fast_xlsx_file.rb
|
133
162
|
- lib/data_list_converter/types/marshal.rb
|
134
163
|
- lib/data_list_converter/types/multi_sheet.rb
|
135
164
|
- lib/data_list_converter/types/records.rb
|
@@ -160,14 +189,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
160
189
|
- !ruby/object:Gem::Version
|
161
190
|
version: '0'
|
162
191
|
requirements: []
|
163
|
-
|
164
|
-
rubygems_version: 2.4.8
|
192
|
+
rubygems_version: 3.0.8
|
165
193
|
signing_key:
|
166
194
|
specification_version: 4
|
167
195
|
summary: convert data between different formats
|
168
|
-
test_files:
|
169
|
-
- test/base_test.rb
|
170
|
-
- test/filters_test.rb
|
171
|
-
- test/helper_test.rb
|
172
|
-
- test/testdata.rb
|
173
|
-
- test/types_test.rb
|
196
|
+
test_files: []
|