smarter_csv 1.6.1 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +133 -0
- data/CHANGELOG.md +22 -1
- data/CONTRIBUTORS.md +3 -0
- data/Gemfile +7 -4
- data/README.md +8 -6
- data/Rakefile +15 -13
- data/ext/smarter_csv/extconf.rb +14 -0
- data/ext/smarter_csv/smarter_csv.c +86 -0
- data/lib/extensions/hash.rb +4 -2
- data/lib/smarter_csv/version.rb +3 -1
- data/lib/smarter_csv.rb +519 -10
- data/smarter_csv.gemspec +22 -7
- metadata +54 -176
- data/.gitignore +0 -10
- data/.rspec +0 -2
- data/.travis.yml +0 -27
- data/lib/smarter_csv/smarter_csv.rb +0 -461
- data/spec/fixtures/additional_separator.csv +0 -6
- data/spec/fixtures/basic.csv +0 -8
- data/spec/fixtures/binary.csv +0 -1
- data/spec/fixtures/carriage_returns_n.csv +0 -18
- data/spec/fixtures/carriage_returns_quoted.csv +0 -3
- data/spec/fixtures/carriage_returns_r.csv +0 -1
- data/spec/fixtures/carriage_returns_rn.csv +0 -18
- data/spec/fixtures/chunk_cornercase.csv +0 -10
- data/spec/fixtures/duplicate_headers.csv +0 -3
- data/spec/fixtures/empty.csv +0 -5
- data/spec/fixtures/empty_columns_1.csv +0 -2
- data/spec/fixtures/empty_columns_2.csv +0 -2
- data/spec/fixtures/hard_sample.csv +0 -2
- data/spec/fixtures/ignore_comments.csv +0 -11
- data/spec/fixtures/ignore_comments2.csv +0 -3
- data/spec/fixtures/key_mapping.csv +0 -2
- data/spec/fixtures/line_endings_n.csv +0 -4
- data/spec/fixtures/line_endings_r.csv +0 -1
- data/spec/fixtures/line_endings_rn.csv +0 -4
- data/spec/fixtures/lots_of_columns.csv +0 -2
- data/spec/fixtures/malformed.csv +0 -3
- data/spec/fixtures/malformed_header.csv +0 -3
- data/spec/fixtures/money.csv +0 -3
- data/spec/fixtures/no_header.csv +0 -7
- data/spec/fixtures/numeric.csv +0 -5
- data/spec/fixtures/pets.csv +0 -5
- data/spec/fixtures/problematic.csv +0 -8
- data/spec/fixtures/quote_char.csv +0 -9
- data/spec/fixtures/quoted.csv +0 -5
- data/spec/fixtures/quoted2.csv +0 -4
- data/spec/fixtures/separator_colon.csv +0 -4
- data/spec/fixtures/separator_comma.csv +0 -4
- data/spec/fixtures/separator_pipe.csv +0 -4
- data/spec/fixtures/separator_semi.csv +0 -4
- data/spec/fixtures/separator_tab.csv +0 -4
- data/spec/fixtures/skip_lines.csv +0 -8
- data/spec/fixtures/trading.csv +0 -3
- data/spec/fixtures/user_import.csv +0 -3
- data/spec/fixtures/valid_unicode.csv +0 -5
- data/spec/fixtures/with_dashes.csv +0 -8
- data/spec/fixtures/with_dates.csv +0 -4
- data/spec/smarter_csv/additional_separator_spec.rb +0 -45
- data/spec/smarter_csv/binary_file2_spec.rb +0 -24
- data/spec/smarter_csv/binary_file_spec.rb +0 -22
- data/spec/smarter_csv/blank_spec.rb +0 -55
- data/spec/smarter_csv/carriage_return_spec.rb +0 -190
- data/spec/smarter_csv/chunked_reading_spec.rb +0 -14
- data/spec/smarter_csv/close_file_spec.rb +0 -15
- data/spec/smarter_csv/column_separator_spec.rb +0 -95
- data/spec/smarter_csv/convert_values_to_numeric_spec.rb +0 -48
- data/spec/smarter_csv/duplicate_headers_spec.rb +0 -76
- data/spec/smarter_csv/empty_columns_spec.rb +0 -74
- data/spec/smarter_csv/extenstions_spec.rb +0 -17
- data/spec/smarter_csv/hard_sample_spec.rb +0 -24
- data/spec/smarter_csv/header_transformation_spec.rb +0 -21
- data/spec/smarter_csv/ignore_comments_spec.rb +0 -45
- data/spec/smarter_csv/invalid_headers_spec.rb +0 -38
- data/spec/smarter_csv/keep_headers_spec.rb +0 -24
- data/spec/smarter_csv/key_mapping_spec.rb +0 -56
- data/spec/smarter_csv/line_ending_spec.rb +0 -43
- data/spec/smarter_csv/load_basic_spec.rb +0 -20
- data/spec/smarter_csv/malformed_spec.rb +0 -25
- data/spec/smarter_csv/no_header_spec.rb +0 -29
- data/spec/smarter_csv/not_downcase_header_spec.rb +0 -24
- data/spec/smarter_csv/parse/column_separator_spec.rb +0 -61
- data/spec/smarter_csv/parse/old_csv_library_spec.rb +0 -74
- data/spec/smarter_csv/parse/rfc4180_and_more_spec.rb +0 -170
- data/spec/smarter_csv/problematic.rb +0 -34
- data/spec/smarter_csv/quoted_spec.rb +0 -52
- data/spec/smarter_csv/remove_empty_values_spec.rb +0 -13
- data/spec/smarter_csv/remove_keys_from_hashes_spec.rb +0 -25
- data/spec/smarter_csv/remove_not_mapped_keys_spec.rb +0 -35
- data/spec/smarter_csv/remove_values_matching_spec.rb +0 -26
- data/spec/smarter_csv/remove_zero_values_spec.rb +0 -25
- data/spec/smarter_csv/skip_lines_spec.rb +0 -29
- data/spec/smarter_csv/strings_as_keys_spec.rb +0 -24
- data/spec/smarter_csv/strip_chars_from_headers_spec.rb +0 -24
- data/spec/smarter_csv/trading_spec.rb +0 -25
- data/spec/smarter_csv/valid_unicode_spec.rb +0 -94
- data/spec/smarter_csv/value_converters_spec.rb +0 -52
- data/spec/spec/spec_helper.rb +0 -17
- data/spec/spec.opts +0 -2
- data/spec/spec_helper.rb +0 -21
|
@@ -1,170 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'fulfills RFC-4180 and more' do
|
|
6
|
-
let(:options) { {col_sep: ',', row_sep: $INPUT_RECORD_SEPARATOR, quote_char: '"' } }
|
|
7
|
-
|
|
8
|
-
context 'parses simple CSV' do
|
|
9
|
-
context 'RFC-4180' do
|
|
10
|
-
it 'separating on col_sep' do
|
|
11
|
-
line = 'aaa,bbb,ccc'
|
|
12
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [%w[aaa bbb ccc], 3]
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
it 'preserves whitespace' do
|
|
16
|
-
line = ' aaa , bbb , ccc '
|
|
17
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [
|
|
18
|
-
[' aaa ', ' bbb ', ' ccc '], 3
|
|
19
|
-
]
|
|
20
|
-
end
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
context 'extending RFC-4180' do
|
|
24
|
-
it 'with extra col_sep' do
|
|
25
|
-
line = 'aaa,bbb,ccc,'
|
|
26
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [
|
|
27
|
-
['aaa', 'bbb', 'ccc', ''], 4
|
|
28
|
-
]
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
it 'with extra col_sep with given header_size' do
|
|
32
|
-
line = 'aaa,bbb,ccc,'
|
|
33
|
-
expect( SmarterCSV.send(:parse, line, options, 3)).to eq [
|
|
34
|
-
['aaa', 'bbb', 'ccc'], 3
|
|
35
|
-
]
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
it 'with multiple extra col_sep' do
|
|
39
|
-
line = 'aaa,bbb,ccc,,,'
|
|
40
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [
|
|
41
|
-
['aaa', 'bbb', 'ccc', '', '', ''], 6
|
|
42
|
-
]
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
it 'with multiple extra col_sep' do
|
|
46
|
-
line = 'aaa,bbb,ccc,,,'
|
|
47
|
-
expect( SmarterCSV.send(:parse, line, options, 3)).to eq [
|
|
48
|
-
['aaa', 'bbb', 'ccc'], 3
|
|
49
|
-
]
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
it 'with multiple complex col_sep' do
|
|
53
|
-
line = 'aaa<=>bbb<=>ccc<=><=><=>'
|
|
54
|
-
expect( SmarterCSV.send(:parse, line, options.merge({col_sep: '<=>'}))).to eq [
|
|
55
|
-
['aaa', 'bbb', 'ccc', '', '', ''], 6
|
|
56
|
-
]
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
it 'with multiple complex col_sep with given header_size' do
|
|
60
|
-
line = 'aaa<=>bbb<=>ccc<=><=><=>'
|
|
61
|
-
expect( SmarterCSV.send(:parse, line, options.merge({col_sep: '<=>'}), 3)).to eq [
|
|
62
|
-
['aaa', 'bbb', 'ccc'], 3
|
|
63
|
-
]
|
|
64
|
-
end
|
|
65
|
-
end
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
context 'parses quoted CSV' do
|
|
69
|
-
context 'RFC-4180' do
|
|
70
|
-
it 'separating on col_sep' do
|
|
71
|
-
line = '"aaa","bbb","ccc"'
|
|
72
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [%w[aaa bbb ccc], 3]
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
it 'parses corner case correctly' do
|
|
76
|
-
line = '"Board 4""","$17.40","10000003427"'
|
|
77
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [
|
|
78
|
-
['Board 4"', '$17.40', '10000003427'], 3
|
|
79
|
-
]
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
it 'quoted parts can contain spaces' do
|
|
83
|
-
line = '" aaa1 aaa2 "," bbb1 bbb2 "," ccc1 ccc2 "'
|
|
84
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [
|
|
85
|
-
[' aaa1 aaa2 ', ' bbb1 bbb2 ', ' ccc1 ccc2 '], 3
|
|
86
|
-
]
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
it 'quoted parts can contain row_sep' do
|
|
90
|
-
line = '"aaa1, aaa2","bbb1, bbb2","ccc1, ccc2"'
|
|
91
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [
|
|
92
|
-
['aaa1, aaa2', 'bbb1, bbb2', 'ccc1, ccc2'], 3
|
|
93
|
-
]
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
it 'quoted parts can contain row_sep' do
|
|
97
|
-
line = '"aaa1, ""aaa2"", aaa3","""bbb1"", bbb2","ccc1, ""ccc2"""'
|
|
98
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [
|
|
99
|
-
['aaa1, "aaa2", aaa3', '"bbb1", bbb2', 'ccc1, "ccc2"'], 3
|
|
100
|
-
]
|
|
101
|
-
end
|
|
102
|
-
|
|
103
|
-
it 'some fields are quoted' do
|
|
104
|
-
line = '1,"board 4""",12.95'
|
|
105
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [
|
|
106
|
-
['1', 'board 4"', '12.95'], 3
|
|
107
|
-
]
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
it 'separating on col_sep' do
|
|
111
|
-
line = '"some","thing","""completely"" different"'
|
|
112
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [
|
|
113
|
-
['some', 'thing', '"completely" different'], 3
|
|
114
|
-
]
|
|
115
|
-
end
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
context 'extending RFC-4180' do
|
|
119
|
-
it 'with extra col_sep, without given header_size' do
|
|
120
|
-
line = '"aaa","bbb","ccc",'
|
|
121
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [
|
|
122
|
-
['aaa', 'bbb', 'ccc', ''], 4
|
|
123
|
-
]
|
|
124
|
-
end
|
|
125
|
-
|
|
126
|
-
it 'with extra col_sep, with given header_size' do
|
|
127
|
-
line = '"aaa","bbb","ccc",'
|
|
128
|
-
expect( SmarterCSV.send(:parse, line, options, 3)).to eq [%w[aaa bbb ccc], 3]
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
it 'with multiple extra col_sep, without given header_size' do
|
|
132
|
-
line = '"aaa","bbb","ccc",,,'
|
|
133
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [
|
|
134
|
-
['aaa', 'bbb', 'ccc', '', '', ''], 6
|
|
135
|
-
]
|
|
136
|
-
end
|
|
137
|
-
|
|
138
|
-
it 'with multiple extra col_sep, with given header_size' do
|
|
139
|
-
line = '"aaa","bbb","ccc",,,'
|
|
140
|
-
expect( SmarterCSV.send(:parse, line, options, 3)).to eq [
|
|
141
|
-
['aaa', 'bbb', 'ccc'], 3
|
|
142
|
-
]
|
|
143
|
-
end
|
|
144
|
-
|
|
145
|
-
it 'with multiple complex extra col_sep, without given header_size' do
|
|
146
|
-
line = '"aaa"<=>"bbb"<=>"ccc"<=><=><=>'
|
|
147
|
-
expect( SmarterCSV.send(:parse, line, options.merge({col_sep: '<=>'}))).to eq [
|
|
148
|
-
['aaa', 'bbb', 'ccc', '', '', ''], 6
|
|
149
|
-
]
|
|
150
|
-
end
|
|
151
|
-
|
|
152
|
-
it 'with multiple complex extra col_sep, with given header_size' do
|
|
153
|
-
line = '"aaa"<=>"bbb"<=>"ccc"<=><=><=>'
|
|
154
|
-
expect( SmarterCSV.send(:parse, line, options.merge({col_sep: '<=>'}), 3)).to eq [
|
|
155
|
-
['aaa', 'bbb', 'ccc'], 3
|
|
156
|
-
]
|
|
157
|
-
end
|
|
158
|
-
end
|
|
159
|
-
end
|
|
160
|
-
|
|
161
|
-
# relaxed parsing compared to RFC-4180
|
|
162
|
-
context 'liberal_parsing' do
|
|
163
|
-
it 'parses corner case correctly' do
|
|
164
|
-
line = 'is,this "three, or four",fields'
|
|
165
|
-
expect( SmarterCSV.send(:parse, line, options)).to eq [
|
|
166
|
-
['is', 'this "three, or four"', 'fields'], 3
|
|
167
|
-
]
|
|
168
|
-
end
|
|
169
|
-
end
|
|
170
|
-
end
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'loading file with UTF-8 characters in the header' do
|
|
6
|
-
|
|
7
|
-
# file which caused issues because of UTF-8 characters in the header
|
|
8
|
-
it 'loads the file with force_utf8 flag set' do
|
|
9
|
-
options = {col_sep: ";", force_utf8: true}
|
|
10
|
-
data = SmarterCSV.process("#{fixture_path}/problematic.csv", options)
|
|
11
|
-
|
|
12
|
-
data.length.should eq 7
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
it 'loads the file with strings as keys' do
|
|
16
|
-
options = {
|
|
17
|
-
file_encoding: 'iso-8859-1:UTF-8', # important!
|
|
18
|
-
col_sep: ";", strings_as_keys: true,
|
|
19
|
-
}
|
|
20
|
-
data = SmarterCSV.process("#{fixture_path}/problematic.csv", options)
|
|
21
|
-
|
|
22
|
-
data.length.should eq 7
|
|
23
|
-
data.first.keys.sort.should eq [
|
|
24
|
-
"compte",
|
|
25
|
-
"date_de_comptabilisation",
|
|
26
|
-
"date_opération",
|
|
27
|
-
"date_valeur",
|
|
28
|
-
"libellé",
|
|
29
|
-
"montant",
|
|
30
|
-
"référence"
|
|
31
|
-
]
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
end
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'loading file with quoted fields' do
|
|
6
|
-
it 'leaving the quotes in the data' do
|
|
7
|
-
options = {}
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/quoted.csv", options)
|
|
9
|
-
data.flatten.size.should == 4
|
|
10
|
-
data[1][:model].should eq 'Venture "Extended Edition"'
|
|
11
|
-
data[1][:description].should be_nil
|
|
12
|
-
data[2][:model].should eq 'Venture "Extended Edition, Very Large"'
|
|
13
|
-
data[2][:description].should be_nil
|
|
14
|
-
data[3][:description].should eq 'MUST SELL! air, moon roof, loaded'
|
|
15
|
-
data.each do |h|
|
|
16
|
-
h[:year].class.should eq Fixnum
|
|
17
|
-
h[:make].should_not be_nil
|
|
18
|
-
h[:model].should_not be_nil
|
|
19
|
-
h[:price].class.should eq Float
|
|
20
|
-
end
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
# quotes inside quoted fields need to be escaped by another double-quote
|
|
24
|
-
it 'removes quotes around quoted fields, but not inside data' do
|
|
25
|
-
options = {}
|
|
26
|
-
data = SmarterCSV.process("#{fixture_path}/quote_char.csv", options)
|
|
27
|
-
|
|
28
|
-
data.length.should eq 6
|
|
29
|
-
data[0][:first_name].should eq "\"John"
|
|
30
|
-
data[0][:last_name].should eq "Cooke\""
|
|
31
|
-
data[1][:first_name].should eq "Jam\ne\nson\""
|
|
32
|
-
data[2][:first_name].should eq "\"Jean"
|
|
33
|
-
data[4][:first_name].should eq "Bo\"bbie"
|
|
34
|
-
data[5][:first_name].should eq 'Mica'
|
|
35
|
-
data[5][:last_name].should eq 'Copeland'
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
# NOTE: quotes inside headers need to be escaped by doubling them
|
|
39
|
-
# e.g. 'correct ""EXAMPLE""'
|
|
40
|
-
# this escaping is illegal: 'incorrect \"EXAMPLE\"' <-- this caused CSV parsing error
|
|
41
|
-
# in case of CSV parsing errirs, use :user_provided_headers, or key_mapping
|
|
42
|
-
#
|
|
43
|
-
it 'removes quotes around headers and extra quotes inside headers' do
|
|
44
|
-
options = {}
|
|
45
|
-
data = SmarterCSV.process("#{fixture_path}/quoted2.csv", options)
|
|
46
|
-
|
|
47
|
-
data.length.should eq 3
|
|
48
|
-
data.first.keys[2].should eq :isbn
|
|
49
|
-
data.first.keys[3].should eq :discounted_price
|
|
50
|
-
data[1][:author].should eq 'Timothy "The Parser" Campbell'
|
|
51
|
-
end
|
|
52
|
-
end
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
it 'remove_empty_values' do
|
|
7
|
-
options = {:row_sep => :auto, :remove_empty_values => true}
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/empty.csv", options)
|
|
9
|
-
data.size.should == 1
|
|
10
|
-
data[0].keys.should == [:not_empty_1, :not_empty_2, :not_empty_3]
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
end
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
it 'remove_values_matching' do
|
|
7
|
-
options = {:remove_zero_values => true, :key_mapping => {:first_name => :vorname, :last_name => :nachname, :fish => nil} }
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
|
9
|
-
data.size.should == 5
|
|
10
|
-
# all the keys should be symbols
|
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
|
12
|
-
|
|
13
|
-
data.each do |hash|
|
|
14
|
-
hash.keys.each do |key|
|
|
15
|
-
[:vorname, :nachname, :dogs, :cats, :birds].should include( key )
|
|
16
|
-
end
|
|
17
|
-
hash.values.should_not include( 0 )
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
data.each do |h|
|
|
21
|
-
h.size.should <= 6
|
|
22
|
-
end
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
end
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe ':remove_unmapped_keys option' do
|
|
6
|
-
|
|
7
|
-
it 'it has no effect on loading a file without options' do
|
|
8
|
-
options = {}
|
|
9
|
-
data = SmarterCSV.process("#{fixture_path}/lots_of_columns.csv", options)
|
|
10
|
-
data.size.should eq 1
|
|
11
|
-
data.first.size.should eq 474 # there are some empty rows in the fixture
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
it 'it has no effect if provided without :key_mapping' do
|
|
15
|
-
options = {:remove_unmapped_keys => true}
|
|
16
|
-
data = SmarterCSV.process("#{fixture_path}/lots_of_columns.csv", options)
|
|
17
|
-
data.size.should eq 1
|
|
18
|
-
data.first.size.should eq 474 # there are some empty rows in the fixture
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
it 'it defaults to false and has no effect if :key_mapping is provided without :remove_unmapped_keys' do
|
|
22
|
-
options = {:key_mapping => {:column_0 => :one, :column_15 => :two, :column_42 => :three}}
|
|
23
|
-
data = SmarterCSV.process("#{fixture_path}/lots_of_columns.csv", options)
|
|
24
|
-
data.size.should eq 1
|
|
25
|
-
data.first.size.should eq 474 # there are some empty rows in the fixture
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
it 'it removes non-mapped keys/columns when set to true and :key_mapping is provided' do
|
|
29
|
-
options = {:remove_unmapped_keys => true, :key_mapping => {:column_0 => :one, :column_15 => :two, :column_42 => :three}}
|
|
30
|
-
data = SmarterCSV.process("#{fixture_path}/lots_of_columns.csv", options)
|
|
31
|
-
data.size.should eq 1
|
|
32
|
-
data.first.size.should eq 2 # column_15 is empty
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
it 'remove_values_matching' do
|
|
7
|
-
options = {:remove_zero_values => true, :remove_empty_values => true, :remove_values_matching => /^\d+$/}
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
|
9
|
-
data.size.should == 5
|
|
10
|
-
# all the keys should be symbols
|
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
|
12
|
-
|
|
13
|
-
data.each do |hash|
|
|
14
|
-
hash.keys.each do |key|
|
|
15
|
-
[:first_name, :last_name].should include( key )
|
|
16
|
-
end
|
|
17
|
-
hash.values.each{|x| x.class.should be == String}
|
|
18
|
-
hash.values.should_not include( 0 )
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
data.each do |h|
|
|
22
|
-
h.size.should <= 6
|
|
23
|
-
end
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
end
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
it 'remove_zero_values' do
|
|
7
|
-
options = {:remove_zero_values => true, :remove_empty_values => true}
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
|
9
|
-
data.size.should == 5
|
|
10
|
-
# all the keys should be symbols
|
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
|
12
|
-
|
|
13
|
-
data.each do |hash|
|
|
14
|
-
hash.keys.each do |key|
|
|
15
|
-
[:first_name, :last_name, :dogs, :cats, :birds, :fish].should include( key )
|
|
16
|
-
end
|
|
17
|
-
hash.values.should_not include( 0 )
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
data.each do |h|
|
|
21
|
-
h.size.should <= 6
|
|
22
|
-
end
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
end
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
it 'loads_csv_file_skipping_lines' do
|
|
7
|
-
options = {skip_lines: 3}
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/skip_lines.csv", options)
|
|
9
|
-
data.size.should == 4
|
|
10
|
-
|
|
11
|
-
data.each do |item|
|
|
12
|
-
item.keys.each do |key|
|
|
13
|
-
[:first_name,:last_name,:dogs,:cats,:birds,:fish].should include(key)
|
|
14
|
-
end
|
|
15
|
-
end
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
it 'loads_csv_with_user_defined_headers' do
|
|
19
|
-
options = {:skip_lines => 3, :headers_in_file => true, :user_provided_headers => [:a,:b,:c,:d,:e,:f]}
|
|
20
|
-
data = SmarterCSV.process("#{fixture_path}/skip_lines.csv", options)
|
|
21
|
-
data.size.should == 4
|
|
22
|
-
|
|
23
|
-
data.each do |item|
|
|
24
|
-
item.keys.each do |key|
|
|
25
|
-
[:a,:b,:c,:d,:e,:f].should include( key )
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
end
|
|
29
|
-
end
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
it 'use_strings_as_keys' do
|
|
7
|
-
options = {:strings_as_keys => true}
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
|
9
|
-
data.size.should == 5
|
|
10
|
-
# all the keys should be symbols
|
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == String}}
|
|
12
|
-
|
|
13
|
-
data.each do |item|
|
|
14
|
-
item.keys.each do |key|
|
|
15
|
-
["first_name", "last_name", "dogs", "cats", "birds", "fish"].should include( key )
|
|
16
|
-
end
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
data.each do |h|
|
|
20
|
-
h.size.should <= 6
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
end
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
it 'strip_whitespace_from_headers' do
|
|
7
|
-
options = {:strip_chars_from_headers => ' '}
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
|
9
|
-
data.size.should == 5
|
|
10
|
-
# all the keys should be symbols
|
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
|
12
|
-
|
|
13
|
-
data.each do |item|
|
|
14
|
-
item.keys.each do |key|
|
|
15
|
-
[:firstname, :lastname, :dogs, :cats, :birds, :fish].should include( key )
|
|
16
|
-
end
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
data.each do |h|
|
|
20
|
-
h.size.should <= 6
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
end
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
# somebody reported that a column called 'options_trader' would be truncated to 'trader'
|
|
6
|
-
|
|
7
|
-
describe 'loads simple file format' do
|
|
8
|
-
|
|
9
|
-
it 'with symbols as keys when using defaults' do
|
|
10
|
-
options = {}
|
|
11
|
-
data = SmarterCSV.process("#{fixture_path}/trading.csv", options)
|
|
12
|
-
|
|
13
|
-
data.flatten.size.should eq 2
|
|
14
|
-
data.each do |item|
|
|
15
|
-
# all keys should be symbols when using v1.x backwards compatible mode
|
|
16
|
-
item.keys.each{|x| x.class.should eq Symbol}
|
|
17
|
-
item[:account_id].class.should eq Fixnum
|
|
18
|
-
item[:options_trader].class.should eq String
|
|
19
|
-
item[:stock_symbol].class.should eq String
|
|
20
|
-
item[:shares_issued].class.should eq Fixnum
|
|
21
|
-
item[:purchase_date].class.should eq String
|
|
22
|
-
end
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
end
|
|
@@ -1,94 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
it 'loads file with unicode strings' do
|
|
9
|
-
options = {}
|
|
10
|
-
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
|
11
|
-
data.flatten.size.should == 4
|
|
12
|
-
data[0][:artist].should eq 'Кино'
|
|
13
|
-
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
|
14
|
-
data[0][:album].should eq 'Группа Крови'
|
|
15
|
-
data[0][:label].should eq 'Moroz Records'
|
|
16
|
-
data[0][:year].should eq 1998
|
|
17
|
-
|
|
18
|
-
data[0].should eq data[1]
|
|
19
|
-
|
|
20
|
-
data[2][:artist].should eq 'Rammstein'
|
|
21
|
-
data[2][:track].should eq 'Frühling in Paris'
|
|
22
|
-
data[2][:album].should eq 'Liebe ist für alle da'
|
|
23
|
-
data[2][:label].should eq 'Vagrant'
|
|
24
|
-
data[2][:year].should eq 2009
|
|
25
|
-
|
|
26
|
-
data[2].should eq data[3]
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
it 'loads file with unicode strings, when forcing utf8' do
|
|
30
|
-
options = {:force_utf8 => true}
|
|
31
|
-
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
|
32
|
-
data.flatten.size.should == 4
|
|
33
|
-
data[0][:artist].should eq 'Кино'
|
|
34
|
-
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
|
35
|
-
data[0][:album].should eq 'Группа Крови'
|
|
36
|
-
data[0][:label].should eq 'Moroz Records'
|
|
37
|
-
data[0][:year].should eq 1998
|
|
38
|
-
|
|
39
|
-
data[0].should eq data[1]
|
|
40
|
-
|
|
41
|
-
data[2][:artist].should eq 'Rammstein'
|
|
42
|
-
data[2][:track].should eq 'Frühling in Paris'
|
|
43
|
-
data[2][:album].should eq 'Liebe ist für alle da'
|
|
44
|
-
data[2][:label].should eq 'Vagrant'
|
|
45
|
-
data[2][:year].should eq 2009
|
|
46
|
-
|
|
47
|
-
data[2].should eq data[3]
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
it 'loads file with unicode strings, when loading from binary input' do
|
|
53
|
-
options = {:file_encoding => 'binary'}
|
|
54
|
-
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
|
55
|
-
data.flatten.size.should == 4
|
|
56
|
-
data[0][:artist].should eq 'Кино'
|
|
57
|
-
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
|
58
|
-
data[0][:album].should eq 'Группа Крови'
|
|
59
|
-
data[0][:label].should eq 'Moroz Records'
|
|
60
|
-
data[0][:year].should eq 1998
|
|
61
|
-
|
|
62
|
-
data[0].should eq data[1]
|
|
63
|
-
|
|
64
|
-
data[2][:artist].should eq 'Rammstein'
|
|
65
|
-
data[2][:track].should eq 'Frühling in Paris'
|
|
66
|
-
data[2][:album].should eq 'Liebe ist für alle da'
|
|
67
|
-
data[2][:label].should eq 'Vagrant'
|
|
68
|
-
data[2][:year].should eq 2009
|
|
69
|
-
|
|
70
|
-
data[2].should eq data[3]
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
it 'loads file with unicode strings, when forcing utf8 with binary input' do
|
|
74
|
-
options = {:file_encoding => 'binary', :force_utf8 => true}
|
|
75
|
-
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
|
76
|
-
data.flatten.size.should == 4
|
|
77
|
-
data[0][:artist].should eq 'Кино'
|
|
78
|
-
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
|
79
|
-
data[0][:album].should eq 'Группа Крови'
|
|
80
|
-
data[0][:label].should eq 'Moroz Records'
|
|
81
|
-
data[0][:year].should eq 1998
|
|
82
|
-
|
|
83
|
-
data[0].should eq data[1]
|
|
84
|
-
|
|
85
|
-
data[2][:artist].should eq 'Rammstein'
|
|
86
|
-
data[2][:track].should eq 'Frühling in Paris'
|
|
87
|
-
data[2][:album].should eq 'Liebe ist für alle da'
|
|
88
|
-
data[2][:label].should eq 'Vagrant'
|
|
89
|
-
data[2][:year].should eq 2009
|
|
90
|
-
|
|
91
|
-
data[2].should eq data[3]
|
|
92
|
-
end
|
|
93
|
-
|
|
94
|
-
end
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
require 'date'
|
|
6
|
-
class DateConverter
|
|
7
|
-
def self.convert(value)
|
|
8
|
-
Date.strptime( value, '%m/%d/%Y')
|
|
9
|
-
end
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
class CurrencyConverter
|
|
13
|
-
def self.convert(value)
|
|
14
|
-
value.sub(/[$]/,'').to_f # would be nice to add a computed column :currency => '€'
|
|
15
|
-
end
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
describe 'be_able_to' do
|
|
19
|
-
it 'convert date values into Date instances' do
|
|
20
|
-
options = {:value_converters => {:date => DateConverter}}
|
|
21
|
-
data = SmarterCSV.process("#{fixture_path}/with_dates.csv", options)
|
|
22
|
-
data.flatten.size.should == 3
|
|
23
|
-
data[0][:date].class.should eq Date
|
|
24
|
-
data[0][:date].to_s.should eq "1998-10-30"
|
|
25
|
-
data[1][:date].to_s.should eq "2011-02-01"
|
|
26
|
-
data[2][:date].to_s.should eq "2013-01-09"
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
it 'converts dollar prices into float values' do
|
|
30
|
-
options = {:value_converters => {:price => CurrencyConverter}}
|
|
31
|
-
data = SmarterCSV.process("#{fixture_path}/money.csv", options)
|
|
32
|
-
data.flatten.size.should == 2
|
|
33
|
-
data[0][:price].class.should eq Float
|
|
34
|
-
data[0][:price].should eq 9.99
|
|
35
|
-
data[1][:price].should eq 14.99
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
it 'convert can use multiple value converters' do
|
|
39
|
-
options = {:value_converters => {:date => DateConverter, :price => CurrencyConverter}}
|
|
40
|
-
data = SmarterCSV.process("#{fixture_path}/with_dates.csv", options)
|
|
41
|
-
data.flatten.size.should == 3
|
|
42
|
-
data[0][:date].class.should eq Date
|
|
43
|
-
data[0][:date].to_s.should eq "1998-10-30"
|
|
44
|
-
data[1][:date].to_s.should eq "2011-02-01"
|
|
45
|
-
data[2][:date].to_s.should eq "2013-01-09"
|
|
46
|
-
|
|
47
|
-
data[0][:price].class.should eq Float
|
|
48
|
-
data[0][:price].should eq 44.50
|
|
49
|
-
data[1][:price].should eq 15.0
|
|
50
|
-
data[2][:price].should eq 0.11
|
|
51
|
-
end
|
|
52
|
-
end
|
data/spec/spec/spec_helper.rb
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
# This file was generated by the `rspec --init` command. Conventionally, all
|
|
2
|
-
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
|
3
|
-
# Require this file using `require "spec_helper"` to ensure that it is only
|
|
4
|
-
# loaded once.
|
|
5
|
-
#
|
|
6
|
-
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
|
7
|
-
RSpec.configure do |config|
|
|
8
|
-
config.treat_symbols_as_metadata_keys_with_true_values = true
|
|
9
|
-
config.run_all_when_everything_filtered = true
|
|
10
|
-
config.filter_run :focus
|
|
11
|
-
|
|
12
|
-
# Run specs in random order to surface order dependencies. If you find an
|
|
13
|
-
# order dependency and want to debug it, you can fix the order by providing
|
|
14
|
-
# the seed, which is printed after each run.
|
|
15
|
-
# --seed 1234
|
|
16
|
-
config.order = 'random'
|
|
17
|
-
end
|
data/spec/spec.opts
DELETED