smarter_csv 1.1.5 → 1.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.rspec +1 -2
- data/.rubocop.yml +154 -0
- data/CHANGELOG.md +364 -0
- data/CONTRIBUTORS.md +56 -0
- data/Gemfile +7 -2
- data/LICENSE.txt +21 -0
- data/README.md +44 -441
- data/Rakefile +39 -19
- data/TO_DO_v2.md +14 -0
- data/docs/_introduction.md +56 -0
- data/docs/basic_api.md +157 -0
- data/docs/batch_processing.md +68 -0
- data/docs/data_transformations.md +50 -0
- data/docs/examples.md +75 -0
- data/docs/header_transformations.md +113 -0
- data/docs/header_validations.md +36 -0
- data/docs/options.md +98 -0
- data/docs/row_col_sep.md +104 -0
- data/docs/value_converters.md +68 -0
- data/ext/smarter_csv/extconf.rb +14 -0
- data/ext/smarter_csv/smarter_csv.c +97 -0
- data/lib/smarter_csv/auto_detection.rb +78 -0
- data/lib/smarter_csv/errors.rb +16 -0
- data/lib/smarter_csv/file_io.rb +50 -0
- data/lib/smarter_csv/hash_transformations.rb +91 -0
- data/lib/smarter_csv/header_transformations.rb +63 -0
- data/lib/smarter_csv/header_validations.rb +34 -0
- data/lib/smarter_csv/headers.rb +68 -0
- data/lib/smarter_csv/options.rb +95 -0
- data/lib/smarter_csv/parser.rb +90 -0
- data/lib/smarter_csv/reader.rb +243 -0
- data/lib/smarter_csv/version.rb +3 -1
- data/lib/smarter_csv/writer.rb +116 -0
- data/lib/smarter_csv.rb +91 -3
- data/smarter_csv.gemspec +43 -20
- metadata +122 -137
- data/.gitignore +0 -8
- data/.travis.yml +0 -19
- data/lib/extensions/hash.rb +0 -7
- data/lib/smarter_csv/smarter_csv.rb +0 -281
- data/spec/fixtures/basic.csv +0 -8
- data/spec/fixtures/binary.csv +0 -1
- data/spec/fixtures/carriage_returns_n.csv +0 -18
- data/spec/fixtures/carriage_returns_quoted.csv +0 -3
- data/spec/fixtures/carriage_returns_r.csv +0 -1
- data/spec/fixtures/carriage_returns_rn.csv +0 -18
- data/spec/fixtures/chunk_cornercase.csv +0 -10
- data/spec/fixtures/empty.csv +0 -5
- data/spec/fixtures/line_endings_n.csv +0 -4
- data/spec/fixtures/line_endings_r.csv +0 -1
- data/spec/fixtures/line_endings_rn.csv +0 -4
- data/spec/fixtures/lots_of_columns.csv +0 -2
- data/spec/fixtures/malformed.csv +0 -3
- data/spec/fixtures/malformed_header.csv +0 -3
- data/spec/fixtures/money.csv +0 -3
- data/spec/fixtures/no_header.csv +0 -7
- data/spec/fixtures/numeric.csv +0 -5
- data/spec/fixtures/pets.csv +0 -5
- data/spec/fixtures/quoted.csv +0 -5
- data/spec/fixtures/separator.csv +0 -4
- data/spec/fixtures/skip_lines.csv +0 -8
- data/spec/fixtures/valid_unicode.csv +0 -5
- data/spec/fixtures/with_dashes.csv +0 -8
- data/spec/fixtures/with_dates.csv +0 -4
- data/spec/smarter_csv/binary_file2_spec.rb +0 -24
- data/spec/smarter_csv/binary_file_spec.rb +0 -22
- data/spec/smarter_csv/carriage_return_spec.rb +0 -170
- data/spec/smarter_csv/chunked_reading_spec.rb +0 -14
- data/spec/smarter_csv/close_file_spec.rb +0 -15
- data/spec/smarter_csv/column_separator_spec.rb +0 -11
- data/spec/smarter_csv/convert_values_to_numeric_spec.rb +0 -48
- data/spec/smarter_csv/extenstions_spec.rb +0 -17
- data/spec/smarter_csv/header_transformation_spec.rb +0 -21
- data/spec/smarter_csv/keep_headers_spec.rb +0 -24
- data/spec/smarter_csv/key_mapping_spec.rb +0 -25
- data/spec/smarter_csv/line_ending_spec.rb +0 -43
- data/spec/smarter_csv/load_basic_spec.rb +0 -20
- data/spec/smarter_csv/malformed_spec.rb +0 -21
- data/spec/smarter_csv/no_header_spec.rb +0 -24
- data/spec/smarter_csv/not_downcase_header_spec.rb +0 -24
- data/spec/smarter_csv/quoted_spec.rb +0 -23
- data/spec/smarter_csv/remove_empty_values_spec.rb +0 -13
- data/spec/smarter_csv/remove_keys_from_hashes_spec.rb +0 -25
- data/spec/smarter_csv/remove_not_mapped_keys_spec.rb +0 -35
- data/spec/smarter_csv/remove_values_matching_spec.rb +0 -26
- data/spec/smarter_csv/remove_zero_values_spec.rb +0 -25
- data/spec/smarter_csv/skip_lines_spec.rb +0 -29
- data/spec/smarter_csv/strings_as_keys_spec.rb +0 -24
- data/spec/smarter_csv/strip_chars_from_headers_spec.rb +0 -24
- data/spec/smarter_csv/valid_unicode_spec.rb +0 -94
- data/spec/smarter_csv/value_converters_spec.rb +0 -52
- data/spec/spec/spec_helper.rb +0 -17
- data/spec/spec.opts +0 -2
- data/spec/spec_helper.rb +0 -21
@@ -1,170 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'process files with line endings explicitly pre-specified' do
|
6
|
-
|
7
|
-
it 'should process a file with \n for line endings and within data fields' do
|
8
|
-
sep = "\n"
|
9
|
-
options = {:row_sep => sep}
|
10
|
-
data = SmarterCSV.process("#{fixture_path}/carriage_returns_n.csv", {:row_sep => sep})
|
11
|
-
data.flatten.size.should == 8
|
12
|
-
data[0][:name].should == "Anfield"
|
13
|
-
data[0][:street].should == "Anfield Road"
|
14
|
-
data[0][:city].should == "Liverpool"
|
15
|
-
data[1][:name].should == ["Highbury", "Highbury House"].join(sep)
|
16
|
-
data[2][:street].should == ["Sir Matt ", "Busby Way"].join(sep)
|
17
|
-
data[3][:city].should == ["Newcastle-upon-tyne ", "Tyne and Wear"].join(sep)
|
18
|
-
data[4][:name].should == ["White Hart Lane", "(The Lane)"].join(sep)
|
19
|
-
data[4][:street].should == ["Bill Nicholson Way ", "748 High Rd"].join(sep)
|
20
|
-
data[4][:city].should == ["Tottenham", "London"].join(sep)
|
21
|
-
data[5][:name].should == "Stamford Bridge"
|
22
|
-
data[5][:street].should == ["Fulham Road", "London"].join(sep)
|
23
|
-
data[5][:city].should be_nil
|
24
|
-
data[6][:name].should == ["Etihad Stadium", "Rowsley St", "Manchester"].join(sep)
|
25
|
-
data[7][:name].should == "Goodison"
|
26
|
-
data[7][:street].should == "Goodison Road"
|
27
|
-
data[7][:city].should == "Liverpool"
|
28
|
-
end
|
29
|
-
|
30
|
-
it 'should process a file with \r for line endings and within data fields' do
|
31
|
-
sep = "\r"
|
32
|
-
data = SmarterCSV.process("#{fixture_path}/carriage_returns_r.csv", {:row_sep => sep})
|
33
|
-
data.flatten.size.should == 8
|
34
|
-
data[0][:name].should == "Anfield"
|
35
|
-
data[0][:street].should == "Anfield Road"
|
36
|
-
data[0][:city].should == "Liverpool"
|
37
|
-
data[1][:name].should == ["Highbury", "Highbury House"].join(sep)
|
38
|
-
data[2][:street].should == ["Sir Matt ", "Busby Way"].join(sep)
|
39
|
-
data[3][:city].should == ["Newcastle-upon-tyne ", "Tyne and Wear"].join(sep)
|
40
|
-
data[4][:name].should == ["White Hart Lane", "(The Lane)"].join(sep)
|
41
|
-
data[4][:street].should == ["Bill Nicholson Way ", "748 High Rd"].join(sep)
|
42
|
-
data[4][:city].should == ["Tottenham", "London"].join(sep)
|
43
|
-
data[5][:name].should == "Stamford Bridge"
|
44
|
-
data[5][:street].should == ["Fulham Road", "London"].join(sep)
|
45
|
-
data[5][:city].should be_nil
|
46
|
-
data[6][:name].should == ["Etihad Stadium", "Rowsley St", "Manchester"].join(sep)
|
47
|
-
data[7][:name].should == "Goodison"
|
48
|
-
data[7][:street].should == "Goodison Road"
|
49
|
-
data[7][:city].should == "Liverpool"
|
50
|
-
end
|
51
|
-
|
52
|
-
it 'should process a file with \r\n for line endings and within data fields' do
|
53
|
-
sep = "\r\n"
|
54
|
-
data = SmarterCSV.process("#{fixture_path}/carriage_returns_rn.csv", {:row_sep => sep})
|
55
|
-
data.flatten.size.should == 8
|
56
|
-
data[0][:name].should == "Anfield"
|
57
|
-
data[0][:street].should == "Anfield Road"
|
58
|
-
data[0][:city].should == "Liverpool"
|
59
|
-
data[1][:name].should == ["Highbury", "Highbury House"].join(sep)
|
60
|
-
data[2][:street].should == ["Sir Matt ", "Busby Way"].join(sep)
|
61
|
-
data[3][:city].should == ["Newcastle-upon-tyne ", "Tyne and Wear"].join(sep)
|
62
|
-
data[4][:name].should == ["White Hart Lane", "(The Lane)"].join(sep)
|
63
|
-
data[4][:street].should == ["Bill Nicholson Way ", "748 High Rd"].join(sep)
|
64
|
-
data[4][:city].should == ["Tottenham", "London"].join(sep)
|
65
|
-
data[5][:name].should == "Stamford Bridge"
|
66
|
-
data[5][:street].should == ["Fulham Road", "London"].join(sep)
|
67
|
-
data[5][:city].should be_nil
|
68
|
-
data[6][:name].should == ["Etihad Stadium", "Rowsley St", "Manchester"].join(sep)
|
69
|
-
data[7][:name].should == "Goodison"
|
70
|
-
data[7][:street].should == "Goodison Road"
|
71
|
-
data[7][:city].should == "Liverpool"
|
72
|
-
end
|
73
|
-
|
74
|
-
it 'should process a file with more quoted text carriage return characters (\r) than line ending characters (\n)' do
|
75
|
-
row_sep = "\n"
|
76
|
-
text_sep = "\r"
|
77
|
-
data = SmarterCSV.process("#{fixture_path}/carriage_returns_quoted.csv", {:row_sep => row_sep})
|
78
|
-
data.flatten.size.should == 2
|
79
|
-
data[0][:band].should == "New Order"
|
80
|
-
data[0][:members].should == ["Bernard Sumner", "Peter Hook", "Stephen Morris", "Gillian Gilbert"].join(text_sep)
|
81
|
-
data[0][:albums].should == ["Movement", "Power, Corruption and Lies", "Low-Life", "Brotherhood", "Substance"].join(text_sep)
|
82
|
-
data[1][:band].should == "Led Zeppelin"
|
83
|
-
data[1][:members].should == ["Jimmy Page", "Robert Plant", "John Bonham", "John Paul Jones"].join(text_sep)
|
84
|
-
data[1][:albums].should == ["Led Zeppelin", "Led Zeppelin II", "Led Zeppelin III", "Led Zeppelin IV"].join(text_sep)
|
85
|
-
end
|
86
|
-
|
87
|
-
end
|
88
|
-
|
89
|
-
describe 'process files with line endings in automatic mode' do
|
90
|
-
|
91
|
-
it 'should process a file with \n for line endings and within data fields' do
|
92
|
-
sep = "\n"
|
93
|
-
data = SmarterCSV.process("#{fixture_path}/carriage_returns_n.csv", {:row_sep => :auto})
|
94
|
-
data.flatten.size.should == 8
|
95
|
-
data[0][:name].should == "Anfield"
|
96
|
-
data[0][:street].should == "Anfield Road"
|
97
|
-
data[0][:city].should == "Liverpool"
|
98
|
-
data[1][:name].should == ["Highbury", "Highbury House"].join(sep)
|
99
|
-
data[2][:street].should == ["Sir Matt ", "Busby Way"].join(sep)
|
100
|
-
data[3][:city].should == ["Newcastle-upon-tyne ", "Tyne and Wear"].join(sep)
|
101
|
-
data[4][:name].should == ["White Hart Lane", "(The Lane)"].join(sep)
|
102
|
-
data[4][:street].should == ["Bill Nicholson Way ", "748 High Rd"].join(sep)
|
103
|
-
data[4][:city].should == ["Tottenham", "London"].join(sep)
|
104
|
-
data[5][:name].should == "Stamford Bridge"
|
105
|
-
data[5][:street].should == ["Fulham Road", "London"].join(sep)
|
106
|
-
data[5][:city].should be_nil
|
107
|
-
data[6][:name].should == ["Etihad Stadium", "Rowsley St", "Manchester"].join(sep)
|
108
|
-
data[7][:name].should == "Goodison"
|
109
|
-
data[7][:street].should == "Goodison Road"
|
110
|
-
data[7][:city].should == "Liverpool"
|
111
|
-
end
|
112
|
-
|
113
|
-
it 'should process a file with \r for line endings and within data fields' do
|
114
|
-
sep = "\r"
|
115
|
-
data = SmarterCSV.process("#{fixture_path}/carriage_returns_r.csv", {:row_sep => :auto})
|
116
|
-
data.flatten.size.should == 8
|
117
|
-
data[0][:name].should == "Anfield"
|
118
|
-
data[0][:street].should == "Anfield Road"
|
119
|
-
data[0][:city].should == "Liverpool"
|
120
|
-
data[1][:name].should == ["Highbury", "Highbury House"].join(sep)
|
121
|
-
data[2][:street].should == ["Sir Matt ", "Busby Way"].join(sep)
|
122
|
-
data[3][:city].should == ["Newcastle-upon-tyne ", "Tyne and Wear"].join(sep)
|
123
|
-
data[4][:name].should == ["White Hart Lane", "(The Lane)"].join(sep)
|
124
|
-
data[4][:street].should == ["Bill Nicholson Way ", "748 High Rd"].join(sep)
|
125
|
-
data[4][:city].should == ["Tottenham", "London"].join(sep)
|
126
|
-
data[5][:name].should == "Stamford Bridge"
|
127
|
-
data[5][:street].should == ["Fulham Road", "London"].join(sep)
|
128
|
-
data[5][:city].should be_nil
|
129
|
-
data[6][:name].should == ["Etihad Stadium", "Rowsley St", "Manchester"].join(sep)
|
130
|
-
data[7][:name].should == "Goodison"
|
131
|
-
data[7][:street].should == "Goodison Road"
|
132
|
-
data[7][:city].should == "Liverpool"
|
133
|
-
end
|
134
|
-
|
135
|
-
it 'should process a file with \r\n for line endings and within data fields' do
|
136
|
-
sep = "\r\n"
|
137
|
-
data = SmarterCSV.process("#{fixture_path}/carriage_returns_rn.csv", {:row_sep => :auto})
|
138
|
-
data.flatten.size.should == 8
|
139
|
-
data[0][:name].should == "Anfield"
|
140
|
-
data[0][:street].should == "Anfield Road"
|
141
|
-
data[0][:city].should == "Liverpool"
|
142
|
-
data[1][:name].should == ["Highbury", "Highbury House"].join(sep)
|
143
|
-
data[2][:street].should == ["Sir Matt ", "Busby Way"].join(sep)
|
144
|
-
data[3][:city].should == ["Newcastle-upon-tyne ", "Tyne and Wear"].join(sep)
|
145
|
-
data[4][:name].should == ["White Hart Lane", "(The Lane)"].join(sep)
|
146
|
-
data[4][:street].should == ["Bill Nicholson Way ", "748 High Rd"].join(sep)
|
147
|
-
data[4][:city].should == ["Tottenham", "London"].join(sep)
|
148
|
-
data[5][:name].should == "Stamford Bridge"
|
149
|
-
data[5][:street].should == ["Fulham Road", "London"].join(sep)
|
150
|
-
data[5][:city].should be_nil
|
151
|
-
data[6][:name].should == ["Etihad Stadium", "Rowsley St", "Manchester"].join(sep)
|
152
|
-
data[7][:name].should == "Goodison"
|
153
|
-
data[7][:street].should == "Goodison Road"
|
154
|
-
data[7][:city].should == "Liverpool"
|
155
|
-
end
|
156
|
-
|
157
|
-
it 'should process a file with more quoted text carriage return characters (\r) than line ending characters (\n)' do
|
158
|
-
row_sep = "\n"
|
159
|
-
text_sep = "\r"
|
160
|
-
data = SmarterCSV.process("#{fixture_path}/carriage_returns_quoted.csv", {:row_sep => :auto})
|
161
|
-
data.flatten.size.should == 2
|
162
|
-
data[0][:band].should == "New Order"
|
163
|
-
data[0][:members].should == ["Bernard Sumner", "Peter Hook", "Stephen Morris", "Gillian Gilbert"].join(text_sep)
|
164
|
-
data[0][:albums].should == ["Movement", "Power, Corruption and Lies", "Low-Life", "Brotherhood", "Substance"].join(text_sep)
|
165
|
-
data[1][:band].should == "Led Zeppelin"
|
166
|
-
data[1][:members].should == ["Jimmy Page", "Robert Plant", "John Bonham", "John Paul Jones"].join(text_sep)
|
167
|
-
data[1][:albums].should == ["Led Zeppelin", "Led Zeppelin II", "Led Zeppelin III", "Led Zeppelin IV"].join(text_sep)
|
168
|
-
end
|
169
|
-
|
170
|
-
end
|
@@ -1,14 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'loads_chunk_cornercase_csv_files' do
|
7
|
-
(0..5).each do |chunk_size| # test for all chunk-sizes
|
8
|
-
options = {:chunk_size => chunk_size, :remove_empty_hashes => true}
|
9
|
-
data = SmarterCSV.process("#{fixture_path}/chunk_cornercase.csv", options)
|
10
|
-
data.flatten.size.should == 5 # end-result must always be 5 rows
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
end
|
@@ -1,15 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'close file after using it' do
|
7
|
-
options = {:col_sep => "\cA", :row_sep => "\cB", :comment_regexp => /^#/, :strings_as_keys => true}
|
8
|
-
|
9
|
-
file = File.new("#{fixture_path}/binary.csv")
|
10
|
-
|
11
|
-
SmarterCSV.process(file, options)
|
12
|
-
|
13
|
-
file.closed?.should == true
|
14
|
-
end
|
15
|
-
end
|
@@ -1,11 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'loads_file_with_different_column_separator' do
|
7
|
-
options = {:col_sep => ';'}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/separator.csv", options)
|
9
|
-
data.flatten.size.should == 3
|
10
|
-
end
|
11
|
-
end
|
@@ -1,48 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'numeric conversion of values' do
|
6
|
-
it 'occurs by default' do
|
7
|
-
options = {}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/numeric.csv", options)
|
9
|
-
data.size.should == 3
|
10
|
-
|
11
|
-
# all the keys should be symbols
|
12
|
-
data.each do |hash|
|
13
|
-
hash[:wealth].should be_a_kind_of(Numeric) unless hash[:wealth].nil?
|
14
|
-
hash[:reference].should be_a_kind_of(Numeric) unless hash[:reference].nil?
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
it 'can be prevented for all values' do
|
19
|
-
options = { :convert_values_to_numeric => false }
|
20
|
-
data = SmarterCSV.process("#{fixture_path}/numeric.csv", options)
|
21
|
-
|
22
|
-
data.each do |hash|
|
23
|
-
hash[:wealth].should be_a_kind_of(String) unless hash[:wealth].nil?
|
24
|
-
hash[:reference].should be_a_kind_of(String) unless hash[:reference].nil?
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
it 'can be prevented for some keys' do
|
29
|
-
options = { :convert_values_to_numeric => { :except => :reference }}
|
30
|
-
data = SmarterCSV.process("#{fixture_path}/numeric.csv", options)
|
31
|
-
|
32
|
-
data.each do |hash|
|
33
|
-
hash[:wealth].should be_a_kind_of(Numeric) unless hash[:wealth].nil?
|
34
|
-
hash[:reference].should be_a_kind_of(String) unless hash[:reference].nil?
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
it 'can occur only for some keys' do
|
39
|
-
options = { :convert_values_to_numeric => { :only => :wealth }}
|
40
|
-
data = SmarterCSV.process("#{fixture_path}/numeric.csv", options)
|
41
|
-
|
42
|
-
data.each do |hash|
|
43
|
-
hash[:wealth].should be_a_kind_of(Numeric) unless hash[:wealth].nil?
|
44
|
-
hash[:reference].should be_a_kind_of(String) unless hash[:reference].nil?
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
@@ -1,17 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe "Hash.zip" do
|
4
|
-
it "constructs a new Hash from two Arrays" do
|
5
|
-
Hash.zip(["a", "b"], [1, 2]).should == { "a" => 1, "b" => 2 }
|
6
|
-
end
|
7
|
-
|
8
|
-
it "constructs an empty Hash if given no keys" do
|
9
|
-
Hash.zip([], []).should == {}
|
10
|
-
Hash.zip([], [1]).should == {}
|
11
|
-
end
|
12
|
-
|
13
|
-
it "uses nil values if there are more keys than values" do
|
14
|
-
Hash.zip(["a"], []).should == { "a" => nil }
|
15
|
-
Hash.zip(["a", "b"], [1]).should == { "a" => 1, "b" => nil }
|
16
|
-
end
|
17
|
-
end
|
@@ -1,21 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'loads_file_with_dashes_in_header_fields as strings' do
|
7
|
-
options = {:strings_as_keys => true}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/with_dashes.csv", options)
|
9
|
-
data.flatten.size.should == 5
|
10
|
-
data[0]['first_name'].should eq 'Dan'
|
11
|
-
data[0]['last_name'].should eq 'McAllister'
|
12
|
-
end
|
13
|
-
|
14
|
-
it 'loads_file_with_dashes_in_header_fields as symbols' do
|
15
|
-
options = {:strings_as_keys => false}
|
16
|
-
data = SmarterCSV.process("#{fixture_path}/with_dashes.csv", options)
|
17
|
-
data.flatten.size.should == 5
|
18
|
-
data[0][:first_name].should eq 'Dan'
|
19
|
-
data[0][:last_name].should eq 'McAllister'
|
20
|
-
end
|
21
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'not_downcase_headers' do
|
7
|
-
options = {:keep_original_headers => true}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
9
|
-
data.size.should == 5
|
10
|
-
# all the keys should be string
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == String}}
|
12
|
-
|
13
|
-
data.each do |item|
|
14
|
-
item.keys.each do |key|
|
15
|
-
['First Name','Last Name','Dogs','Cats','Birds','Fish'].should include( key )
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
data.each do |h|
|
20
|
-
h.size.should <= 6
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
@@ -1,25 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'remove_values_matching' do
|
7
|
-
options = {:remove_zero_values => true, :key_mapping => {:first_name => :vorname, :last_name => :nachname} }
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
9
|
-
data.size.should == 5
|
10
|
-
# all the keys should be symbols
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
12
|
-
|
13
|
-
data.each do |hash|
|
14
|
-
hash.keys.each do |key|
|
15
|
-
[:vorname, :nachname, :dogs, :cats, :birds, :fish].should include( key )
|
16
|
-
end
|
17
|
-
hash.values.should_not include( 0 )
|
18
|
-
end
|
19
|
-
|
20
|
-
data.each do |h|
|
21
|
-
h.size.should <= 6
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
end
|
@@ -1,43 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'process files with line endings explicitly pre-specified' do
|
6
|
-
it 'reads file with \n line endings' do
|
7
|
-
options = {:row_sep => "\n"}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_n.csv", options)
|
9
|
-
data.size.should == 3
|
10
|
-
end
|
11
|
-
|
12
|
-
it 'reads file with \r line endings' do
|
13
|
-
options = {:row_sep => "\r"}
|
14
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_r.csv", options)
|
15
|
-
data.size.should == 3
|
16
|
-
end
|
17
|
-
|
18
|
-
it 'reads file with \r\n line endings' do
|
19
|
-
options = {:row_sep => "\r\n"}
|
20
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_rn.csv", options)
|
21
|
-
data.size.should == 3
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
describe 'process files with line endings in automatic mode' do
|
26
|
-
it 'reads file with \n line endings' do
|
27
|
-
options = {:row_sep => :auto}
|
28
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_n.csv", options)
|
29
|
-
data.size.should == 3
|
30
|
-
end
|
31
|
-
|
32
|
-
it 'reads file with \r line endings' do
|
33
|
-
options = {:row_sep => :auto}
|
34
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_r.csv", options)
|
35
|
-
data.size.should == 3
|
36
|
-
end
|
37
|
-
|
38
|
-
it 'reads file with \r\n line endings' do
|
39
|
-
options = {:row_sep => :auto}
|
40
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_rn.csv", options)
|
41
|
-
data.size.should == 3
|
42
|
-
end
|
43
|
-
end
|
@@ -1,20 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'loads_basic_csv_file' do
|
7
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv")
|
8
|
-
data.size.should == 5
|
9
|
-
|
10
|
-
# all the keys should be symbols
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
12
|
-
data.each do |h|
|
13
|
-
h.keys.each do |key|
|
14
|
-
[:first_name, :last_name, :dogs, :cats, :birds, :fish].should include( key )
|
15
|
-
end
|
16
|
-
h.size.should <= 6
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
end
|
@@ -1,21 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'malformed_csv' do
|
6
|
-
subject { lambda { SmarterCSV.process(csv_path) } }
|
7
|
-
|
8
|
-
context "malformed header" do
|
9
|
-
let(:csv_path) { "#{fixture_path}/malformed_header.csv" }
|
10
|
-
it { should raise_error(CSV::MalformedCSVError) }
|
11
|
-
it { should raise_error(/(Missing or stray quote in line 1|CSV::MalformedCSVError)/) }
|
12
|
-
it { should raise_error(CSV::MalformedCSVError) }
|
13
|
-
end
|
14
|
-
|
15
|
-
context "malformed content" do
|
16
|
-
let(:csv_path) { "#{fixture_path}/malformed.csv" }
|
17
|
-
it { should raise_error(CSV::MalformedCSVError) }
|
18
|
-
it { should raise_error(/(Missing or stray quote in line 1|CSV::MalformedCSVError)/) }
|
19
|
-
it { should raise_error(CSV::MalformedCSVError) }
|
20
|
-
end
|
21
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'loads_csv_file_without_header' do
|
7
|
-
options = {:headers_in_file => false, :user_provided_headers => [:a,:b,:c,:d,:e,:f]}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/no_header.csv", options)
|
9
|
-
data.size.should == 5
|
10
|
-
# all the keys should be symbols
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
12
|
-
|
13
|
-
data.each do |item|
|
14
|
-
item.keys.each do |key|
|
15
|
-
[:a,:b,:c,:d,:e,:f].should include( key )
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
data.each do |h|
|
20
|
-
h.size.should <= 6
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'not_downcase_headers' do
|
7
|
-
options = {:downcase_header => false}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
9
|
-
data.size.should == 5
|
10
|
-
# all the keys should be symbols
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
12
|
-
|
13
|
-
data.each do |item|
|
14
|
-
item.keys.each do |key|
|
15
|
-
[:First_Name, :Last_Name, :Dogs, :Cats, :Birds, :Fish].should include( key )
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
data.each do |h|
|
20
|
-
h.size.should <= 6
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
@@ -1,23 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
|
7
|
-
it 'loads_file_with_quoted_fields' do
|
8
|
-
options = {}
|
9
|
-
data = SmarterCSV.process("#{fixture_path}/quoted.csv", options)
|
10
|
-
data.flatten.size.should == 4
|
11
|
-
data[1][:model].should eq 'Venture "Extended Edition"'
|
12
|
-
data[1][:description].should be_nil
|
13
|
-
data[2][:model].should eq 'Venture "Extended Edition, Very Large"'
|
14
|
-
data[2][:description].should be_nil
|
15
|
-
data.each do |h|
|
16
|
-
h[:year].class.should eq Fixnum
|
17
|
-
h[:make].should_not be_nil
|
18
|
-
h[:model].should_not be_nil
|
19
|
-
h[:price].class.should eq Float
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
end
|
@@ -1,13 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'remove_empty_values' do
|
7
|
-
options = {:row_sep => :auto, :remove_empty_values => true}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/empty.csv", options)
|
9
|
-
data.size.should == 1
|
10
|
-
data[0].keys.should == [:not_empty_1, :not_empty_2, :not_empty_3]
|
11
|
-
end
|
12
|
-
|
13
|
-
end
|
@@ -1,25 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'remove_values_matching' do
|
7
|
-
options = {:remove_zero_values => true, :key_mapping => {:first_name => :vorname, :last_name => :nachname, :fish => nil} }
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
9
|
-
data.size.should == 5
|
10
|
-
# all the keys should be symbols
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
12
|
-
|
13
|
-
data.each do |hash|
|
14
|
-
hash.keys.each do |key|
|
15
|
-
[:vorname, :nachname, :dogs, :cats, :birds].should include( key )
|
16
|
-
end
|
17
|
-
hash.values.should_not include( 0 )
|
18
|
-
end
|
19
|
-
|
20
|
-
data.each do |h|
|
21
|
-
h.size.should <= 6
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
end
|
@@ -1,35 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe ':remove_unmapped_keys option' do
|
6
|
-
|
7
|
-
it 'it has no effect on loading a file without options' do
|
8
|
-
options = {}
|
9
|
-
data = SmarterCSV.process("#{fixture_path}/lots_of_columns.csv", options)
|
10
|
-
data.size.should eq 1
|
11
|
-
data.first.size.should eq 474 # there are some empty rows in the fixture
|
12
|
-
end
|
13
|
-
|
14
|
-
it 'it has no effect if provided without :key_mapping' do
|
15
|
-
options = {:remove_unmapped_keys => true}
|
16
|
-
data = SmarterCSV.process("#{fixture_path}/lots_of_columns.csv", options)
|
17
|
-
data.size.should eq 1
|
18
|
-
data.first.size.should eq 474 # there are some empty rows in the fixture
|
19
|
-
end
|
20
|
-
|
21
|
-
it 'it defaults to false and has no effect if :key_mapping is provided without :remove_unmapped_keys' do
|
22
|
-
options = {:key_mapping => {:column_0 => :one, :column_15 => :two, :column_42 => :three}}
|
23
|
-
data = SmarterCSV.process("#{fixture_path}/lots_of_columns.csv", options)
|
24
|
-
data.size.should eq 1
|
25
|
-
data.first.size.should eq 474 # there are some empty rows in the fixture
|
26
|
-
end
|
27
|
-
|
28
|
-
it 'it removes non-mapped keys/columns when set to true and :key_mapping is provided' do
|
29
|
-
options = {:remove_unmapped_keys => true, :key_mapping => {:column_0 => :one, :column_15 => :two, :column_42 => :three}}
|
30
|
-
data = SmarterCSV.process("#{fixture_path}/lots_of_columns.csv", options)
|
31
|
-
data.size.should eq 1
|
32
|
-
data.first.size.should eq 2 # column_15 is empty
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
@@ -1,26 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'remove_values_matching' do
|
7
|
-
options = {:remove_zero_values => true, :remove_empty_values => true, :remove_values_matching => /^\d+$/}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
9
|
-
data.size.should == 5
|
10
|
-
# all the keys should be symbols
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
12
|
-
|
13
|
-
data.each do |hash|
|
14
|
-
hash.keys.each do |key|
|
15
|
-
[:first_name, :last_name].should include( key )
|
16
|
-
end
|
17
|
-
hash.values.each{|x| x.class.should be == String}
|
18
|
-
hash.values.should_not include( 0 )
|
19
|
-
end
|
20
|
-
|
21
|
-
data.each do |h|
|
22
|
-
h.size.should <= 6
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
end
|
@@ -1,25 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'remove_zero_values' do
|
7
|
-
options = {:remove_zero_values => true, :remove_empty_values => true}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
9
|
-
data.size.should == 5
|
10
|
-
# all the keys should be symbols
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
12
|
-
|
13
|
-
data.each do |hash|
|
14
|
-
hash.keys.each do |key|
|
15
|
-
[:first_name, :last_name, :dogs, :cats, :birds, :fish].should include( key )
|
16
|
-
end
|
17
|
-
hash.values.should_not include( 0 )
|
18
|
-
end
|
19
|
-
|
20
|
-
data.each do |h|
|
21
|
-
h.size.should <= 6
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
end
|