smarter_csv 1.6.1 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +133 -0
- data/CHANGELOG.md +22 -1
- data/CONTRIBUTORS.md +3 -0
- data/Gemfile +7 -4
- data/README.md +8 -6
- data/Rakefile +15 -13
- data/ext/smarter_csv/extconf.rb +14 -0
- data/ext/smarter_csv/smarter_csv.c +86 -0
- data/lib/extensions/hash.rb +4 -2
- data/lib/smarter_csv/version.rb +3 -1
- data/lib/smarter_csv.rb +519 -10
- data/smarter_csv.gemspec +22 -7
- metadata +54 -176
- data/.gitignore +0 -10
- data/.rspec +0 -2
- data/.travis.yml +0 -27
- data/lib/smarter_csv/smarter_csv.rb +0 -461
- data/spec/fixtures/additional_separator.csv +0 -6
- data/spec/fixtures/basic.csv +0 -8
- data/spec/fixtures/binary.csv +0 -1
- data/spec/fixtures/carriage_returns_n.csv +0 -18
- data/spec/fixtures/carriage_returns_quoted.csv +0 -3
- data/spec/fixtures/carriage_returns_r.csv +0 -1
- data/spec/fixtures/carriage_returns_rn.csv +0 -18
- data/spec/fixtures/chunk_cornercase.csv +0 -10
- data/spec/fixtures/duplicate_headers.csv +0 -3
- data/spec/fixtures/empty.csv +0 -5
- data/spec/fixtures/empty_columns_1.csv +0 -2
- data/spec/fixtures/empty_columns_2.csv +0 -2
- data/spec/fixtures/hard_sample.csv +0 -2
- data/spec/fixtures/ignore_comments.csv +0 -11
- data/spec/fixtures/ignore_comments2.csv +0 -3
- data/spec/fixtures/key_mapping.csv +0 -2
- data/spec/fixtures/line_endings_n.csv +0 -4
- data/spec/fixtures/line_endings_r.csv +0 -1
- data/spec/fixtures/line_endings_rn.csv +0 -4
- data/spec/fixtures/lots_of_columns.csv +0 -2
- data/spec/fixtures/malformed.csv +0 -3
- data/spec/fixtures/malformed_header.csv +0 -3
- data/spec/fixtures/money.csv +0 -3
- data/spec/fixtures/no_header.csv +0 -7
- data/spec/fixtures/numeric.csv +0 -5
- data/spec/fixtures/pets.csv +0 -5
- data/spec/fixtures/problematic.csv +0 -8
- data/spec/fixtures/quote_char.csv +0 -9
- data/spec/fixtures/quoted.csv +0 -5
- data/spec/fixtures/quoted2.csv +0 -4
- data/spec/fixtures/separator_colon.csv +0 -4
- data/spec/fixtures/separator_comma.csv +0 -4
- data/spec/fixtures/separator_pipe.csv +0 -4
- data/spec/fixtures/separator_semi.csv +0 -4
- data/spec/fixtures/separator_tab.csv +0 -4
- data/spec/fixtures/skip_lines.csv +0 -8
- data/spec/fixtures/trading.csv +0 -3
- data/spec/fixtures/user_import.csv +0 -3
- data/spec/fixtures/valid_unicode.csv +0 -5
- data/spec/fixtures/with_dashes.csv +0 -8
- data/spec/fixtures/with_dates.csv +0 -4
- data/spec/smarter_csv/additional_separator_spec.rb +0 -45
- data/spec/smarter_csv/binary_file2_spec.rb +0 -24
- data/spec/smarter_csv/binary_file_spec.rb +0 -22
- data/spec/smarter_csv/blank_spec.rb +0 -55
- data/spec/smarter_csv/carriage_return_spec.rb +0 -190
- data/spec/smarter_csv/chunked_reading_spec.rb +0 -14
- data/spec/smarter_csv/close_file_spec.rb +0 -15
- data/spec/smarter_csv/column_separator_spec.rb +0 -95
- data/spec/smarter_csv/convert_values_to_numeric_spec.rb +0 -48
- data/spec/smarter_csv/duplicate_headers_spec.rb +0 -76
- data/spec/smarter_csv/empty_columns_spec.rb +0 -74
- data/spec/smarter_csv/extenstions_spec.rb +0 -17
- data/spec/smarter_csv/hard_sample_spec.rb +0 -24
- data/spec/smarter_csv/header_transformation_spec.rb +0 -21
- data/spec/smarter_csv/ignore_comments_spec.rb +0 -45
- data/spec/smarter_csv/invalid_headers_spec.rb +0 -38
- data/spec/smarter_csv/keep_headers_spec.rb +0 -24
- data/spec/smarter_csv/key_mapping_spec.rb +0 -56
- data/spec/smarter_csv/line_ending_spec.rb +0 -43
- data/spec/smarter_csv/load_basic_spec.rb +0 -20
- data/spec/smarter_csv/malformed_spec.rb +0 -25
- data/spec/smarter_csv/no_header_spec.rb +0 -29
- data/spec/smarter_csv/not_downcase_header_spec.rb +0 -24
- data/spec/smarter_csv/parse/column_separator_spec.rb +0 -61
- data/spec/smarter_csv/parse/old_csv_library_spec.rb +0 -74
- data/spec/smarter_csv/parse/rfc4180_and_more_spec.rb +0 -170
- data/spec/smarter_csv/problematic.rb +0 -34
- data/spec/smarter_csv/quoted_spec.rb +0 -52
- data/spec/smarter_csv/remove_empty_values_spec.rb +0 -13
- data/spec/smarter_csv/remove_keys_from_hashes_spec.rb +0 -25
- data/spec/smarter_csv/remove_not_mapped_keys_spec.rb +0 -35
- data/spec/smarter_csv/remove_values_matching_spec.rb +0 -26
- data/spec/smarter_csv/remove_zero_values_spec.rb +0 -25
- data/spec/smarter_csv/skip_lines_spec.rb +0 -29
- data/spec/smarter_csv/strings_as_keys_spec.rb +0 -24
- data/spec/smarter_csv/strip_chars_from_headers_spec.rb +0 -24
- data/spec/smarter_csv/trading_spec.rb +0 -25
- data/spec/smarter_csv/valid_unicode_spec.rb +0 -94
- data/spec/smarter_csv/value_converters_spec.rb +0 -52
- data/spec/spec/spec_helper.rb +0 -17
- data/spec/spec.opts +0 -2
- data/spec/spec_helper.rb +0 -21
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'duplicate headers' do
|
|
6
|
-
describe 'without special handling / default behavior' do
|
|
7
|
-
it 'raises error on duplicate headers' do
|
|
8
|
-
expect {
|
|
9
|
-
SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", {})
|
|
10
|
-
}.to raise_exception(SmarterCSV::DuplicateHeaders)
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
it 'raises error on duplicate given headers' do
|
|
14
|
-
expect {
|
|
15
|
-
options = {:user_provided_headers => [:a,:b,:c,:d,:a]}
|
|
16
|
-
SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
|
17
|
-
}.to raise_exception(SmarterCSV::DuplicateHeaders)
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
it 'does not raise error on missing mapped headers and includes missing headers in message' do
|
|
21
|
-
# the mapping is right, but the underlying csv file is bad
|
|
22
|
-
options = {:key_mapping => {:email => :a, :firstname => :b, :lastname => :c, :manager_email => :d, :age => :e} }
|
|
23
|
-
expect {
|
|
24
|
-
SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
|
25
|
-
}.not_to raise_exception(SmarterCSV::KeyMappingError)
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
describe 'with special handling' do
|
|
30
|
-
context 'with given suffix' do
|
|
31
|
-
let(:options) { {duplicate_header_suffix: '_'} }
|
|
32
|
-
|
|
33
|
-
it 'reads whole file' do
|
|
34
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
|
35
|
-
expect(data.size).to eq 2
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
it 'generates the correct keys' do
|
|
39
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
|
40
|
-
expect(data.first.keys).to eq [:email, :firstname, :lastname, :email_2, :age]
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
it 'enumerates when duplicate headers are given' do
|
|
44
|
-
options.merge!({:user_provided_headers => [:a,:b,:c,:a,:a]})
|
|
45
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
|
46
|
-
expect(data.first.keys).to eq [:a, :b, :c, :a_2, :a_3]
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
it 'can remap duplicated headers' do
|
|
50
|
-
options.merge!({:key_mapping => {:email => :a, :firstname => :b, :lastname => :c, :email_2 => :d, :age => :e}})
|
|
51
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
|
52
|
-
expect(data.first).to eq({a: 'tom@bla.com', b: 'Tom', c: 'Sawyer', d: 'mike@bla.com', e: 34})
|
|
53
|
-
end
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
context 'with empty suffix' do
|
|
57
|
-
let(:options) { {duplicate_header_suffix: ''} }
|
|
58
|
-
|
|
59
|
-
it 'reads whole file' do
|
|
60
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
|
61
|
-
expect(data.size).to eq 2
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
it 'generates the correct keys' do
|
|
65
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
|
66
|
-
expect(data.first.keys).to eq [:email, :firstname, :lastname, :email2, :age]
|
|
67
|
-
end
|
|
68
|
-
|
|
69
|
-
it 'enumerates when duplicate headers are given' do
|
|
70
|
-
options.merge!({:user_provided_headers => [:a,:b,:c,:a,:a]})
|
|
71
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
|
72
|
-
expect(data.first.keys).to eq [:a, :b, :c, :a2, :a3]
|
|
73
|
-
end
|
|
74
|
-
end
|
|
75
|
-
end
|
|
76
|
-
end
|
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'can handle empty columns' do
|
|
6
|
-
|
|
7
|
-
describe 'default behavior' do
|
|
8
|
-
it 'has empty columns at end' do
|
|
9
|
-
data = SmarterCSV.process("#{fixture_path}/empty_columns_1.csv")
|
|
10
|
-
data.size.should eq 1
|
|
11
|
-
item = data.first
|
|
12
|
-
item[:id].should == 123
|
|
13
|
-
item[:col1].should == nil
|
|
14
|
-
item[:col2].should == nil
|
|
15
|
-
item[:col3].should == nil
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
it 'has empty columns in the middle' do
|
|
19
|
-
data = SmarterCSV.process("#{fixture_path}/empty_columns_2.csv")
|
|
20
|
-
data.size.should eq 1
|
|
21
|
-
item = data.first
|
|
22
|
-
item[:id].should == 123
|
|
23
|
-
item[:col1].should == nil
|
|
24
|
-
item[:col2].should == nil
|
|
25
|
-
item[:col3].should == 1
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
describe 'with remove_empty_values: true' do
|
|
30
|
-
options = {remove_empty_values: true}
|
|
31
|
-
it 'has empty columns at end' do
|
|
32
|
-
data = SmarterCSV.process("#{fixture_path}/empty_columns_1.csv", options)
|
|
33
|
-
data.size.should eq 1
|
|
34
|
-
item = data.first
|
|
35
|
-
item[:id].should == 123
|
|
36
|
-
item[:col1].should == nil
|
|
37
|
-
item[:col2].should == nil
|
|
38
|
-
item[:col3].should == nil
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
it 'has empty columns in the middle' do
|
|
42
|
-
data = SmarterCSV.process("#{fixture_path}/empty_columns_2.csv", options)
|
|
43
|
-
data.size.should eq 1
|
|
44
|
-
item = data.first
|
|
45
|
-
item[:id].should == 123
|
|
46
|
-
item[:col1].should == nil
|
|
47
|
-
item[:col2].should == nil
|
|
48
|
-
item[:col3].should == 1
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
describe 'with remove_empty_values: false' do
|
|
53
|
-
options = {remove_empty_values: false}
|
|
54
|
-
it 'has empty columns at end' do
|
|
55
|
-
data = SmarterCSV.process("#{fixture_path}/empty_columns_1.csv", options)
|
|
56
|
-
data.size.should eq 1
|
|
57
|
-
item = data.first
|
|
58
|
-
item[:id].should == 123
|
|
59
|
-
item[:col1].should == ''
|
|
60
|
-
item[:col2].should == ''
|
|
61
|
-
item[:col3].should == ''
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
it 'has empty columns in the middle' do
|
|
65
|
-
data = SmarterCSV.process("#{fixture_path}/empty_columns_2.csv", options)
|
|
66
|
-
data.size.should eq 1
|
|
67
|
-
item = data.first
|
|
68
|
-
item[:id].should == 123
|
|
69
|
-
item[:col1].should == ''
|
|
70
|
-
item[:col2].should == ''
|
|
71
|
-
item[:col3].should == 1
|
|
72
|
-
end
|
|
73
|
-
end
|
|
74
|
-
end
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
describe "Hash.zip" do
|
|
4
|
-
it "constructs a new Hash from two Arrays" do
|
|
5
|
-
Hash.zip(["a", "b"], [1, 2]).should == { "a" => 1, "b" => 2 }
|
|
6
|
-
end
|
|
7
|
-
|
|
8
|
-
it "constructs an empty Hash if given no keys" do
|
|
9
|
-
Hash.zip([], []).should == {}
|
|
10
|
-
Hash.zip([], [1]).should == {}
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
it "uses nil values if there are more keys than values" do
|
|
14
|
-
Hash.zip(["a"], []).should == { "a" => nil }
|
|
15
|
-
Hash.zip(["a", "b"], [1]).should == { "a" => 1, "b" => nil }
|
|
16
|
-
end
|
|
17
|
-
end
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'can handle the difficult CSV file' do
|
|
6
|
-
|
|
7
|
-
it 'loads the data with default values' do
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/hard_sample.csv")
|
|
9
|
-
data.size.should eq 1
|
|
10
|
-
item = data.first
|
|
11
|
-
item.keys.count.should == 48
|
|
12
|
-
item[:name].should == '#MR1220817'
|
|
13
|
-
item[:shipping_method].should == 'Livraison Standard GRATUITE, 2-5 jours avec suivi'
|
|
14
|
-
item[:lineitem_name].should == 'Cire Épilation Nacrée'
|
|
15
|
-
item[:phone].should == 3366012111111
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
# the main problem is the data line starting with a # character, but not being a comment
|
|
19
|
-
it 'fails to load the CSV file with incorrectly set comment_regexp' do
|
|
20
|
-
options = {comment_regexp: /\A#/ }
|
|
21
|
-
data = SmarterCSV.process("#{fixture_path}/hard_sample.csv", options)
|
|
22
|
-
data.size.should eq 0
|
|
23
|
-
end
|
|
24
|
-
end
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
it 'loads_file_with_dashes_in_header_fields as strings' do
|
|
7
|
-
options = {:strings_as_keys => true}
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/with_dashes.csv", options)
|
|
9
|
-
data.flatten.size.should == 5
|
|
10
|
-
data[0]['first_name'].should eq 'Dan'
|
|
11
|
-
data[0]['last_name'].should eq 'McAllister'
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
it 'loads_file_with_dashes_in_header_fields as symbols' do
|
|
15
|
-
options = {:strings_as_keys => false}
|
|
16
|
-
data = SmarterCSV.process("#{fixture_path}/with_dashes.csv", options)
|
|
17
|
-
data.flatten.size.should == 5
|
|
18
|
-
data[0][:first_name].should eq 'Dan'
|
|
19
|
-
data[0][:last_name].should eq 'McAllister'
|
|
20
|
-
end
|
|
21
|
-
end
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
it 'by default does not ignore comments in CSV files' do
|
|
7
|
-
options = {}
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/ignore_comments.csv", options)
|
|
9
|
-
|
|
10
|
-
data.size.should eq 8
|
|
11
|
-
|
|
12
|
-
# all the keys should be symbols
|
|
13
|
-
data.each{|item| item.keys.each{|x| x.is_a?(Symbol).should be_truthy}}
|
|
14
|
-
data.each do |h|
|
|
15
|
-
h.keys.each do |key|
|
|
16
|
-
[:"not_a_comment#first_name", :last_name, :dogs, :cats, :birds, :fish].should include( key )
|
|
17
|
-
end
|
|
18
|
-
end
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
it 'ignore comments in CSV files using comment_regexp' do
|
|
22
|
-
options = {comment_regexp: /\A#/}
|
|
23
|
-
data = SmarterCSV.process("#{fixture_path}/ignore_comments.csv", options)
|
|
24
|
-
|
|
25
|
-
data.size.should eq 5
|
|
26
|
-
|
|
27
|
-
# all the keys should be symbols
|
|
28
|
-
data.each{|item| item.keys.each{|x| x.is_a?(Symbol).should be_truthy}}
|
|
29
|
-
data.each do |h|
|
|
30
|
-
h.keys.each do |key|
|
|
31
|
-
[:"not_a_comment#first_name", :last_name, :dogs, :cats, :birds, :fish].should include( key )
|
|
32
|
-
end
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
it 'ignore comments in CSV files with CRLF' do
|
|
37
|
-
options = {row_sep: "\r\n"}
|
|
38
|
-
data = SmarterCSV.process("#{fixture_path}/ignore_comments2.csv", options)
|
|
39
|
-
|
|
40
|
-
# all the keys should be symbols
|
|
41
|
-
data.size.should eq 1
|
|
42
|
-
data.first[:h1].should eq 'a'
|
|
43
|
-
data.first[:h2].should eq "b\r\n#c"
|
|
44
|
-
end
|
|
45
|
-
end
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'test exceptions for invalid headers' do
|
|
6
|
-
it 'does not raise an error if no required headers are given' do
|
|
7
|
-
options = {:required_headers => nil} # order does not matter
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/user_import.csv", options)
|
|
9
|
-
data.size.should eq 2
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
it 'does not raise an error if no required headers are given' do
|
|
13
|
-
options = {:required_headers => []} # order does not matter
|
|
14
|
-
data = SmarterCSV.process("#{fixture_path}/user_import.csv", options)
|
|
15
|
-
data.size.should eq 2
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
it 'does not raise an error if the required headers are present' do
|
|
19
|
-
options = {:required_headers => [:lastname,:email,:firstname,:manager_email]} # order does not matter
|
|
20
|
-
data = SmarterCSV.process("#{fixture_path}/user_import.csv", options)
|
|
21
|
-
data.size.should eq 2
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
it 'raises an error if a required header is missing' do
|
|
25
|
-
expect {
|
|
26
|
-
options = {:required_headers => [:lastname,:email,:employee_id,:firstname,:manager_email]} # order does not matter
|
|
27
|
-
SmarterCSV.process("#{fixture_path}/user_import.csv", options)
|
|
28
|
-
}.to raise_exception(SmarterCSV::MissingHeaders)
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
it 'does not raise error on missing mapped headers and includes missing headers in message' do
|
|
32
|
-
# :age does not exist in the CSV header
|
|
33
|
-
options = {:key_mapping => {:email => :a, :firstname => :b, :lastname => :c, :manager_email => :d, :age => :e} }
|
|
34
|
-
expect {
|
|
35
|
-
SmarterCSV.process("#{fixture_path}/user_import.csv", options)
|
|
36
|
-
}.not_to raise_exception(SmarterCSV::KeyMappingError)
|
|
37
|
-
end
|
|
38
|
-
end
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
it 'not_downcase_headers' do
|
|
7
|
-
options = {:keep_original_headers => true}
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
|
9
|
-
data.size.should == 5
|
|
10
|
-
# all the keys should be string
|
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == String}}
|
|
12
|
-
|
|
13
|
-
data.each do |item|
|
|
14
|
-
item.keys.each do |key|
|
|
15
|
-
['First Name','Last Name','Dogs','Cats','Birds','Fish'].should include( key )
|
|
16
|
-
end
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
data.each do |h|
|
|
20
|
-
h.size.should <= 6
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
end
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
it 'remove_values_matching' do
|
|
7
|
-
options = {:remove_zero_values => true, :key_mapping => {:first_name => :vorname, :last_name => :nachname} }
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
|
9
|
-
data.size.should == 5
|
|
10
|
-
# all the keys should be symbols
|
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
|
12
|
-
|
|
13
|
-
data.each do |hash|
|
|
14
|
-
hash.keys.each do |key|
|
|
15
|
-
[:vorname, :nachname, :dogs, :cats, :birds, :fish].should include( key )
|
|
16
|
-
end
|
|
17
|
-
hash.values.should_not include( 0 )
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
data.each do |h|
|
|
21
|
-
h.size.should <= 6
|
|
22
|
-
end
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
describe 'when keep_original_headers' do
|
|
26
|
-
it 'without key mapping' do
|
|
27
|
-
options = {:keep_original_headers => true}
|
|
28
|
-
data = SmarterCSV.process("#{fixture_path}/key_mapping.csv", options)
|
|
29
|
-
data.size.should == 1
|
|
30
|
-
data.first.keys.should == ['THIS', 'THAT', 'other']
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
it 'sets key_mapping to a symbol' do
|
|
34
|
-
options = {:keep_original_headers => true, :key_mapping => {'other' => :other}}
|
|
35
|
-
data = SmarterCSV.process("#{fixture_path}/key_mapping.csv", options)
|
|
36
|
-
data.size.should == 1
|
|
37
|
-
data.first.keys.should == ['THIS', 'THAT', :other]
|
|
38
|
-
end
|
|
39
|
-
|
|
40
|
-
# this previously would set the key to a symbol :OTHER, which was a bug!
|
|
41
|
-
it 'sets key_mapping to a string' do
|
|
42
|
-
options = {:keep_original_headers => true, :key_mapping => {'other' => 'OTHER'}}
|
|
43
|
-
data = SmarterCSV.process("#{fixture_path}/key_mapping.csv", options)
|
|
44
|
-
data.size.should == 1
|
|
45
|
-
data.first.keys.should == ['THIS', 'THAT', 'OTHER']
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
# users now have to explicitly set this to a symbol, or change the expected keys to be strings.
|
|
49
|
-
it 'sets key_mapping to a symbol' do
|
|
50
|
-
options = {:keep_original_headers => true, :key_mapping => {'other' => :OTHER}}
|
|
51
|
-
data = SmarterCSV.process("#{fixture_path}/key_mapping.csv", options)
|
|
52
|
-
data.size.should == 1
|
|
53
|
-
data.first.keys.should == ['THIS', 'THAT', :OTHER]
|
|
54
|
-
end
|
|
55
|
-
end
|
|
56
|
-
end
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'process files with line endings explicitly pre-specified' do
|
|
6
|
-
it 'reads file with \n line endings' do
|
|
7
|
-
options = {:row_sep => "\n"}
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_n.csv", options)
|
|
9
|
-
data.size.should == 3
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
it 'reads file with \r line endings' do
|
|
13
|
-
options = {:row_sep => "\r"}
|
|
14
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_r.csv", options)
|
|
15
|
-
data.size.should == 3
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
it 'reads file with \r\n line endings' do
|
|
19
|
-
options = {:row_sep => "\r\n"}
|
|
20
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_rn.csv", options)
|
|
21
|
-
data.size.should == 3
|
|
22
|
-
end
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
describe 'process files with line endings in automatic mode' do
|
|
26
|
-
it 'reads file with \n line endings' do
|
|
27
|
-
options = {:row_sep => :auto}
|
|
28
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_n.csv", options)
|
|
29
|
-
data.size.should == 3
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
it 'reads file with \r line endings' do
|
|
33
|
-
options = {:row_sep => :auto}
|
|
34
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_r.csv", options)
|
|
35
|
-
data.size.should == 3
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
it 'reads file with \r\n line endings' do
|
|
39
|
-
options = {:row_sep => :auto}
|
|
40
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_rn.csv", options)
|
|
41
|
-
data.size.should == 3
|
|
42
|
-
end
|
|
43
|
-
end
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
it 'loads_basic_csv_file' do
|
|
7
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv")
|
|
8
|
-
data.size.should == 5
|
|
9
|
-
|
|
10
|
-
# all the keys should be symbols
|
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
|
12
|
-
data.each do |h|
|
|
13
|
-
h.keys.each do |key|
|
|
14
|
-
[:first_name, :last_name, :dogs, :cats, :birds, :fish].should include( key )
|
|
15
|
-
end
|
|
16
|
-
h.size.should <= 6
|
|
17
|
-
end
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
end
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
# according to RFC-4180 quotes inside of "words" shouldbe doubled, but our parser is robust against that.
|
|
6
|
-
describe 'malformed CSV quotes' do
|
|
7
|
-
context "malformed quotes in header" do
|
|
8
|
-
let(:csv_path) { "#{fixture_path}/malformed_header.csv" }
|
|
9
|
-
it 'should be resilient against single quotes' do
|
|
10
|
-
data = SmarterCSV.process(csv_path)
|
|
11
|
-
expect(data[0]).to eq({:name=>"Arnold Schwarzenegger", :dobdob=>"1947-07-30"})
|
|
12
|
-
expect(data[1]).to eq({:name=>"Jeff Bridges", :dobdob=>"1949-12-04"})
|
|
13
|
-
end
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
context "malformed quotes in content" do
|
|
17
|
-
let(:csv_path) { "#{fixture_path}/malformed.csv" }
|
|
18
|
-
|
|
19
|
-
it 'should be resilient against single quotes' do
|
|
20
|
-
data = SmarterCSV.process(csv_path)
|
|
21
|
-
expect(data[0]).to eq({:name=>"Arnold Schwarzenegger", :dob=>"1947-07-30"})
|
|
22
|
-
expect(data[1]).to eq({:name=>"Jeff \"the dude\" Bridges", :dob=>"1949-12-04"})
|
|
23
|
-
end
|
|
24
|
-
end
|
|
25
|
-
end
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'no header in file' do
|
|
6
|
-
let(:headers) { [:a,:b,:c,:d,:e,:f] }
|
|
7
|
-
let(:options) { {:headers_in_file => false, :user_provided_headers => headers} }
|
|
8
|
-
subject(:data) { SmarterCSV.process("#{fixture_path}/no_header.csv", options) }
|
|
9
|
-
|
|
10
|
-
it 'load the correct number of records' do
|
|
11
|
-
data.size.should == 5
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
it 'uses given symbols for all records' do
|
|
15
|
-
data.each do |item|
|
|
16
|
-
item.keys.each do |key|
|
|
17
|
-
[:a,:b,:c,:d,:e,:f].should include( key )
|
|
18
|
-
end
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
it 'loads the correct data' do
|
|
23
|
-
data[0].should == {a: "Dan", b: "McAllister", c: 2, d: 0}
|
|
24
|
-
data[1].should == {a: "Lucy", b: "Laweless", d: 5, e: 0}
|
|
25
|
-
data[2].should == {a: "Miles", b: "O'Brian", c: 0, d: 0, e: 0, f: 21}
|
|
26
|
-
data[3].should == {a: "Nancy", b: "Homes", c: 2, d: 0, e: 1}
|
|
27
|
-
data[4].should == {a: "Hernán", b: "Curaçon", c: 3, d: 0, e: 0}
|
|
28
|
-
end
|
|
29
|
-
end
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
fixture_path = 'spec/fixtures'
|
|
4
|
-
|
|
5
|
-
describe 'be_able_to' do
|
|
6
|
-
it 'not_downcase_headers' do
|
|
7
|
-
options = {:downcase_header => false}
|
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
|
9
|
-
data.size.should == 5
|
|
10
|
-
# all the keys should be symbols
|
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
|
12
|
-
|
|
13
|
-
data.each do |item|
|
|
14
|
-
item.keys.each do |key|
|
|
15
|
-
[:First_Name, :Last_Name, :Dogs, :Cats, :Birds, :Fish].should include( key )
|
|
16
|
-
end
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
data.each do |h|
|
|
20
|
-
h.size.should <= 6
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
end
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
describe 'parse with col_sep' do
|
|
4
|
-
let(:options) { {quote_char: '"'} }
|
|
5
|
-
|
|
6
|
-
it 'parses with comma' do
|
|
7
|
-
line = "a,b,,d"
|
|
8
|
-
options.merge!({col_sep: ","})
|
|
9
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
|
10
|
-
expect(array).to eq ['a', 'b', '', 'd']
|
|
11
|
-
expect(array_size).to eq 4
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
it 'parses trailing commas' do
|
|
15
|
-
line = "a,b,c,,"
|
|
16
|
-
options.merge!({col_sep: ","})
|
|
17
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
|
18
|
-
expect(array).to eq ['a', 'b', 'c', '', '']
|
|
19
|
-
expect(array_size).to eq 5
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
it 'parses with space' do
|
|
23
|
-
line = "a b d"
|
|
24
|
-
options.merge!({col_sep: " "})
|
|
25
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
|
26
|
-
expect(array).to eq ['a', 'b', '', 'd']
|
|
27
|
-
expect(array_size).to eq 4
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
it 'parses with tab' do
|
|
31
|
-
line = "a\tb\t\td"
|
|
32
|
-
options.merge!({col_sep: "\t"})
|
|
33
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
|
34
|
-
expect(array).to eq ['a', 'b', '', 'd']
|
|
35
|
-
expect(array_size).to eq 4
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
it 'parses with multiple space separator' do
|
|
39
|
-
line = "a b d"
|
|
40
|
-
options.merge!({col_sep: " "})
|
|
41
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
|
42
|
-
expect(array).to eq ['a b', '', 'd']
|
|
43
|
-
expect(array_size).to eq 3
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
it 'parses with multiple char separator' do
|
|
47
|
-
line = '<=><=>A<=>B<=>C'
|
|
48
|
-
options.merge!({col_sep: "<=>"})
|
|
49
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
|
50
|
-
expect(array).to eq ["", "", "A", "B", "C"]
|
|
51
|
-
expect(array_size).to eq 5
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
it 'parses trailing multiple char separator' do
|
|
55
|
-
line = '<=><=>A<=>B<=>C<=><=>'
|
|
56
|
-
options.merge!({col_sep: "<=>"})
|
|
57
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
|
58
|
-
expect(array).to eq ["", "", "A", "B", "C", "", ""]
|
|
59
|
-
expect(array_size).to eq 7
|
|
60
|
-
end
|
|
61
|
-
end
|
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
describe 'old CSV library parsing tests' do
|
|
4
|
-
let(:options) { {quote_char: '"', col_sep: ","} }
|
|
5
|
-
|
|
6
|
-
[ ["\t", ["\t"]],
|
|
7
|
-
["foo,\"\"\"\"\"\",baz", ["foo", "\"\"", "baz"]],
|
|
8
|
-
["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
|
|
9
|
-
["\"\"\"\n\",\"\"\"\n\"", ["\"\n", "\"\n"]],
|
|
10
|
-
["foo,\"\r\n\",baz", ["foo", "\r\n", "baz"]],
|
|
11
|
-
["\"\"", [""]],
|
|
12
|
-
["foo,\"\"\"\",baz", ["foo", "\"", "baz"]],
|
|
13
|
-
["foo,\"\r.\n\",baz", ["foo", "\r.\n", "baz"]],
|
|
14
|
-
["foo,\"\r\",baz", ["foo", "\r", "baz"]],
|
|
15
|
-
["foo,\"\",baz", ["foo", "", "baz"]],
|
|
16
|
-
["\",\"", [","]],
|
|
17
|
-
["foo", ["foo"]],
|
|
18
|
-
[",,", ['', '', '']],
|
|
19
|
-
[",", ['', '']],
|
|
20
|
-
["foo,\"\n\",baz", ["foo", "\n", "baz"]],
|
|
21
|
-
["foo,,baz", ["foo", '', "baz"]],
|
|
22
|
-
["\"\"\"\r\",\"\"\"\r\"", ["\"\r", "\"\r"]],
|
|
23
|
-
["\",\",\",\"", [",", ","]],
|
|
24
|
-
["foo,bar,", ["foo", "bar", '']],
|
|
25
|
-
[",foo,bar", ['', "foo", "bar"]],
|
|
26
|
-
["foo,bar", ["foo", "bar"]],
|
|
27
|
-
[";", [";"]],
|
|
28
|
-
["\t,\t", ["\t", "\t"]],
|
|
29
|
-
["foo,\"\r\n\r\",baz", ["foo", "\r\n\r", "baz"]],
|
|
30
|
-
["foo,\"\r\n\n\",baz", ["foo", "\r\n\n", "baz"]],
|
|
31
|
-
["foo,\"foo,bar\",baz", ["foo", "foo,bar", "baz"]],
|
|
32
|
-
[";,;", [";", ";"]]
|
|
33
|
-
].each do |line, result|
|
|
34
|
-
it "parses #{line}" do
|
|
35
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
|
36
|
-
expect(array).to eq result
|
|
37
|
-
end
|
|
38
|
-
end
|
|
39
|
-
|
|
40
|
-
[ ["foo,\"\"\"\"\"\",baz", ["foo", "\"\"", "baz"]],
|
|
41
|
-
["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
|
|
42
|
-
["foo,\"\r\n\",baz", ["foo", "\r\n", "baz"]],
|
|
43
|
-
["\"\"", [""]],
|
|
44
|
-
["foo,\"\"\"\",baz", ["foo", "\"", "baz"]],
|
|
45
|
-
["foo,\"\r.\n\",baz", ["foo", "\r.\n", "baz"]],
|
|
46
|
-
["foo,\"\r\",baz", ["foo", "\r", "baz"]],
|
|
47
|
-
["foo,\"\",baz", ["foo", "", "baz"]],
|
|
48
|
-
["foo", ["foo"]],
|
|
49
|
-
[",,", ['', '', '']],
|
|
50
|
-
[",", ['', '']],
|
|
51
|
-
["foo,\"\n\",baz", ["foo", "\n", "baz"]],
|
|
52
|
-
["foo,,baz", ["foo", '', "baz"]],
|
|
53
|
-
["foo,bar", ["foo", "bar"]],
|
|
54
|
-
["foo,\"\r\n\n\",baz", ["foo", "\r\n\n", "baz"]],
|
|
55
|
-
["foo,\"foo,bar\",baz", ["foo", "foo,bar", "baz"]]
|
|
56
|
-
].each do |line, result|
|
|
57
|
-
it "parses #{line}" do
|
|
58
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
|
59
|
-
expect(array).to eq result
|
|
60
|
-
end
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
it 'mixed quotes' do
|
|
64
|
-
line = %Q{Ten Thousand,10000, 2710 ,,"10,000","It's ""10 Grand"", baby",10K}
|
|
65
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
|
66
|
-
expect(array).to eq ["Ten Thousand", "10000", " 2710 ", "", "10,000", "It's \"10 Grand\", baby", "10K"]
|
|
67
|
-
end
|
|
68
|
-
|
|
69
|
-
it 'single quotes in fields' do
|
|
70
|
-
line = 'Indoor Chrome,49.2"" L x 49.2"" W x 20.5"" H,Chrome,"Crystal,Metal,Wood",23.12'
|
|
71
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
|
72
|
-
expect(array).to eq ['Indoor Chrome', '49.2" L x 49.2" W x 20.5" H', 'Chrome', 'Crystal,Metal,Wood', '23.12']
|
|
73
|
-
end
|
|
74
|
-
end
|