smarter_csv 1.6.1 → 1.7.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +133 -0
- data/CHANGELOG.md +22 -1
- data/CONTRIBUTORS.md +3 -0
- data/Gemfile +7 -4
- data/README.md +8 -6
- data/Rakefile +15 -13
- data/ext/smarter_csv/extconf.rb +14 -0
- data/ext/smarter_csv/smarter_csv.c +86 -0
- data/lib/extensions/hash.rb +4 -2
- data/lib/smarter_csv/version.rb +3 -1
- data/lib/smarter_csv.rb +519 -10
- data/smarter_csv.gemspec +22 -7
- metadata +54 -176
- data/.gitignore +0 -10
- data/.rspec +0 -2
- data/.travis.yml +0 -27
- data/lib/smarter_csv/smarter_csv.rb +0 -461
- data/spec/fixtures/additional_separator.csv +0 -6
- data/spec/fixtures/basic.csv +0 -8
- data/spec/fixtures/binary.csv +0 -1
- data/spec/fixtures/carriage_returns_n.csv +0 -18
- data/spec/fixtures/carriage_returns_quoted.csv +0 -3
- data/spec/fixtures/carriage_returns_r.csv +0 -1
- data/spec/fixtures/carriage_returns_rn.csv +0 -18
- data/spec/fixtures/chunk_cornercase.csv +0 -10
- data/spec/fixtures/duplicate_headers.csv +0 -3
- data/spec/fixtures/empty.csv +0 -5
- data/spec/fixtures/empty_columns_1.csv +0 -2
- data/spec/fixtures/empty_columns_2.csv +0 -2
- data/spec/fixtures/hard_sample.csv +0 -2
- data/spec/fixtures/ignore_comments.csv +0 -11
- data/spec/fixtures/ignore_comments2.csv +0 -3
- data/spec/fixtures/key_mapping.csv +0 -2
- data/spec/fixtures/line_endings_n.csv +0 -4
- data/spec/fixtures/line_endings_r.csv +0 -1
- data/spec/fixtures/line_endings_rn.csv +0 -4
- data/spec/fixtures/lots_of_columns.csv +0 -2
- data/spec/fixtures/malformed.csv +0 -3
- data/spec/fixtures/malformed_header.csv +0 -3
- data/spec/fixtures/money.csv +0 -3
- data/spec/fixtures/no_header.csv +0 -7
- data/spec/fixtures/numeric.csv +0 -5
- data/spec/fixtures/pets.csv +0 -5
- data/spec/fixtures/problematic.csv +0 -8
- data/spec/fixtures/quote_char.csv +0 -9
- data/spec/fixtures/quoted.csv +0 -5
- data/spec/fixtures/quoted2.csv +0 -4
- data/spec/fixtures/separator_colon.csv +0 -4
- data/spec/fixtures/separator_comma.csv +0 -4
- data/spec/fixtures/separator_pipe.csv +0 -4
- data/spec/fixtures/separator_semi.csv +0 -4
- data/spec/fixtures/separator_tab.csv +0 -4
- data/spec/fixtures/skip_lines.csv +0 -8
- data/spec/fixtures/trading.csv +0 -3
- data/spec/fixtures/user_import.csv +0 -3
- data/spec/fixtures/valid_unicode.csv +0 -5
- data/spec/fixtures/with_dashes.csv +0 -8
- data/spec/fixtures/with_dates.csv +0 -4
- data/spec/smarter_csv/additional_separator_spec.rb +0 -45
- data/spec/smarter_csv/binary_file2_spec.rb +0 -24
- data/spec/smarter_csv/binary_file_spec.rb +0 -22
- data/spec/smarter_csv/blank_spec.rb +0 -55
- data/spec/smarter_csv/carriage_return_spec.rb +0 -190
- data/spec/smarter_csv/chunked_reading_spec.rb +0 -14
- data/spec/smarter_csv/close_file_spec.rb +0 -15
- data/spec/smarter_csv/column_separator_spec.rb +0 -95
- data/spec/smarter_csv/convert_values_to_numeric_spec.rb +0 -48
- data/spec/smarter_csv/duplicate_headers_spec.rb +0 -76
- data/spec/smarter_csv/empty_columns_spec.rb +0 -74
- data/spec/smarter_csv/extenstions_spec.rb +0 -17
- data/spec/smarter_csv/hard_sample_spec.rb +0 -24
- data/spec/smarter_csv/header_transformation_spec.rb +0 -21
- data/spec/smarter_csv/ignore_comments_spec.rb +0 -45
- data/spec/smarter_csv/invalid_headers_spec.rb +0 -38
- data/spec/smarter_csv/keep_headers_spec.rb +0 -24
- data/spec/smarter_csv/key_mapping_spec.rb +0 -56
- data/spec/smarter_csv/line_ending_spec.rb +0 -43
- data/spec/smarter_csv/load_basic_spec.rb +0 -20
- data/spec/smarter_csv/malformed_spec.rb +0 -25
- data/spec/smarter_csv/no_header_spec.rb +0 -29
- data/spec/smarter_csv/not_downcase_header_spec.rb +0 -24
- data/spec/smarter_csv/parse/column_separator_spec.rb +0 -61
- data/spec/smarter_csv/parse/old_csv_library_spec.rb +0 -74
- data/spec/smarter_csv/parse/rfc4180_and_more_spec.rb +0 -170
- data/spec/smarter_csv/problematic.rb +0 -34
- data/spec/smarter_csv/quoted_spec.rb +0 -52
- data/spec/smarter_csv/remove_empty_values_spec.rb +0 -13
- data/spec/smarter_csv/remove_keys_from_hashes_spec.rb +0 -25
- data/spec/smarter_csv/remove_not_mapped_keys_spec.rb +0 -35
- data/spec/smarter_csv/remove_values_matching_spec.rb +0 -26
- data/spec/smarter_csv/remove_zero_values_spec.rb +0 -25
- data/spec/smarter_csv/skip_lines_spec.rb +0 -29
- data/spec/smarter_csv/strings_as_keys_spec.rb +0 -24
- data/spec/smarter_csv/strip_chars_from_headers_spec.rb +0 -24
- data/spec/smarter_csv/trading_spec.rb +0 -25
- data/spec/smarter_csv/valid_unicode_spec.rb +0 -94
- data/spec/smarter_csv/value_converters_spec.rb +0 -52
- data/spec/spec/spec_helper.rb +0 -17
- data/spec/spec.opts +0 -2
- data/spec/spec_helper.rb +0 -21
@@ -1,76 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'duplicate headers' do
|
6
|
-
describe 'without special handling / default behavior' do
|
7
|
-
it 'raises error on duplicate headers' do
|
8
|
-
expect {
|
9
|
-
SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", {})
|
10
|
-
}.to raise_exception(SmarterCSV::DuplicateHeaders)
|
11
|
-
end
|
12
|
-
|
13
|
-
it 'raises error on duplicate given headers' do
|
14
|
-
expect {
|
15
|
-
options = {:user_provided_headers => [:a,:b,:c,:d,:a]}
|
16
|
-
SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
17
|
-
}.to raise_exception(SmarterCSV::DuplicateHeaders)
|
18
|
-
end
|
19
|
-
|
20
|
-
it 'does not raise error on missing mapped headers and includes missing headers in message' do
|
21
|
-
# the mapping is right, but the underlying csv file is bad
|
22
|
-
options = {:key_mapping => {:email => :a, :firstname => :b, :lastname => :c, :manager_email => :d, :age => :e} }
|
23
|
-
expect {
|
24
|
-
SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
25
|
-
}.not_to raise_exception(SmarterCSV::KeyMappingError)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
describe 'with special handling' do
|
30
|
-
context 'with given suffix' do
|
31
|
-
let(:options) { {duplicate_header_suffix: '_'} }
|
32
|
-
|
33
|
-
it 'reads whole file' do
|
34
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
35
|
-
expect(data.size).to eq 2
|
36
|
-
end
|
37
|
-
|
38
|
-
it 'generates the correct keys' do
|
39
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
40
|
-
expect(data.first.keys).to eq [:email, :firstname, :lastname, :email_2, :age]
|
41
|
-
end
|
42
|
-
|
43
|
-
it 'enumerates when duplicate headers are given' do
|
44
|
-
options.merge!({:user_provided_headers => [:a,:b,:c,:a,:a]})
|
45
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
46
|
-
expect(data.first.keys).to eq [:a, :b, :c, :a_2, :a_3]
|
47
|
-
end
|
48
|
-
|
49
|
-
it 'can remap duplicated headers' do
|
50
|
-
options.merge!({:key_mapping => {:email => :a, :firstname => :b, :lastname => :c, :email_2 => :d, :age => :e}})
|
51
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
52
|
-
expect(data.first).to eq({a: 'tom@bla.com', b: 'Tom', c: 'Sawyer', d: 'mike@bla.com', e: 34})
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
context 'with empty suffix' do
|
57
|
-
let(:options) { {duplicate_header_suffix: ''} }
|
58
|
-
|
59
|
-
it 'reads whole file' do
|
60
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
61
|
-
expect(data.size).to eq 2
|
62
|
-
end
|
63
|
-
|
64
|
-
it 'generates the correct keys' do
|
65
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
66
|
-
expect(data.first.keys).to eq [:email, :firstname, :lastname, :email2, :age]
|
67
|
-
end
|
68
|
-
|
69
|
-
it 'enumerates when duplicate headers are given' do
|
70
|
-
options.merge!({:user_provided_headers => [:a,:b,:c,:a,:a]})
|
71
|
-
data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
|
72
|
-
expect(data.first.keys).to eq [:a, :b, :c, :a2, :a3]
|
73
|
-
end
|
74
|
-
end
|
75
|
-
end
|
76
|
-
end
|
@@ -1,74 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'can handle empty columns' do
|
6
|
-
|
7
|
-
describe 'default behavior' do
|
8
|
-
it 'has empty columns at end' do
|
9
|
-
data = SmarterCSV.process("#{fixture_path}/empty_columns_1.csv")
|
10
|
-
data.size.should eq 1
|
11
|
-
item = data.first
|
12
|
-
item[:id].should == 123
|
13
|
-
item[:col1].should == nil
|
14
|
-
item[:col2].should == nil
|
15
|
-
item[:col3].should == nil
|
16
|
-
end
|
17
|
-
|
18
|
-
it 'has empty columns in the middle' do
|
19
|
-
data = SmarterCSV.process("#{fixture_path}/empty_columns_2.csv")
|
20
|
-
data.size.should eq 1
|
21
|
-
item = data.first
|
22
|
-
item[:id].should == 123
|
23
|
-
item[:col1].should == nil
|
24
|
-
item[:col2].should == nil
|
25
|
-
item[:col3].should == 1
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
describe 'with remove_empty_values: true' do
|
30
|
-
options = {remove_empty_values: true}
|
31
|
-
it 'has empty columns at end' do
|
32
|
-
data = SmarterCSV.process("#{fixture_path}/empty_columns_1.csv", options)
|
33
|
-
data.size.should eq 1
|
34
|
-
item = data.first
|
35
|
-
item[:id].should == 123
|
36
|
-
item[:col1].should == nil
|
37
|
-
item[:col2].should == nil
|
38
|
-
item[:col3].should == nil
|
39
|
-
end
|
40
|
-
|
41
|
-
it 'has empty columns in the middle' do
|
42
|
-
data = SmarterCSV.process("#{fixture_path}/empty_columns_2.csv", options)
|
43
|
-
data.size.should eq 1
|
44
|
-
item = data.first
|
45
|
-
item[:id].should == 123
|
46
|
-
item[:col1].should == nil
|
47
|
-
item[:col2].should == nil
|
48
|
-
item[:col3].should == 1
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
describe 'with remove_empty_values: false' do
|
53
|
-
options = {remove_empty_values: false}
|
54
|
-
it 'has empty columns at end' do
|
55
|
-
data = SmarterCSV.process("#{fixture_path}/empty_columns_1.csv", options)
|
56
|
-
data.size.should eq 1
|
57
|
-
item = data.first
|
58
|
-
item[:id].should == 123
|
59
|
-
item[:col1].should == ''
|
60
|
-
item[:col2].should == ''
|
61
|
-
item[:col3].should == ''
|
62
|
-
end
|
63
|
-
|
64
|
-
it 'has empty columns in the middle' do
|
65
|
-
data = SmarterCSV.process("#{fixture_path}/empty_columns_2.csv", options)
|
66
|
-
data.size.should eq 1
|
67
|
-
item = data.first
|
68
|
-
item[:id].should == 123
|
69
|
-
item[:col1].should == ''
|
70
|
-
item[:col2].should == ''
|
71
|
-
item[:col3].should == 1
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
@@ -1,17 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe "Hash.zip" do
|
4
|
-
it "constructs a new Hash from two Arrays" do
|
5
|
-
Hash.zip(["a", "b"], [1, 2]).should == { "a" => 1, "b" => 2 }
|
6
|
-
end
|
7
|
-
|
8
|
-
it "constructs an empty Hash if given no keys" do
|
9
|
-
Hash.zip([], []).should == {}
|
10
|
-
Hash.zip([], [1]).should == {}
|
11
|
-
end
|
12
|
-
|
13
|
-
it "uses nil values if there are more keys than values" do
|
14
|
-
Hash.zip(["a"], []).should == { "a" => nil }
|
15
|
-
Hash.zip(["a", "b"], [1]).should == { "a" => 1, "b" => nil }
|
16
|
-
end
|
17
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'can handle the difficult CSV file' do
|
6
|
-
|
7
|
-
it 'loads the data with default values' do
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/hard_sample.csv")
|
9
|
-
data.size.should eq 1
|
10
|
-
item = data.first
|
11
|
-
item.keys.count.should == 48
|
12
|
-
item[:name].should == '#MR1220817'
|
13
|
-
item[:shipping_method].should == 'Livraison Standard GRATUITE, 2-5 jours avec suivi'
|
14
|
-
item[:lineitem_name].should == 'Cire Épilation Nacrée'
|
15
|
-
item[:phone].should == 3366012111111
|
16
|
-
end
|
17
|
-
|
18
|
-
# the main problem is the data line starting with a # character, but not being a comment
|
19
|
-
it 'fails to load the CSV file with incorrectly set comment_regexp' do
|
20
|
-
options = {comment_regexp: /\A#/ }
|
21
|
-
data = SmarterCSV.process("#{fixture_path}/hard_sample.csv", options)
|
22
|
-
data.size.should eq 0
|
23
|
-
end
|
24
|
-
end
|
@@ -1,21 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'loads_file_with_dashes_in_header_fields as strings' do
|
7
|
-
options = {:strings_as_keys => true}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/with_dashes.csv", options)
|
9
|
-
data.flatten.size.should == 5
|
10
|
-
data[0]['first_name'].should eq 'Dan'
|
11
|
-
data[0]['last_name'].should eq 'McAllister'
|
12
|
-
end
|
13
|
-
|
14
|
-
it 'loads_file_with_dashes_in_header_fields as symbols' do
|
15
|
-
options = {:strings_as_keys => false}
|
16
|
-
data = SmarterCSV.process("#{fixture_path}/with_dashes.csv", options)
|
17
|
-
data.flatten.size.should == 5
|
18
|
-
data[0][:first_name].should eq 'Dan'
|
19
|
-
data[0][:last_name].should eq 'McAllister'
|
20
|
-
end
|
21
|
-
end
|
@@ -1,45 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'by default does not ignore comments in CSV files' do
|
7
|
-
options = {}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/ignore_comments.csv", options)
|
9
|
-
|
10
|
-
data.size.should eq 8
|
11
|
-
|
12
|
-
# all the keys should be symbols
|
13
|
-
data.each{|item| item.keys.each{|x| x.is_a?(Symbol).should be_truthy}}
|
14
|
-
data.each do |h|
|
15
|
-
h.keys.each do |key|
|
16
|
-
[:"not_a_comment#first_name", :last_name, :dogs, :cats, :birds, :fish].should include( key )
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
it 'ignore comments in CSV files using comment_regexp' do
|
22
|
-
options = {comment_regexp: /\A#/}
|
23
|
-
data = SmarterCSV.process("#{fixture_path}/ignore_comments.csv", options)
|
24
|
-
|
25
|
-
data.size.should eq 5
|
26
|
-
|
27
|
-
# all the keys should be symbols
|
28
|
-
data.each{|item| item.keys.each{|x| x.is_a?(Symbol).should be_truthy}}
|
29
|
-
data.each do |h|
|
30
|
-
h.keys.each do |key|
|
31
|
-
[:"not_a_comment#first_name", :last_name, :dogs, :cats, :birds, :fish].should include( key )
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
it 'ignore comments in CSV files with CRLF' do
|
37
|
-
options = {row_sep: "\r\n"}
|
38
|
-
data = SmarterCSV.process("#{fixture_path}/ignore_comments2.csv", options)
|
39
|
-
|
40
|
-
# all the keys should be symbols
|
41
|
-
data.size.should eq 1
|
42
|
-
data.first[:h1].should eq 'a'
|
43
|
-
data.first[:h2].should eq "b\r\n#c"
|
44
|
-
end
|
45
|
-
end
|
@@ -1,38 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'test exceptions for invalid headers' do
|
6
|
-
it 'does not raise an error if no required headers are given' do
|
7
|
-
options = {:required_headers => nil} # order does not matter
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/user_import.csv", options)
|
9
|
-
data.size.should eq 2
|
10
|
-
end
|
11
|
-
|
12
|
-
it 'does not raise an error if no required headers are given' do
|
13
|
-
options = {:required_headers => []} # order does not matter
|
14
|
-
data = SmarterCSV.process("#{fixture_path}/user_import.csv", options)
|
15
|
-
data.size.should eq 2
|
16
|
-
end
|
17
|
-
|
18
|
-
it 'does not raise an error if the required headers are present' do
|
19
|
-
options = {:required_headers => [:lastname,:email,:firstname,:manager_email]} # order does not matter
|
20
|
-
data = SmarterCSV.process("#{fixture_path}/user_import.csv", options)
|
21
|
-
data.size.should eq 2
|
22
|
-
end
|
23
|
-
|
24
|
-
it 'raises an error if a required header is missing' do
|
25
|
-
expect {
|
26
|
-
options = {:required_headers => [:lastname,:email,:employee_id,:firstname,:manager_email]} # order does not matter
|
27
|
-
SmarterCSV.process("#{fixture_path}/user_import.csv", options)
|
28
|
-
}.to raise_exception(SmarterCSV::MissingHeaders)
|
29
|
-
end
|
30
|
-
|
31
|
-
it 'does not raise error on missing mapped headers and includes missing headers in message' do
|
32
|
-
# :age does not exist in the CSV header
|
33
|
-
options = {:key_mapping => {:email => :a, :firstname => :b, :lastname => :c, :manager_email => :d, :age => :e} }
|
34
|
-
expect {
|
35
|
-
SmarterCSV.process("#{fixture_path}/user_import.csv", options)
|
36
|
-
}.not_to raise_exception(SmarterCSV::KeyMappingError)
|
37
|
-
end
|
38
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'not_downcase_headers' do
|
7
|
-
options = {:keep_original_headers => true}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
9
|
-
data.size.should == 5
|
10
|
-
# all the keys should be string
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == String}}
|
12
|
-
|
13
|
-
data.each do |item|
|
14
|
-
item.keys.each do |key|
|
15
|
-
['First Name','Last Name','Dogs','Cats','Birds','Fish'].should include( key )
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
data.each do |h|
|
20
|
-
h.size.should <= 6
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
@@ -1,56 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'remove_values_matching' do
|
7
|
-
options = {:remove_zero_values => true, :key_mapping => {:first_name => :vorname, :last_name => :nachname} }
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
9
|
-
data.size.should == 5
|
10
|
-
# all the keys should be symbols
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
12
|
-
|
13
|
-
data.each do |hash|
|
14
|
-
hash.keys.each do |key|
|
15
|
-
[:vorname, :nachname, :dogs, :cats, :birds, :fish].should include( key )
|
16
|
-
end
|
17
|
-
hash.values.should_not include( 0 )
|
18
|
-
end
|
19
|
-
|
20
|
-
data.each do |h|
|
21
|
-
h.size.should <= 6
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
describe 'when keep_original_headers' do
|
26
|
-
it 'without key mapping' do
|
27
|
-
options = {:keep_original_headers => true}
|
28
|
-
data = SmarterCSV.process("#{fixture_path}/key_mapping.csv", options)
|
29
|
-
data.size.should == 1
|
30
|
-
data.first.keys.should == ['THIS', 'THAT', 'other']
|
31
|
-
end
|
32
|
-
|
33
|
-
it 'sets key_mapping to a symbol' do
|
34
|
-
options = {:keep_original_headers => true, :key_mapping => {'other' => :other}}
|
35
|
-
data = SmarterCSV.process("#{fixture_path}/key_mapping.csv", options)
|
36
|
-
data.size.should == 1
|
37
|
-
data.first.keys.should == ['THIS', 'THAT', :other]
|
38
|
-
end
|
39
|
-
|
40
|
-
# this previously would set the key to a symbol :OTHER, which was a bug!
|
41
|
-
it 'sets key_mapping to a string' do
|
42
|
-
options = {:keep_original_headers => true, :key_mapping => {'other' => 'OTHER'}}
|
43
|
-
data = SmarterCSV.process("#{fixture_path}/key_mapping.csv", options)
|
44
|
-
data.size.should == 1
|
45
|
-
data.first.keys.should == ['THIS', 'THAT', 'OTHER']
|
46
|
-
end
|
47
|
-
|
48
|
-
# users now have to explicitly set this to a symbol, or change the expected keys to be strings.
|
49
|
-
it 'sets key_mapping to a symbol' do
|
50
|
-
options = {:keep_original_headers => true, :key_mapping => {'other' => :OTHER}}
|
51
|
-
data = SmarterCSV.process("#{fixture_path}/key_mapping.csv", options)
|
52
|
-
data.size.should == 1
|
53
|
-
data.first.keys.should == ['THIS', 'THAT', :OTHER]
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
@@ -1,43 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'process files with line endings explicitly pre-specified' do
|
6
|
-
it 'reads file with \n line endings' do
|
7
|
-
options = {:row_sep => "\n"}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_n.csv", options)
|
9
|
-
data.size.should == 3
|
10
|
-
end
|
11
|
-
|
12
|
-
it 'reads file with \r line endings' do
|
13
|
-
options = {:row_sep => "\r"}
|
14
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_r.csv", options)
|
15
|
-
data.size.should == 3
|
16
|
-
end
|
17
|
-
|
18
|
-
it 'reads file with \r\n line endings' do
|
19
|
-
options = {:row_sep => "\r\n"}
|
20
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_rn.csv", options)
|
21
|
-
data.size.should == 3
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
describe 'process files with line endings in automatic mode' do
|
26
|
-
it 'reads file with \n line endings' do
|
27
|
-
options = {:row_sep => :auto}
|
28
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_n.csv", options)
|
29
|
-
data.size.should == 3
|
30
|
-
end
|
31
|
-
|
32
|
-
it 'reads file with \r line endings' do
|
33
|
-
options = {:row_sep => :auto}
|
34
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_r.csv", options)
|
35
|
-
data.size.should == 3
|
36
|
-
end
|
37
|
-
|
38
|
-
it 'reads file with \r\n line endings' do
|
39
|
-
options = {:row_sep => :auto}
|
40
|
-
data = SmarterCSV.process("#{fixture_path}/line_endings_rn.csv", options)
|
41
|
-
data.size.should == 3
|
42
|
-
end
|
43
|
-
end
|
@@ -1,20 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'loads_basic_csv_file' do
|
7
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv")
|
8
|
-
data.size.should == 5
|
9
|
-
|
10
|
-
# all the keys should be symbols
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
12
|
-
data.each do |h|
|
13
|
-
h.keys.each do |key|
|
14
|
-
[:first_name, :last_name, :dogs, :cats, :birds, :fish].should include( key )
|
15
|
-
end
|
16
|
-
h.size.should <= 6
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
end
|
@@ -1,25 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
# according to RFC-4180 quotes inside of "words" shouldbe doubled, but our parser is robust against that.
|
6
|
-
describe 'malformed CSV quotes' do
|
7
|
-
context "malformed quotes in header" do
|
8
|
-
let(:csv_path) { "#{fixture_path}/malformed_header.csv" }
|
9
|
-
it 'should be resilient against single quotes' do
|
10
|
-
data = SmarterCSV.process(csv_path)
|
11
|
-
expect(data[0]).to eq({:name=>"Arnold Schwarzenegger", :dobdob=>"1947-07-30"})
|
12
|
-
expect(data[1]).to eq({:name=>"Jeff Bridges", :dobdob=>"1949-12-04"})
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
context "malformed quotes in content" do
|
17
|
-
let(:csv_path) { "#{fixture_path}/malformed.csv" }
|
18
|
-
|
19
|
-
it 'should be resilient against single quotes' do
|
20
|
-
data = SmarterCSV.process(csv_path)
|
21
|
-
expect(data[0]).to eq({:name=>"Arnold Schwarzenegger", :dob=>"1947-07-30"})
|
22
|
-
expect(data[1]).to eq({:name=>"Jeff \"the dude\" Bridges", :dob=>"1949-12-04"})
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
@@ -1,29 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'no header in file' do
|
6
|
-
let(:headers) { [:a,:b,:c,:d,:e,:f] }
|
7
|
-
let(:options) { {:headers_in_file => false, :user_provided_headers => headers} }
|
8
|
-
subject(:data) { SmarterCSV.process("#{fixture_path}/no_header.csv", options) }
|
9
|
-
|
10
|
-
it 'load the correct number of records' do
|
11
|
-
data.size.should == 5
|
12
|
-
end
|
13
|
-
|
14
|
-
it 'uses given symbols for all records' do
|
15
|
-
data.each do |item|
|
16
|
-
item.keys.each do |key|
|
17
|
-
[:a,:b,:c,:d,:e,:f].should include( key )
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
it 'loads the correct data' do
|
23
|
-
data[0].should == {a: "Dan", b: "McAllister", c: 2, d: 0}
|
24
|
-
data[1].should == {a: "Lucy", b: "Laweless", d: 5, e: 0}
|
25
|
-
data[2].should == {a: "Miles", b: "O'Brian", c: 0, d: 0, e: 0, f: 21}
|
26
|
-
data[3].should == {a: "Nancy", b: "Homes", c: 2, d: 0, e: 1}
|
27
|
-
data[4].should == {a: "Hernán", b: "Curaçon", c: 3, d: 0, e: 0}
|
28
|
-
end
|
29
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'not_downcase_headers' do
|
7
|
-
options = {:downcase_header => false}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
9
|
-
data.size.should == 5
|
10
|
-
# all the keys should be symbols
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
12
|
-
|
13
|
-
data.each do |item|
|
14
|
-
item.keys.each do |key|
|
15
|
-
[:First_Name, :Last_Name, :Dogs, :Cats, :Birds, :Fish].should include( key )
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
data.each do |h|
|
20
|
-
h.size.should <= 6
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
@@ -1,61 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe 'parse with col_sep' do
|
4
|
-
let(:options) { {quote_char: '"'} }
|
5
|
-
|
6
|
-
it 'parses with comma' do
|
7
|
-
line = "a,b,,d"
|
8
|
-
options.merge!({col_sep: ","})
|
9
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
10
|
-
expect(array).to eq ['a', 'b', '', 'd']
|
11
|
-
expect(array_size).to eq 4
|
12
|
-
end
|
13
|
-
|
14
|
-
it 'parses trailing commas' do
|
15
|
-
line = "a,b,c,,"
|
16
|
-
options.merge!({col_sep: ","})
|
17
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
18
|
-
expect(array).to eq ['a', 'b', 'c', '', '']
|
19
|
-
expect(array_size).to eq 5
|
20
|
-
end
|
21
|
-
|
22
|
-
it 'parses with space' do
|
23
|
-
line = "a b d"
|
24
|
-
options.merge!({col_sep: " "})
|
25
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
26
|
-
expect(array).to eq ['a', 'b', '', 'd']
|
27
|
-
expect(array_size).to eq 4
|
28
|
-
end
|
29
|
-
|
30
|
-
it 'parses with tab' do
|
31
|
-
line = "a\tb\t\td"
|
32
|
-
options.merge!({col_sep: "\t"})
|
33
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
34
|
-
expect(array).to eq ['a', 'b', '', 'd']
|
35
|
-
expect(array_size).to eq 4
|
36
|
-
end
|
37
|
-
|
38
|
-
it 'parses with multiple space separator' do
|
39
|
-
line = "a b d"
|
40
|
-
options.merge!({col_sep: " "})
|
41
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
42
|
-
expect(array).to eq ['a b', '', 'd']
|
43
|
-
expect(array_size).to eq 3
|
44
|
-
end
|
45
|
-
|
46
|
-
it 'parses with multiple char separator' do
|
47
|
-
line = '<=><=>A<=>B<=>C'
|
48
|
-
options.merge!({col_sep: "<=>"})
|
49
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
50
|
-
expect(array).to eq ["", "", "A", "B", "C"]
|
51
|
-
expect(array_size).to eq 5
|
52
|
-
end
|
53
|
-
|
54
|
-
it 'parses trailing multiple char separator' do
|
55
|
-
line = '<=><=>A<=>B<=>C<=><=>'
|
56
|
-
options.merge!({col_sep: "<=>"})
|
57
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
58
|
-
expect(array).to eq ["", "", "A", "B", "C", "", ""]
|
59
|
-
expect(array_size).to eq 7
|
60
|
-
end
|
61
|
-
end
|
@@ -1,74 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe 'old CSV library parsing tests' do
|
4
|
-
let(:options) { {quote_char: '"', col_sep: ","} }
|
5
|
-
|
6
|
-
[ ["\t", ["\t"]],
|
7
|
-
["foo,\"\"\"\"\"\",baz", ["foo", "\"\"", "baz"]],
|
8
|
-
["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
|
9
|
-
["\"\"\"\n\",\"\"\"\n\"", ["\"\n", "\"\n"]],
|
10
|
-
["foo,\"\r\n\",baz", ["foo", "\r\n", "baz"]],
|
11
|
-
["\"\"", [""]],
|
12
|
-
["foo,\"\"\"\",baz", ["foo", "\"", "baz"]],
|
13
|
-
["foo,\"\r.\n\",baz", ["foo", "\r.\n", "baz"]],
|
14
|
-
["foo,\"\r\",baz", ["foo", "\r", "baz"]],
|
15
|
-
["foo,\"\",baz", ["foo", "", "baz"]],
|
16
|
-
["\",\"", [","]],
|
17
|
-
["foo", ["foo"]],
|
18
|
-
[",,", ['', '', '']],
|
19
|
-
[",", ['', '']],
|
20
|
-
["foo,\"\n\",baz", ["foo", "\n", "baz"]],
|
21
|
-
["foo,,baz", ["foo", '', "baz"]],
|
22
|
-
["\"\"\"\r\",\"\"\"\r\"", ["\"\r", "\"\r"]],
|
23
|
-
["\",\",\",\"", [",", ","]],
|
24
|
-
["foo,bar,", ["foo", "bar", '']],
|
25
|
-
[",foo,bar", ['', "foo", "bar"]],
|
26
|
-
["foo,bar", ["foo", "bar"]],
|
27
|
-
[";", [";"]],
|
28
|
-
["\t,\t", ["\t", "\t"]],
|
29
|
-
["foo,\"\r\n\r\",baz", ["foo", "\r\n\r", "baz"]],
|
30
|
-
["foo,\"\r\n\n\",baz", ["foo", "\r\n\n", "baz"]],
|
31
|
-
["foo,\"foo,bar\",baz", ["foo", "foo,bar", "baz"]],
|
32
|
-
[";,;", [";", ";"]]
|
33
|
-
].each do |line, result|
|
34
|
-
it "parses #{line}" do
|
35
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
36
|
-
expect(array).to eq result
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
[ ["foo,\"\"\"\"\"\",baz", ["foo", "\"\"", "baz"]],
|
41
|
-
["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
|
42
|
-
["foo,\"\r\n\",baz", ["foo", "\r\n", "baz"]],
|
43
|
-
["\"\"", [""]],
|
44
|
-
["foo,\"\"\"\",baz", ["foo", "\"", "baz"]],
|
45
|
-
["foo,\"\r.\n\",baz", ["foo", "\r.\n", "baz"]],
|
46
|
-
["foo,\"\r\",baz", ["foo", "\r", "baz"]],
|
47
|
-
["foo,\"\",baz", ["foo", "", "baz"]],
|
48
|
-
["foo", ["foo"]],
|
49
|
-
[",,", ['', '', '']],
|
50
|
-
[",", ['', '']],
|
51
|
-
["foo,\"\n\",baz", ["foo", "\n", "baz"]],
|
52
|
-
["foo,,baz", ["foo", '', "baz"]],
|
53
|
-
["foo,bar", ["foo", "bar"]],
|
54
|
-
["foo,\"\r\n\n\",baz", ["foo", "\r\n\n", "baz"]],
|
55
|
-
["foo,\"foo,bar\",baz", ["foo", "foo,bar", "baz"]]
|
56
|
-
].each do |line, result|
|
57
|
-
it "parses #{line}" do
|
58
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
59
|
-
expect(array).to eq result
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
it 'mixed quotes' do
|
64
|
-
line = %Q{Ten Thousand,10000, 2710 ,,"10,000","It's ""10 Grand"", baby",10K}
|
65
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
66
|
-
expect(array).to eq ["Ten Thousand", "10000", " 2710 ", "", "10,000", "It's \"10 Grand\", baby", "10K"]
|
67
|
-
end
|
68
|
-
|
69
|
-
it 'single quotes in fields' do
|
70
|
-
line = 'Indoor Chrome,49.2"" L x 49.2"" W x 20.5"" H,Chrome,"Crystal,Metal,Wood",23.12'
|
71
|
-
array, array_size = SmarterCSV.send(:parse, line, options)
|
72
|
-
expect(array).to eq ['Indoor Chrome', '49.2" L x 49.2" W x 20.5" H', 'Chrome', 'Crystal,Metal,Wood', '23.12']
|
73
|
-
end
|
74
|
-
end
|