smarter_csv 1.6.1 → 1.7.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (101) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +133 -0
  3. data/CHANGELOG.md +22 -1
  4. data/CONTRIBUTORS.md +3 -0
  5. data/Gemfile +7 -4
  6. data/README.md +8 -6
  7. data/Rakefile +15 -13
  8. data/ext/smarter_csv/extconf.rb +14 -0
  9. data/ext/smarter_csv/smarter_csv.c +86 -0
  10. data/lib/extensions/hash.rb +4 -2
  11. data/lib/smarter_csv/version.rb +3 -1
  12. data/lib/smarter_csv.rb +519 -10
  13. data/smarter_csv.gemspec +22 -7
  14. metadata +54 -176
  15. data/.gitignore +0 -10
  16. data/.rspec +0 -2
  17. data/.travis.yml +0 -27
  18. data/lib/smarter_csv/smarter_csv.rb +0 -461
  19. data/spec/fixtures/additional_separator.csv +0 -6
  20. data/spec/fixtures/basic.csv +0 -8
  21. data/spec/fixtures/binary.csv +0 -1
  22. data/spec/fixtures/carriage_returns_n.csv +0 -18
  23. data/spec/fixtures/carriage_returns_quoted.csv +0 -3
  24. data/spec/fixtures/carriage_returns_r.csv +0 -1
  25. data/spec/fixtures/carriage_returns_rn.csv +0 -18
  26. data/spec/fixtures/chunk_cornercase.csv +0 -10
  27. data/spec/fixtures/duplicate_headers.csv +0 -3
  28. data/spec/fixtures/empty.csv +0 -5
  29. data/spec/fixtures/empty_columns_1.csv +0 -2
  30. data/spec/fixtures/empty_columns_2.csv +0 -2
  31. data/spec/fixtures/hard_sample.csv +0 -2
  32. data/spec/fixtures/ignore_comments.csv +0 -11
  33. data/spec/fixtures/ignore_comments2.csv +0 -3
  34. data/spec/fixtures/key_mapping.csv +0 -2
  35. data/spec/fixtures/line_endings_n.csv +0 -4
  36. data/spec/fixtures/line_endings_r.csv +0 -1
  37. data/spec/fixtures/line_endings_rn.csv +0 -4
  38. data/spec/fixtures/lots_of_columns.csv +0 -2
  39. data/spec/fixtures/malformed.csv +0 -3
  40. data/spec/fixtures/malformed_header.csv +0 -3
  41. data/spec/fixtures/money.csv +0 -3
  42. data/spec/fixtures/no_header.csv +0 -7
  43. data/spec/fixtures/numeric.csv +0 -5
  44. data/spec/fixtures/pets.csv +0 -5
  45. data/spec/fixtures/problematic.csv +0 -8
  46. data/spec/fixtures/quote_char.csv +0 -9
  47. data/spec/fixtures/quoted.csv +0 -5
  48. data/spec/fixtures/quoted2.csv +0 -4
  49. data/spec/fixtures/separator_colon.csv +0 -4
  50. data/spec/fixtures/separator_comma.csv +0 -4
  51. data/spec/fixtures/separator_pipe.csv +0 -4
  52. data/spec/fixtures/separator_semi.csv +0 -4
  53. data/spec/fixtures/separator_tab.csv +0 -4
  54. data/spec/fixtures/skip_lines.csv +0 -8
  55. data/spec/fixtures/trading.csv +0 -3
  56. data/spec/fixtures/user_import.csv +0 -3
  57. data/spec/fixtures/valid_unicode.csv +0 -5
  58. data/spec/fixtures/with_dashes.csv +0 -8
  59. data/spec/fixtures/with_dates.csv +0 -4
  60. data/spec/smarter_csv/additional_separator_spec.rb +0 -45
  61. data/spec/smarter_csv/binary_file2_spec.rb +0 -24
  62. data/spec/smarter_csv/binary_file_spec.rb +0 -22
  63. data/spec/smarter_csv/blank_spec.rb +0 -55
  64. data/spec/smarter_csv/carriage_return_spec.rb +0 -190
  65. data/spec/smarter_csv/chunked_reading_spec.rb +0 -14
  66. data/spec/smarter_csv/close_file_spec.rb +0 -15
  67. data/spec/smarter_csv/column_separator_spec.rb +0 -95
  68. data/spec/smarter_csv/convert_values_to_numeric_spec.rb +0 -48
  69. data/spec/smarter_csv/duplicate_headers_spec.rb +0 -76
  70. data/spec/smarter_csv/empty_columns_spec.rb +0 -74
  71. data/spec/smarter_csv/extenstions_spec.rb +0 -17
  72. data/spec/smarter_csv/hard_sample_spec.rb +0 -24
  73. data/spec/smarter_csv/header_transformation_spec.rb +0 -21
  74. data/spec/smarter_csv/ignore_comments_spec.rb +0 -45
  75. data/spec/smarter_csv/invalid_headers_spec.rb +0 -38
  76. data/spec/smarter_csv/keep_headers_spec.rb +0 -24
  77. data/spec/smarter_csv/key_mapping_spec.rb +0 -56
  78. data/spec/smarter_csv/line_ending_spec.rb +0 -43
  79. data/spec/smarter_csv/load_basic_spec.rb +0 -20
  80. data/spec/smarter_csv/malformed_spec.rb +0 -25
  81. data/spec/smarter_csv/no_header_spec.rb +0 -29
  82. data/spec/smarter_csv/not_downcase_header_spec.rb +0 -24
  83. data/spec/smarter_csv/parse/column_separator_spec.rb +0 -61
  84. data/spec/smarter_csv/parse/old_csv_library_spec.rb +0 -74
  85. data/spec/smarter_csv/parse/rfc4180_and_more_spec.rb +0 -170
  86. data/spec/smarter_csv/problematic.rb +0 -34
  87. data/spec/smarter_csv/quoted_spec.rb +0 -52
  88. data/spec/smarter_csv/remove_empty_values_spec.rb +0 -13
  89. data/spec/smarter_csv/remove_keys_from_hashes_spec.rb +0 -25
  90. data/spec/smarter_csv/remove_not_mapped_keys_spec.rb +0 -35
  91. data/spec/smarter_csv/remove_values_matching_spec.rb +0 -26
  92. data/spec/smarter_csv/remove_zero_values_spec.rb +0 -25
  93. data/spec/smarter_csv/skip_lines_spec.rb +0 -29
  94. data/spec/smarter_csv/strings_as_keys_spec.rb +0 -24
  95. data/spec/smarter_csv/strip_chars_from_headers_spec.rb +0 -24
  96. data/spec/smarter_csv/trading_spec.rb +0 -25
  97. data/spec/smarter_csv/valid_unicode_spec.rb +0 -94
  98. data/spec/smarter_csv/value_converters_spec.rb +0 -52
  99. data/spec/spec/spec_helper.rb +0 -17
  100. data/spec/spec.opts +0 -2
  101. data/spec/spec_helper.rb +0 -21
@@ -1,76 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'duplicate headers' do
6
- describe 'without special handling / default behavior' do
7
- it 'raises error on duplicate headers' do
8
- expect {
9
- SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", {})
10
- }.to raise_exception(SmarterCSV::DuplicateHeaders)
11
- end
12
-
13
- it 'raises error on duplicate given headers' do
14
- expect {
15
- options = {:user_provided_headers => [:a,:b,:c,:d,:a]}
16
- SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
17
- }.to raise_exception(SmarterCSV::DuplicateHeaders)
18
- end
19
-
20
- it 'does not raise error on missing mapped headers and includes missing headers in message' do
21
- # the mapping is right, but the underlying csv file is bad
22
- options = {:key_mapping => {:email => :a, :firstname => :b, :lastname => :c, :manager_email => :d, :age => :e} }
23
- expect {
24
- SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
25
- }.not_to raise_exception(SmarterCSV::KeyMappingError)
26
- end
27
- end
28
-
29
- describe 'with special handling' do
30
- context 'with given suffix' do
31
- let(:options) { {duplicate_header_suffix: '_'} }
32
-
33
- it 'reads whole file' do
34
- data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
35
- expect(data.size).to eq 2
36
- end
37
-
38
- it 'generates the correct keys' do
39
- data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
40
- expect(data.first.keys).to eq [:email, :firstname, :lastname, :email_2, :age]
41
- end
42
-
43
- it 'enumerates when duplicate headers are given' do
44
- options.merge!({:user_provided_headers => [:a,:b,:c,:a,:a]})
45
- data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
46
- expect(data.first.keys).to eq [:a, :b, :c, :a_2, :a_3]
47
- end
48
-
49
- it 'can remap duplicated headers' do
50
- options.merge!({:key_mapping => {:email => :a, :firstname => :b, :lastname => :c, :email_2 => :d, :age => :e}})
51
- data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
52
- expect(data.first).to eq({a: 'tom@bla.com', b: 'Tom', c: 'Sawyer', d: 'mike@bla.com', e: 34})
53
- end
54
- end
55
-
56
- context 'with empty suffix' do
57
- let(:options) { {duplicate_header_suffix: ''} }
58
-
59
- it 'reads whole file' do
60
- data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
61
- expect(data.size).to eq 2
62
- end
63
-
64
- it 'generates the correct keys' do
65
- data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
66
- expect(data.first.keys).to eq [:email, :firstname, :lastname, :email2, :age]
67
- end
68
-
69
- it 'enumerates when duplicate headers are given' do
70
- options.merge!({:user_provided_headers => [:a,:b,:c,:a,:a]})
71
- data = SmarterCSV.process("#{fixture_path}/duplicate_headers.csv", options)
72
- expect(data.first.keys).to eq [:a, :b, :c, :a2, :a3]
73
- end
74
- end
75
- end
76
- end
@@ -1,74 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'can handle empty columns' do
6
-
7
- describe 'default behavior' do
8
- it 'has empty columns at end' do
9
- data = SmarterCSV.process("#{fixture_path}/empty_columns_1.csv")
10
- data.size.should eq 1
11
- item = data.first
12
- item[:id].should == 123
13
- item[:col1].should == nil
14
- item[:col2].should == nil
15
- item[:col3].should == nil
16
- end
17
-
18
- it 'has empty columns in the middle' do
19
- data = SmarterCSV.process("#{fixture_path}/empty_columns_2.csv")
20
- data.size.should eq 1
21
- item = data.first
22
- item[:id].should == 123
23
- item[:col1].should == nil
24
- item[:col2].should == nil
25
- item[:col3].should == 1
26
- end
27
- end
28
-
29
- describe 'with remove_empty_values: true' do
30
- options = {remove_empty_values: true}
31
- it 'has empty columns at end' do
32
- data = SmarterCSV.process("#{fixture_path}/empty_columns_1.csv", options)
33
- data.size.should eq 1
34
- item = data.first
35
- item[:id].should == 123
36
- item[:col1].should == nil
37
- item[:col2].should == nil
38
- item[:col3].should == nil
39
- end
40
-
41
- it 'has empty columns in the middle' do
42
- data = SmarterCSV.process("#{fixture_path}/empty_columns_2.csv", options)
43
- data.size.should eq 1
44
- item = data.first
45
- item[:id].should == 123
46
- item[:col1].should == nil
47
- item[:col2].should == nil
48
- item[:col3].should == 1
49
- end
50
- end
51
-
52
- describe 'with remove_empty_values: false' do
53
- options = {remove_empty_values: false}
54
- it 'has empty columns at end' do
55
- data = SmarterCSV.process("#{fixture_path}/empty_columns_1.csv", options)
56
- data.size.should eq 1
57
- item = data.first
58
- item[:id].should == 123
59
- item[:col1].should == ''
60
- item[:col2].should == ''
61
- item[:col3].should == ''
62
- end
63
-
64
- it 'has empty columns in the middle' do
65
- data = SmarterCSV.process("#{fixture_path}/empty_columns_2.csv", options)
66
- data.size.should eq 1
67
- item = data.first
68
- item[:id].should == 123
69
- item[:col1].should == ''
70
- item[:col2].should == ''
71
- item[:col3].should == 1
72
- end
73
- end
74
- end
@@ -1,17 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe "Hash.zip" do
4
- it "constructs a new Hash from two Arrays" do
5
- Hash.zip(["a", "b"], [1, 2]).should == { "a" => 1, "b" => 2 }
6
- end
7
-
8
- it "constructs an empty Hash if given no keys" do
9
- Hash.zip([], []).should == {}
10
- Hash.zip([], [1]).should == {}
11
- end
12
-
13
- it "uses nil values if there are more keys than values" do
14
- Hash.zip(["a"], []).should == { "a" => nil }
15
- Hash.zip(["a", "b"], [1]).should == { "a" => 1, "b" => nil }
16
- end
17
- end
@@ -1,24 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'can handle the difficult CSV file' do
6
-
7
- it 'loads the data with default values' do
8
- data = SmarterCSV.process("#{fixture_path}/hard_sample.csv")
9
- data.size.should eq 1
10
- item = data.first
11
- item.keys.count.should == 48
12
- item[:name].should == '#MR1220817'
13
- item[:shipping_method].should == 'Livraison Standard GRATUITE, 2-5 jours avec suivi'
14
- item[:lineitem_name].should == 'Cire Épilation Nacrée'
15
- item[:phone].should == 3366012111111
16
- end
17
-
18
- # the main problem is the data line starting with a # character, but not being a comment
19
- it 'fails to load the CSV file with incorrectly set comment_regexp' do
20
- options = {comment_regexp: /\A#/ }
21
- data = SmarterCSV.process("#{fixture_path}/hard_sample.csv", options)
22
- data.size.should eq 0
23
- end
24
- end
@@ -1,21 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
- it 'loads_file_with_dashes_in_header_fields as strings' do
7
- options = {:strings_as_keys => true}
8
- data = SmarterCSV.process("#{fixture_path}/with_dashes.csv", options)
9
- data.flatten.size.should == 5
10
- data[0]['first_name'].should eq 'Dan'
11
- data[0]['last_name'].should eq 'McAllister'
12
- end
13
-
14
- it 'loads_file_with_dashes_in_header_fields as symbols' do
15
- options = {:strings_as_keys => false}
16
- data = SmarterCSV.process("#{fixture_path}/with_dashes.csv", options)
17
- data.flatten.size.should == 5
18
- data[0][:first_name].should eq 'Dan'
19
- data[0][:last_name].should eq 'McAllister'
20
- end
21
- end
@@ -1,45 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
- it 'by default does not ignore comments in CSV files' do
7
- options = {}
8
- data = SmarterCSV.process("#{fixture_path}/ignore_comments.csv", options)
9
-
10
- data.size.should eq 8
11
-
12
- # all the keys should be symbols
13
- data.each{|item| item.keys.each{|x| x.is_a?(Symbol).should be_truthy}}
14
- data.each do |h|
15
- h.keys.each do |key|
16
- [:"not_a_comment#first_name", :last_name, :dogs, :cats, :birds, :fish].should include( key )
17
- end
18
- end
19
- end
20
-
21
- it 'ignore comments in CSV files using comment_regexp' do
22
- options = {comment_regexp: /\A#/}
23
- data = SmarterCSV.process("#{fixture_path}/ignore_comments.csv", options)
24
-
25
- data.size.should eq 5
26
-
27
- # all the keys should be symbols
28
- data.each{|item| item.keys.each{|x| x.is_a?(Symbol).should be_truthy}}
29
- data.each do |h|
30
- h.keys.each do |key|
31
- [:"not_a_comment#first_name", :last_name, :dogs, :cats, :birds, :fish].should include( key )
32
- end
33
- end
34
- end
35
-
36
- it 'ignore comments in CSV files with CRLF' do
37
- options = {row_sep: "\r\n"}
38
- data = SmarterCSV.process("#{fixture_path}/ignore_comments2.csv", options)
39
-
40
- # all the keys should be symbols
41
- data.size.should eq 1
42
- data.first[:h1].should eq 'a'
43
- data.first[:h2].should eq "b\r\n#c"
44
- end
45
- end
@@ -1,38 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'test exceptions for invalid headers' do
6
- it 'does not raise an error if no required headers are given' do
7
- options = {:required_headers => nil} # order does not matter
8
- data = SmarterCSV.process("#{fixture_path}/user_import.csv", options)
9
- data.size.should eq 2
10
- end
11
-
12
- it 'does not raise an error if no required headers are given' do
13
- options = {:required_headers => []} # order does not matter
14
- data = SmarterCSV.process("#{fixture_path}/user_import.csv", options)
15
- data.size.should eq 2
16
- end
17
-
18
- it 'does not raise an error if the required headers are present' do
19
- options = {:required_headers => [:lastname,:email,:firstname,:manager_email]} # order does not matter
20
- data = SmarterCSV.process("#{fixture_path}/user_import.csv", options)
21
- data.size.should eq 2
22
- end
23
-
24
- it 'raises an error if a required header is missing' do
25
- expect {
26
- options = {:required_headers => [:lastname,:email,:employee_id,:firstname,:manager_email]} # order does not matter
27
- SmarterCSV.process("#{fixture_path}/user_import.csv", options)
28
- }.to raise_exception(SmarterCSV::MissingHeaders)
29
- end
30
-
31
- it 'does not raise error on missing mapped headers and includes missing headers in message' do
32
- # :age does not exist in the CSV header
33
- options = {:key_mapping => {:email => :a, :firstname => :b, :lastname => :c, :manager_email => :d, :age => :e} }
34
- expect {
35
- SmarterCSV.process("#{fixture_path}/user_import.csv", options)
36
- }.not_to raise_exception(SmarterCSV::KeyMappingError)
37
- end
38
- end
@@ -1,24 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
- it 'not_downcase_headers' do
7
- options = {:keep_original_headers => true}
8
- data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
9
- data.size.should == 5
10
- # all the keys should be string
11
- data.each{|item| item.keys.each{|x| x.class.should be == String}}
12
-
13
- data.each do |item|
14
- item.keys.each do |key|
15
- ['First Name','Last Name','Dogs','Cats','Birds','Fish'].should include( key )
16
- end
17
- end
18
-
19
- data.each do |h|
20
- h.size.should <= 6
21
- end
22
- end
23
-
24
- end
@@ -1,56 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
- it 'remove_values_matching' do
7
- options = {:remove_zero_values => true, :key_mapping => {:first_name => :vorname, :last_name => :nachname} }
8
- data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
9
- data.size.should == 5
10
- # all the keys should be symbols
11
- data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
12
-
13
- data.each do |hash|
14
- hash.keys.each do |key|
15
- [:vorname, :nachname, :dogs, :cats, :birds, :fish].should include( key )
16
- end
17
- hash.values.should_not include( 0 )
18
- end
19
-
20
- data.each do |h|
21
- h.size.should <= 6
22
- end
23
- end
24
-
25
- describe 'when keep_original_headers' do
26
- it 'without key mapping' do
27
- options = {:keep_original_headers => true}
28
- data = SmarterCSV.process("#{fixture_path}/key_mapping.csv", options)
29
- data.size.should == 1
30
- data.first.keys.should == ['THIS', 'THAT', 'other']
31
- end
32
-
33
- it 'sets key_mapping to a symbol' do
34
- options = {:keep_original_headers => true, :key_mapping => {'other' => :other}}
35
- data = SmarterCSV.process("#{fixture_path}/key_mapping.csv", options)
36
- data.size.should == 1
37
- data.first.keys.should == ['THIS', 'THAT', :other]
38
- end
39
-
40
- # this previously would set the key to a symbol :OTHER, which was a bug!
41
- it 'sets key_mapping to a string' do
42
- options = {:keep_original_headers => true, :key_mapping => {'other' => 'OTHER'}}
43
- data = SmarterCSV.process("#{fixture_path}/key_mapping.csv", options)
44
- data.size.should == 1
45
- data.first.keys.should == ['THIS', 'THAT', 'OTHER']
46
- end
47
-
48
- # users now have to explicitly set this to a symbol, or change the expected keys to be strings.
49
- it 'sets key_mapping to a symbol' do
50
- options = {:keep_original_headers => true, :key_mapping => {'other' => :OTHER}}
51
- data = SmarterCSV.process("#{fixture_path}/key_mapping.csv", options)
52
- data.size.should == 1
53
- data.first.keys.should == ['THIS', 'THAT', :OTHER]
54
- end
55
- end
56
- end
@@ -1,43 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'process files with line endings explicitly pre-specified' do
6
- it 'reads file with \n line endings' do
7
- options = {:row_sep => "\n"}
8
- data = SmarterCSV.process("#{fixture_path}/line_endings_n.csv", options)
9
- data.size.should == 3
10
- end
11
-
12
- it 'reads file with \r line endings' do
13
- options = {:row_sep => "\r"}
14
- data = SmarterCSV.process("#{fixture_path}/line_endings_r.csv", options)
15
- data.size.should == 3
16
- end
17
-
18
- it 'reads file with \r\n line endings' do
19
- options = {:row_sep => "\r\n"}
20
- data = SmarterCSV.process("#{fixture_path}/line_endings_rn.csv", options)
21
- data.size.should == 3
22
- end
23
- end
24
-
25
- describe 'process files with line endings in automatic mode' do
26
- it 'reads file with \n line endings' do
27
- options = {:row_sep => :auto}
28
- data = SmarterCSV.process("#{fixture_path}/line_endings_n.csv", options)
29
- data.size.should == 3
30
- end
31
-
32
- it 'reads file with \r line endings' do
33
- options = {:row_sep => :auto}
34
- data = SmarterCSV.process("#{fixture_path}/line_endings_r.csv", options)
35
- data.size.should == 3
36
- end
37
-
38
- it 'reads file with \r\n line endings' do
39
- options = {:row_sep => :auto}
40
- data = SmarterCSV.process("#{fixture_path}/line_endings_rn.csv", options)
41
- data.size.should == 3
42
- end
43
- end
@@ -1,20 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
- it 'loads_basic_csv_file' do
7
- data = SmarterCSV.process("#{fixture_path}/basic.csv")
8
- data.size.should == 5
9
-
10
- # all the keys should be symbols
11
- data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
12
- data.each do |h|
13
- h.keys.each do |key|
14
- [:first_name, :last_name, :dogs, :cats, :birds, :fish].should include( key )
15
- end
16
- h.size.should <= 6
17
- end
18
- end
19
-
20
- end
@@ -1,25 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- # according to RFC-4180 quotes inside of "words" shouldbe doubled, but our parser is robust against that.
6
- describe 'malformed CSV quotes' do
7
- context "malformed quotes in header" do
8
- let(:csv_path) { "#{fixture_path}/malformed_header.csv" }
9
- it 'should be resilient against single quotes' do
10
- data = SmarterCSV.process(csv_path)
11
- expect(data[0]).to eq({:name=>"Arnold Schwarzenegger", :dobdob=>"1947-07-30"})
12
- expect(data[1]).to eq({:name=>"Jeff Bridges", :dobdob=>"1949-12-04"})
13
- end
14
- end
15
-
16
- context "malformed quotes in content" do
17
- let(:csv_path) { "#{fixture_path}/malformed.csv" }
18
-
19
- it 'should be resilient against single quotes' do
20
- data = SmarterCSV.process(csv_path)
21
- expect(data[0]).to eq({:name=>"Arnold Schwarzenegger", :dob=>"1947-07-30"})
22
- expect(data[1]).to eq({:name=>"Jeff \"the dude\" Bridges", :dob=>"1949-12-04"})
23
- end
24
- end
25
- end
@@ -1,29 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'no header in file' do
6
- let(:headers) { [:a,:b,:c,:d,:e,:f] }
7
- let(:options) { {:headers_in_file => false, :user_provided_headers => headers} }
8
- subject(:data) { SmarterCSV.process("#{fixture_path}/no_header.csv", options) }
9
-
10
- it 'load the correct number of records' do
11
- data.size.should == 5
12
- end
13
-
14
- it 'uses given symbols for all records' do
15
- data.each do |item|
16
- item.keys.each do |key|
17
- [:a,:b,:c,:d,:e,:f].should include( key )
18
- end
19
- end
20
- end
21
-
22
- it 'loads the correct data' do
23
- data[0].should == {a: "Dan", b: "McAllister", c: 2, d: 0}
24
- data[1].should == {a: "Lucy", b: "Laweless", d: 5, e: 0}
25
- data[2].should == {a: "Miles", b: "O'Brian", c: 0, d: 0, e: 0, f: 21}
26
- data[3].should == {a: "Nancy", b: "Homes", c: 2, d: 0, e: 1}
27
- data[4].should == {a: "Hernán", b: "Curaçon", c: 3, d: 0, e: 0}
28
- end
29
- end
@@ -1,24 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
- it 'not_downcase_headers' do
7
- options = {:downcase_header => false}
8
- data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
9
- data.size.should == 5
10
- # all the keys should be symbols
11
- data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
12
-
13
- data.each do |item|
14
- item.keys.each do |key|
15
- [:First_Name, :Last_Name, :Dogs, :Cats, :Birds, :Fish].should include( key )
16
- end
17
- end
18
-
19
- data.each do |h|
20
- h.size.should <= 6
21
- end
22
- end
23
-
24
- end
@@ -1,61 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe 'parse with col_sep' do
4
- let(:options) { {quote_char: '"'} }
5
-
6
- it 'parses with comma' do
7
- line = "a,b,,d"
8
- options.merge!({col_sep: ","})
9
- array, array_size = SmarterCSV.send(:parse, line, options)
10
- expect(array).to eq ['a', 'b', '', 'd']
11
- expect(array_size).to eq 4
12
- end
13
-
14
- it 'parses trailing commas' do
15
- line = "a,b,c,,"
16
- options.merge!({col_sep: ","})
17
- array, array_size = SmarterCSV.send(:parse, line, options)
18
- expect(array).to eq ['a', 'b', 'c', '', '']
19
- expect(array_size).to eq 5
20
- end
21
-
22
- it 'parses with space' do
23
- line = "a b d"
24
- options.merge!({col_sep: " "})
25
- array, array_size = SmarterCSV.send(:parse, line, options)
26
- expect(array).to eq ['a', 'b', '', 'd']
27
- expect(array_size).to eq 4
28
- end
29
-
30
- it 'parses with tab' do
31
- line = "a\tb\t\td"
32
- options.merge!({col_sep: "\t"})
33
- array, array_size = SmarterCSV.send(:parse, line, options)
34
- expect(array).to eq ['a', 'b', '', 'd']
35
- expect(array_size).to eq 4
36
- end
37
-
38
- it 'parses with multiple space separator' do
39
- line = "a b d"
40
- options.merge!({col_sep: " "})
41
- array, array_size = SmarterCSV.send(:parse, line, options)
42
- expect(array).to eq ['a b', '', 'd']
43
- expect(array_size).to eq 3
44
- end
45
-
46
- it 'parses with multiple char separator' do
47
- line = '<=><=>A<=>B<=>C'
48
- options.merge!({col_sep: "<=>"})
49
- array, array_size = SmarterCSV.send(:parse, line, options)
50
- expect(array).to eq ["", "", "A", "B", "C"]
51
- expect(array_size).to eq 5
52
- end
53
-
54
- it 'parses trailing multiple char separator' do
55
- line = '<=><=>A<=>B<=>C<=><=>'
56
- options.merge!({col_sep: "<=>"})
57
- array, array_size = SmarterCSV.send(:parse, line, options)
58
- expect(array).to eq ["", "", "A", "B", "C", "", ""]
59
- expect(array_size).to eq 7
60
- end
61
- end
@@ -1,74 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe 'old CSV library parsing tests' do
4
- let(:options) { {quote_char: '"', col_sep: ","} }
5
-
6
- [ ["\t", ["\t"]],
7
- ["foo,\"\"\"\"\"\",baz", ["foo", "\"\"", "baz"]],
8
- ["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
9
- ["\"\"\"\n\",\"\"\"\n\"", ["\"\n", "\"\n"]],
10
- ["foo,\"\r\n\",baz", ["foo", "\r\n", "baz"]],
11
- ["\"\"", [""]],
12
- ["foo,\"\"\"\",baz", ["foo", "\"", "baz"]],
13
- ["foo,\"\r.\n\",baz", ["foo", "\r.\n", "baz"]],
14
- ["foo,\"\r\",baz", ["foo", "\r", "baz"]],
15
- ["foo,\"\",baz", ["foo", "", "baz"]],
16
- ["\",\"", [","]],
17
- ["foo", ["foo"]],
18
- [",,", ['', '', '']],
19
- [",", ['', '']],
20
- ["foo,\"\n\",baz", ["foo", "\n", "baz"]],
21
- ["foo,,baz", ["foo", '', "baz"]],
22
- ["\"\"\"\r\",\"\"\"\r\"", ["\"\r", "\"\r"]],
23
- ["\",\",\",\"", [",", ","]],
24
- ["foo,bar,", ["foo", "bar", '']],
25
- [",foo,bar", ['', "foo", "bar"]],
26
- ["foo,bar", ["foo", "bar"]],
27
- [";", [";"]],
28
- ["\t,\t", ["\t", "\t"]],
29
- ["foo,\"\r\n\r\",baz", ["foo", "\r\n\r", "baz"]],
30
- ["foo,\"\r\n\n\",baz", ["foo", "\r\n\n", "baz"]],
31
- ["foo,\"foo,bar\",baz", ["foo", "foo,bar", "baz"]],
32
- [";,;", [";", ";"]]
33
- ].each do |line, result|
34
- it "parses #{line}" do
35
- array, array_size = SmarterCSV.send(:parse, line, options)
36
- expect(array).to eq result
37
- end
38
- end
39
-
40
- [ ["foo,\"\"\"\"\"\",baz", ["foo", "\"\"", "baz"]],
41
- ["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
42
- ["foo,\"\r\n\",baz", ["foo", "\r\n", "baz"]],
43
- ["\"\"", [""]],
44
- ["foo,\"\"\"\",baz", ["foo", "\"", "baz"]],
45
- ["foo,\"\r.\n\",baz", ["foo", "\r.\n", "baz"]],
46
- ["foo,\"\r\",baz", ["foo", "\r", "baz"]],
47
- ["foo,\"\",baz", ["foo", "", "baz"]],
48
- ["foo", ["foo"]],
49
- [",,", ['', '', '']],
50
- [",", ['', '']],
51
- ["foo,\"\n\",baz", ["foo", "\n", "baz"]],
52
- ["foo,,baz", ["foo", '', "baz"]],
53
- ["foo,bar", ["foo", "bar"]],
54
- ["foo,\"\r\n\n\",baz", ["foo", "\r\n\n", "baz"]],
55
- ["foo,\"foo,bar\",baz", ["foo", "foo,bar", "baz"]]
56
- ].each do |line, result|
57
- it "parses #{line}" do
58
- array, array_size = SmarterCSV.send(:parse, line, options)
59
- expect(array).to eq result
60
- end
61
- end
62
-
63
- it 'mixed quotes' do
64
- line = %Q{Ten Thousand,10000, 2710 ,,"10,000","It's ""10 Grand"", baby",10K}
65
- array, array_size = SmarterCSV.send(:parse, line, options)
66
- expect(array).to eq ["Ten Thousand", "10000", " 2710 ", "", "10,000", "It's \"10 Grand\", baby", "10K"]
67
- end
68
-
69
- it 'single quotes in fields' do
70
- line = 'Indoor Chrome,49.2"" L x 49.2"" W x 20.5"" H,Chrome,"Crystal,Metal,Wood",23.12'
71
- array, array_size = SmarterCSV.send(:parse, line, options)
72
- expect(array).to eq ['Indoor Chrome', '49.2" L x 49.2" W x 20.5" H', 'Chrome', 'Crystal,Metal,Wood', '23.12']
73
- end
74
- end