smarter_csv 1.1.5 → 1.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. checksums.yaml +5 -5
  2. data/.rspec +1 -2
  3. data/.rubocop.yml +154 -0
  4. data/CHANGELOG.md +364 -0
  5. data/CONTRIBUTORS.md +56 -0
  6. data/Gemfile +7 -2
  7. data/LICENSE.txt +21 -0
  8. data/README.md +44 -441
  9. data/Rakefile +39 -19
  10. data/TO_DO_v2.md +14 -0
  11. data/docs/_introduction.md +56 -0
  12. data/docs/basic_api.md +157 -0
  13. data/docs/batch_processing.md +68 -0
  14. data/docs/data_transformations.md +50 -0
  15. data/docs/examples.md +75 -0
  16. data/docs/header_transformations.md +113 -0
  17. data/docs/header_validations.md +36 -0
  18. data/docs/options.md +98 -0
  19. data/docs/row_col_sep.md +104 -0
  20. data/docs/value_converters.md +68 -0
  21. data/ext/smarter_csv/extconf.rb +14 -0
  22. data/ext/smarter_csv/smarter_csv.c +97 -0
  23. data/lib/smarter_csv/auto_detection.rb +78 -0
  24. data/lib/smarter_csv/errors.rb +16 -0
  25. data/lib/smarter_csv/file_io.rb +50 -0
  26. data/lib/smarter_csv/hash_transformations.rb +91 -0
  27. data/lib/smarter_csv/header_transformations.rb +63 -0
  28. data/lib/smarter_csv/header_validations.rb +34 -0
  29. data/lib/smarter_csv/headers.rb +68 -0
  30. data/lib/smarter_csv/options.rb +95 -0
  31. data/lib/smarter_csv/parser.rb +90 -0
  32. data/lib/smarter_csv/reader.rb +243 -0
  33. data/lib/smarter_csv/version.rb +3 -1
  34. data/lib/smarter_csv/writer.rb +116 -0
  35. data/lib/smarter_csv.rb +91 -3
  36. data/smarter_csv.gemspec +43 -20
  37. metadata +122 -137
  38. data/.gitignore +0 -8
  39. data/.travis.yml +0 -19
  40. data/lib/extensions/hash.rb +0 -7
  41. data/lib/smarter_csv/smarter_csv.rb +0 -281
  42. data/spec/fixtures/basic.csv +0 -8
  43. data/spec/fixtures/binary.csv +0 -1
  44. data/spec/fixtures/carriage_returns_n.csv +0 -18
  45. data/spec/fixtures/carriage_returns_quoted.csv +0 -3
  46. data/spec/fixtures/carriage_returns_r.csv +0 -1
  47. data/spec/fixtures/carriage_returns_rn.csv +0 -18
  48. data/spec/fixtures/chunk_cornercase.csv +0 -10
  49. data/spec/fixtures/empty.csv +0 -5
  50. data/spec/fixtures/line_endings_n.csv +0 -4
  51. data/spec/fixtures/line_endings_r.csv +0 -1
  52. data/spec/fixtures/line_endings_rn.csv +0 -4
  53. data/spec/fixtures/lots_of_columns.csv +0 -2
  54. data/spec/fixtures/malformed.csv +0 -3
  55. data/spec/fixtures/malformed_header.csv +0 -3
  56. data/spec/fixtures/money.csv +0 -3
  57. data/spec/fixtures/no_header.csv +0 -7
  58. data/spec/fixtures/numeric.csv +0 -5
  59. data/spec/fixtures/pets.csv +0 -5
  60. data/spec/fixtures/quoted.csv +0 -5
  61. data/spec/fixtures/separator.csv +0 -4
  62. data/spec/fixtures/skip_lines.csv +0 -8
  63. data/spec/fixtures/valid_unicode.csv +0 -5
  64. data/spec/fixtures/with_dashes.csv +0 -8
  65. data/spec/fixtures/with_dates.csv +0 -4
  66. data/spec/smarter_csv/binary_file2_spec.rb +0 -24
  67. data/spec/smarter_csv/binary_file_spec.rb +0 -22
  68. data/spec/smarter_csv/carriage_return_spec.rb +0 -170
  69. data/spec/smarter_csv/chunked_reading_spec.rb +0 -14
  70. data/spec/smarter_csv/close_file_spec.rb +0 -15
  71. data/spec/smarter_csv/column_separator_spec.rb +0 -11
  72. data/spec/smarter_csv/convert_values_to_numeric_spec.rb +0 -48
  73. data/spec/smarter_csv/extenstions_spec.rb +0 -17
  74. data/spec/smarter_csv/header_transformation_spec.rb +0 -21
  75. data/spec/smarter_csv/keep_headers_spec.rb +0 -24
  76. data/spec/smarter_csv/key_mapping_spec.rb +0 -25
  77. data/spec/smarter_csv/line_ending_spec.rb +0 -43
  78. data/spec/smarter_csv/load_basic_spec.rb +0 -20
  79. data/spec/smarter_csv/malformed_spec.rb +0 -21
  80. data/spec/smarter_csv/no_header_spec.rb +0 -24
  81. data/spec/smarter_csv/not_downcase_header_spec.rb +0 -24
  82. data/spec/smarter_csv/quoted_spec.rb +0 -23
  83. data/spec/smarter_csv/remove_empty_values_spec.rb +0 -13
  84. data/spec/smarter_csv/remove_keys_from_hashes_spec.rb +0 -25
  85. data/spec/smarter_csv/remove_not_mapped_keys_spec.rb +0 -35
  86. data/spec/smarter_csv/remove_values_matching_spec.rb +0 -26
  87. data/spec/smarter_csv/remove_zero_values_spec.rb +0 -25
  88. data/spec/smarter_csv/skip_lines_spec.rb +0 -29
  89. data/spec/smarter_csv/strings_as_keys_spec.rb +0 -24
  90. data/spec/smarter_csv/strip_chars_from_headers_spec.rb +0 -24
  91. data/spec/smarter_csv/valid_unicode_spec.rb +0 -94
  92. data/spec/smarter_csv/value_converters_spec.rb +0 -52
  93. data/spec/spec/spec_helper.rb +0 -17
  94. data/spec/spec.opts +0 -2
  95. data/spec/spec_helper.rb +0 -21
@@ -1,29 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
- it 'loads_csv_file_skipping_lines' do
7
- options = {skip_lines: 3}
8
- data = SmarterCSV.process("#{fixture_path}/skip_lines.csv", options)
9
- data.size.should == 4
10
-
11
- data.each do |item|
12
- item.keys.each do |key|
13
- [:first_name,:last_name,:dogs,:cats,:birds,:fish].should include(key)
14
- end
15
- end
16
- end
17
-
18
- it 'loads_csv_with_user_defined_headers' do
19
- options = {:skip_lines => 3, :headers_in_file => true, :user_provided_headers => [:a,:b,:c,:d,:e,:f]}
20
- data = SmarterCSV.process("#{fixture_path}/skip_lines.csv", options)
21
- data.size.should == 4
22
-
23
- data.each do |item|
24
- item.keys.each do |key|
25
- [:a,:b,:c,:d,:e,:f].should include( key )
26
- end
27
- end
28
- end
29
- end
@@ -1,24 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
- it 'use_strings_as_keys' do
7
- options = {:strings_as_keys => true}
8
- data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
9
- data.size.should == 5
10
- # all the keys should be symbols
11
- data.each{|item| item.keys.each{|x| x.class.should be == String}}
12
-
13
- data.each do |item|
14
- item.keys.each do |key|
15
- ["first_name", "last_name", "dogs", "cats", "birds", "fish"].should include( key )
16
- end
17
- end
18
-
19
- data.each do |h|
20
- h.size.should <= 6
21
- end
22
- end
23
-
24
- end
@@ -1,24 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
- it 'strip_whitespace_from_headers' do
7
- options = {:strip_chars_from_headers => ' '}
8
- data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
9
- data.size.should == 5
10
- # all the keys should be symbols
11
- data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
12
-
13
- data.each do |item|
14
- item.keys.each do |key|
15
- [:firstname, :lastname, :dogs, :cats, :birds, :fish].should include( key )
16
- end
17
- end
18
-
19
- data.each do |h|
20
- h.size.should <= 6
21
- end
22
- end
23
-
24
- end
@@ -1,94 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
-
7
-
8
- it 'loads file with unicode strings' do
9
- options = {}
10
- data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
11
- data.flatten.size.should == 4
12
- data[0][:artist].should eq 'Кино'
13
- data[0][:track].should eq 'Мама, мы все сошли с ума'
14
- data[0][:album].should eq 'Группа Крови'
15
- data[0][:label].should eq 'Moroz Records'
16
- data[0][:year].should eq 1998
17
-
18
- data[0].should eq data[1]
19
-
20
- data[2][:artist].should eq 'Rammstein'
21
- data[2][:track].should eq 'Frühling in Paris'
22
- data[2][:album].should eq 'Liebe ist für alle da'
23
- data[2][:label].should eq 'Vagrant'
24
- data[2][:year].should eq 2009
25
-
26
- data[2].should eq data[3]
27
- end
28
-
29
- it 'loads file with unicode strings, when forcing utf8' do
30
- options = {:force_utf8 => true}
31
- data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
32
- data.flatten.size.should == 4
33
- data[0][:artist].should eq 'Кино'
34
- data[0][:track].should eq 'Мама, мы все сошли с ума'
35
- data[0][:album].should eq 'Группа Крови'
36
- data[0][:label].should eq 'Moroz Records'
37
- data[0][:year].should eq 1998
38
-
39
- data[0].should eq data[1]
40
-
41
- data[2][:artist].should eq 'Rammstein'
42
- data[2][:track].should eq 'Frühling in Paris'
43
- data[2][:album].should eq 'Liebe ist für alle da'
44
- data[2][:label].should eq 'Vagrant'
45
- data[2][:year].should eq 2009
46
-
47
- data[2].should eq data[3]
48
- end
49
-
50
-
51
-
52
- it 'loads file with unicode strings, when loading from binary input' do
53
- options = {:file_encoding => 'binary'}
54
- data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
55
- data.flatten.size.should == 4
56
- data[0][:artist].should eq 'Кино'
57
- data[0][:track].should eq 'Мама, мы все сошли с ума'
58
- data[0][:album].should eq 'Группа Крови'
59
- data[0][:label].should eq 'Moroz Records'
60
- data[0][:year].should eq 1998
61
-
62
- data[0].should eq data[1]
63
-
64
- data[2][:artist].should eq 'Rammstein'
65
- data[2][:track].should eq 'Frühling in Paris'
66
- data[2][:album].should eq 'Liebe ist für alle da'
67
- data[2][:label].should eq 'Vagrant'
68
- data[2][:year].should eq 2009
69
-
70
- data[2].should eq data[3]
71
- end
72
-
73
- it 'loads file with unicode strings, when forcing utf8 with binary input' do
74
- options = {:file_encoding => 'binary', :force_utf8 => true}
75
- data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
76
- data.flatten.size.should == 4
77
- data[0][:artist].should eq 'Кино'
78
- data[0][:track].should eq 'Мама, мы все сошли с ума'
79
- data[0][:album].should eq 'Группа Крови'
80
- data[0][:label].should eq 'Moroz Records'
81
- data[0][:year].should eq 1998
82
-
83
- data[0].should eq data[1]
84
-
85
- data[2][:artist].should eq 'Rammstein'
86
- data[2][:track].should eq 'Frühling in Paris'
87
- data[2][:album].should eq 'Liebe ist für alle da'
88
- data[2][:label].should eq 'Vagrant'
89
- data[2][:year].should eq 2009
90
-
91
- data[2].should eq data[3]
92
- end
93
-
94
- end
@@ -1,52 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- require 'date'
6
- class DateConverter
7
- def self.convert(value)
8
- Date.strptime( value, '%m/%d/%Y')
9
- end
10
- end
11
-
12
- class CurrencyConverter
13
- def self.convert(value)
14
- value.sub(/[$]/,'').to_f # would be nice to add a computed column :currency => '€'
15
- end
16
- end
17
-
18
- describe 'be_able_to' do
19
- it 'convert date values into Date instances' do
20
- options = {:value_converters => {:date => DateConverter}}
21
- data = SmarterCSV.process("#{fixture_path}/with_dates.csv", options)
22
- data.flatten.size.should == 3
23
- data[0][:date].class.should eq Date
24
- data[0][:date].to_s.should eq "1998-10-30"
25
- data[1][:date].to_s.should eq "2011-02-01"
26
- data[2][:date].to_s.should eq "2013-01-09"
27
- end
28
-
29
- it 'converts dollar prices into float values' do
30
- options = {:value_converters => {:price => CurrencyConverter}}
31
- data = SmarterCSV.process("#{fixture_path}/money.csv", options)
32
- data.flatten.size.should == 2
33
- data[0][:price].class.should eq Float
34
- data[0][:price].should eq 9.99
35
- data[1][:price].should eq 14.99
36
- end
37
-
38
- it 'convert can use multiple value converters' do
39
- options = {:value_converters => {:date => DateConverter, :price => CurrencyConverter}}
40
- data = SmarterCSV.process("#{fixture_path}/with_dates.csv", options)
41
- data.flatten.size.should == 3
42
- data[0][:date].class.should eq Date
43
- data[0][:date].to_s.should eq "1998-10-30"
44
- data[1][:date].to_s.should eq "2011-02-01"
45
- data[2][:date].to_s.should eq "2013-01-09"
46
-
47
- data[0][:price].class.should eq Float
48
- data[0][:price].should eq 44.50
49
- data[1][:price].should eq 15.0
50
- data[2][:price].should eq 0.11
51
- end
52
- end
@@ -1,17 +0,0 @@
1
- # This file was generated by the `rspec --init` command. Conventionally, all
2
- # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
3
- # Require this file using `require "spec_helper"` to ensure that it is only
4
- # loaded once.
5
- #
6
- # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
7
- RSpec.configure do |config|
8
- config.treat_symbols_as_metadata_keys_with_true_values = true
9
- config.run_all_when_everything_filtered = true
10
- config.filter_run :focus
11
-
12
- # Run specs in random order to surface order dependencies. If you find an
13
- # order dependency and want to debug it, you can fix the order by providing
14
- # the seed, which is printed after each run.
15
- # --seed 1234
16
- config.order = 'random'
17
- end
data/spec/spec.opts DELETED
@@ -1,2 +0,0 @@
1
- --color
2
- --backtrace
data/spec/spec_helper.rb DELETED
@@ -1,21 +0,0 @@
1
- require 'rubygems'
2
- require 'bundler/setup'
3
-
4
- Bundler.require(:default)
5
-
6
- require 'smarter_csv'
7
-
8
-
9
- RSpec.configure do |config|
10
- config.treat_symbols_as_metadata_keys_with_true_values = true
11
- config.filter_run :focus => true
12
- config.run_all_when_everything_filtered = true
13
- # config.fixture_path = 'spec/fixtures'
14
-
15
- # config.mock_with :rr
16
- # config.before(:each) do
17
- # Project.delete_all
18
- # Category.delete_all
19
- # end
20
- end
21
-