smarter_csv 1.1.5 → 1.12.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (95) hide show
  1. checksums.yaml +5 -5
  2. data/.rspec +1 -2
  3. data/.rubocop.yml +154 -0
  4. data/CHANGELOG.md +364 -0
  5. data/CONTRIBUTORS.md +56 -0
  6. data/Gemfile +7 -2
  7. data/LICENSE.txt +21 -0
  8. data/README.md +44 -441
  9. data/Rakefile +39 -19
  10. data/TO_DO_v2.md +14 -0
  11. data/docs/_introduction.md +56 -0
  12. data/docs/basic_api.md +157 -0
  13. data/docs/batch_processing.md +68 -0
  14. data/docs/data_transformations.md +50 -0
  15. data/docs/examples.md +75 -0
  16. data/docs/header_transformations.md +113 -0
  17. data/docs/header_validations.md +36 -0
  18. data/docs/options.md +98 -0
  19. data/docs/row_col_sep.md +104 -0
  20. data/docs/value_converters.md +68 -0
  21. data/ext/smarter_csv/extconf.rb +14 -0
  22. data/ext/smarter_csv/smarter_csv.c +97 -0
  23. data/lib/smarter_csv/auto_detection.rb +78 -0
  24. data/lib/smarter_csv/errors.rb +16 -0
  25. data/lib/smarter_csv/file_io.rb +50 -0
  26. data/lib/smarter_csv/hash_transformations.rb +91 -0
  27. data/lib/smarter_csv/header_transformations.rb +63 -0
  28. data/lib/smarter_csv/header_validations.rb +34 -0
  29. data/lib/smarter_csv/headers.rb +68 -0
  30. data/lib/smarter_csv/options.rb +95 -0
  31. data/lib/smarter_csv/parser.rb +90 -0
  32. data/lib/smarter_csv/reader.rb +243 -0
  33. data/lib/smarter_csv/version.rb +3 -1
  34. data/lib/smarter_csv/writer.rb +116 -0
  35. data/lib/smarter_csv.rb +91 -3
  36. data/smarter_csv.gemspec +43 -20
  37. metadata +122 -137
  38. data/.gitignore +0 -8
  39. data/.travis.yml +0 -19
  40. data/lib/extensions/hash.rb +0 -7
  41. data/lib/smarter_csv/smarter_csv.rb +0 -281
  42. data/spec/fixtures/basic.csv +0 -8
  43. data/spec/fixtures/binary.csv +0 -1
  44. data/spec/fixtures/carriage_returns_n.csv +0 -18
  45. data/spec/fixtures/carriage_returns_quoted.csv +0 -3
  46. data/spec/fixtures/carriage_returns_r.csv +0 -1
  47. data/spec/fixtures/carriage_returns_rn.csv +0 -18
  48. data/spec/fixtures/chunk_cornercase.csv +0 -10
  49. data/spec/fixtures/empty.csv +0 -5
  50. data/spec/fixtures/line_endings_n.csv +0 -4
  51. data/spec/fixtures/line_endings_r.csv +0 -1
  52. data/spec/fixtures/line_endings_rn.csv +0 -4
  53. data/spec/fixtures/lots_of_columns.csv +0 -2
  54. data/spec/fixtures/malformed.csv +0 -3
  55. data/spec/fixtures/malformed_header.csv +0 -3
  56. data/spec/fixtures/money.csv +0 -3
  57. data/spec/fixtures/no_header.csv +0 -7
  58. data/spec/fixtures/numeric.csv +0 -5
  59. data/spec/fixtures/pets.csv +0 -5
  60. data/spec/fixtures/quoted.csv +0 -5
  61. data/spec/fixtures/separator.csv +0 -4
  62. data/spec/fixtures/skip_lines.csv +0 -8
  63. data/spec/fixtures/valid_unicode.csv +0 -5
  64. data/spec/fixtures/with_dashes.csv +0 -8
  65. data/spec/fixtures/with_dates.csv +0 -4
  66. data/spec/smarter_csv/binary_file2_spec.rb +0 -24
  67. data/spec/smarter_csv/binary_file_spec.rb +0 -22
  68. data/spec/smarter_csv/carriage_return_spec.rb +0 -170
  69. data/spec/smarter_csv/chunked_reading_spec.rb +0 -14
  70. data/spec/smarter_csv/close_file_spec.rb +0 -15
  71. data/spec/smarter_csv/column_separator_spec.rb +0 -11
  72. data/spec/smarter_csv/convert_values_to_numeric_spec.rb +0 -48
  73. data/spec/smarter_csv/extenstions_spec.rb +0 -17
  74. data/spec/smarter_csv/header_transformation_spec.rb +0 -21
  75. data/spec/smarter_csv/keep_headers_spec.rb +0 -24
  76. data/spec/smarter_csv/key_mapping_spec.rb +0 -25
  77. data/spec/smarter_csv/line_ending_spec.rb +0 -43
  78. data/spec/smarter_csv/load_basic_spec.rb +0 -20
  79. data/spec/smarter_csv/malformed_spec.rb +0 -21
  80. data/spec/smarter_csv/no_header_spec.rb +0 -24
  81. data/spec/smarter_csv/not_downcase_header_spec.rb +0 -24
  82. data/spec/smarter_csv/quoted_spec.rb +0 -23
  83. data/spec/smarter_csv/remove_empty_values_spec.rb +0 -13
  84. data/spec/smarter_csv/remove_keys_from_hashes_spec.rb +0 -25
  85. data/spec/smarter_csv/remove_not_mapped_keys_spec.rb +0 -35
  86. data/spec/smarter_csv/remove_values_matching_spec.rb +0 -26
  87. data/spec/smarter_csv/remove_zero_values_spec.rb +0 -25
  88. data/spec/smarter_csv/skip_lines_spec.rb +0 -29
  89. data/spec/smarter_csv/strings_as_keys_spec.rb +0 -24
  90. data/spec/smarter_csv/strip_chars_from_headers_spec.rb +0 -24
  91. data/spec/smarter_csv/valid_unicode_spec.rb +0 -94
  92. data/spec/smarter_csv/value_converters_spec.rb +0 -52
  93. data/spec/spec/spec_helper.rb +0 -17
  94. data/spec/spec.opts +0 -2
  95. data/spec/spec_helper.rb +0 -21
@@ -1,29 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
- it 'loads_csv_file_skipping_lines' do
7
- options = {skip_lines: 3}
8
- data = SmarterCSV.process("#{fixture_path}/skip_lines.csv", options)
9
- data.size.should == 4
10
-
11
- data.each do |item|
12
- item.keys.each do |key|
13
- [:first_name,:last_name,:dogs,:cats,:birds,:fish].should include(key)
14
- end
15
- end
16
- end
17
-
18
- it 'loads_csv_with_user_defined_headers' do
19
- options = {:skip_lines => 3, :headers_in_file => true, :user_provided_headers => [:a,:b,:c,:d,:e,:f]}
20
- data = SmarterCSV.process("#{fixture_path}/skip_lines.csv", options)
21
- data.size.should == 4
22
-
23
- data.each do |item|
24
- item.keys.each do |key|
25
- [:a,:b,:c,:d,:e,:f].should include( key )
26
- end
27
- end
28
- end
29
- end
@@ -1,24 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
- it 'use_strings_as_keys' do
7
- options = {:strings_as_keys => true}
8
- data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
9
- data.size.should == 5
10
- # all the keys should be symbols
11
- data.each{|item| item.keys.each{|x| x.class.should be == String}}
12
-
13
- data.each do |item|
14
- item.keys.each do |key|
15
- ["first_name", "last_name", "dogs", "cats", "birds", "fish"].should include( key )
16
- end
17
- end
18
-
19
- data.each do |h|
20
- h.size.should <= 6
21
- end
22
- end
23
-
24
- end
@@ -1,24 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
- it 'strip_whitespace_from_headers' do
7
- options = {:strip_chars_from_headers => ' '}
8
- data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
9
- data.size.should == 5
10
- # all the keys should be symbols
11
- data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
12
-
13
- data.each do |item|
14
- item.keys.each do |key|
15
- [:firstname, :lastname, :dogs, :cats, :birds, :fish].should include( key )
16
- end
17
- end
18
-
19
- data.each do |h|
20
- h.size.should <= 6
21
- end
22
- end
23
-
24
- end
@@ -1,94 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- describe 'be_able_to' do
6
-
7
-
8
- it 'loads file with unicode strings' do
9
- options = {}
10
- data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
11
- data.flatten.size.should == 4
12
- data[0][:artist].should eq 'Кино'
13
- data[0][:track].should eq 'Мама, мы все сошли с ума'
14
- data[0][:album].should eq 'Группа Крови'
15
- data[0][:label].should eq 'Moroz Records'
16
- data[0][:year].should eq 1998
17
-
18
- data[0].should eq data[1]
19
-
20
- data[2][:artist].should eq 'Rammstein'
21
- data[2][:track].should eq 'Frühling in Paris'
22
- data[2][:album].should eq 'Liebe ist für alle da'
23
- data[2][:label].should eq 'Vagrant'
24
- data[2][:year].should eq 2009
25
-
26
- data[2].should eq data[3]
27
- end
28
-
29
- it 'loads file with unicode strings, when forcing utf8' do
30
- options = {:force_utf8 => true}
31
- data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
32
- data.flatten.size.should == 4
33
- data[0][:artist].should eq 'Кино'
34
- data[0][:track].should eq 'Мама, мы все сошли с ума'
35
- data[0][:album].should eq 'Группа Крови'
36
- data[0][:label].should eq 'Moroz Records'
37
- data[0][:year].should eq 1998
38
-
39
- data[0].should eq data[1]
40
-
41
- data[2][:artist].should eq 'Rammstein'
42
- data[2][:track].should eq 'Frühling in Paris'
43
- data[2][:album].should eq 'Liebe ist für alle da'
44
- data[2][:label].should eq 'Vagrant'
45
- data[2][:year].should eq 2009
46
-
47
- data[2].should eq data[3]
48
- end
49
-
50
-
51
-
52
- it 'loads file with unicode strings, when loading from binary input' do
53
- options = {:file_encoding => 'binary'}
54
- data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
55
- data.flatten.size.should == 4
56
- data[0][:artist].should eq 'Кино'
57
- data[0][:track].should eq 'Мама, мы все сошли с ума'
58
- data[0][:album].should eq 'Группа Крови'
59
- data[0][:label].should eq 'Moroz Records'
60
- data[0][:year].should eq 1998
61
-
62
- data[0].should eq data[1]
63
-
64
- data[2][:artist].should eq 'Rammstein'
65
- data[2][:track].should eq 'Frühling in Paris'
66
- data[2][:album].should eq 'Liebe ist für alle da'
67
- data[2][:label].should eq 'Vagrant'
68
- data[2][:year].should eq 2009
69
-
70
- data[2].should eq data[3]
71
- end
72
-
73
- it 'loads file with unicode strings, when forcing utf8 with binary input' do
74
- options = {:file_encoding => 'binary', :force_utf8 => true}
75
- data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
76
- data.flatten.size.should == 4
77
- data[0][:artist].should eq 'Кино'
78
- data[0][:track].should eq 'Мама, мы все сошли с ума'
79
- data[0][:album].should eq 'Группа Крови'
80
- data[0][:label].should eq 'Moroz Records'
81
- data[0][:year].should eq 1998
82
-
83
- data[0].should eq data[1]
84
-
85
- data[2][:artist].should eq 'Rammstein'
86
- data[2][:track].should eq 'Frühling in Paris'
87
- data[2][:album].should eq 'Liebe ist für alle da'
88
- data[2][:label].should eq 'Vagrant'
89
- data[2][:year].should eq 2009
90
-
91
- data[2].should eq data[3]
92
- end
93
-
94
- end
@@ -1,52 +0,0 @@
1
- require 'spec_helper'
2
-
3
- fixture_path = 'spec/fixtures'
4
-
5
- require 'date'
6
- class DateConverter
7
- def self.convert(value)
8
- Date.strptime( value, '%m/%d/%Y')
9
- end
10
- end
11
-
12
- class CurrencyConverter
13
- def self.convert(value)
14
- value.sub(/[$]/,'').to_f # would be nice to add a computed column :currency => '€'
15
- end
16
- end
17
-
18
- describe 'be_able_to' do
19
- it 'convert date values into Date instances' do
20
- options = {:value_converters => {:date => DateConverter}}
21
- data = SmarterCSV.process("#{fixture_path}/with_dates.csv", options)
22
- data.flatten.size.should == 3
23
- data[0][:date].class.should eq Date
24
- data[0][:date].to_s.should eq "1998-10-30"
25
- data[1][:date].to_s.should eq "2011-02-01"
26
- data[2][:date].to_s.should eq "2013-01-09"
27
- end
28
-
29
- it 'converts dollar prices into float values' do
30
- options = {:value_converters => {:price => CurrencyConverter}}
31
- data = SmarterCSV.process("#{fixture_path}/money.csv", options)
32
- data.flatten.size.should == 2
33
- data[0][:price].class.should eq Float
34
- data[0][:price].should eq 9.99
35
- data[1][:price].should eq 14.99
36
- end
37
-
38
- it 'convert can use multiple value converters' do
39
- options = {:value_converters => {:date => DateConverter, :price => CurrencyConverter}}
40
- data = SmarterCSV.process("#{fixture_path}/with_dates.csv", options)
41
- data.flatten.size.should == 3
42
- data[0][:date].class.should eq Date
43
- data[0][:date].to_s.should eq "1998-10-30"
44
- data[1][:date].to_s.should eq "2011-02-01"
45
- data[2][:date].to_s.should eq "2013-01-09"
46
-
47
- data[0][:price].class.should eq Float
48
- data[0][:price].should eq 44.50
49
- data[1][:price].should eq 15.0
50
- data[2][:price].should eq 0.11
51
- end
52
- end
@@ -1,17 +0,0 @@
1
- # This file was generated by the `rspec --init` command. Conventionally, all
2
- # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
3
- # Require this file using `require "spec_helper"` to ensure that it is only
4
- # loaded once.
5
- #
6
- # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
7
- RSpec.configure do |config|
8
- config.treat_symbols_as_metadata_keys_with_true_values = true
9
- config.run_all_when_everything_filtered = true
10
- config.filter_run :focus
11
-
12
- # Run specs in random order to surface order dependencies. If you find an
13
- # order dependency and want to debug it, you can fix the order by providing
14
- # the seed, which is printed after each run.
15
- # --seed 1234
16
- config.order = 'random'
17
- end
data/spec/spec.opts DELETED
@@ -1,2 +0,0 @@
1
- --color
2
- --backtrace
data/spec/spec_helper.rb DELETED
@@ -1,21 +0,0 @@
1
- require 'rubygems'
2
- require 'bundler/setup'
3
-
4
- Bundler.require(:default)
5
-
6
- require 'smarter_csv'
7
-
8
-
9
- RSpec.configure do |config|
10
- config.treat_symbols_as_metadata_keys_with_true_values = true
11
- config.filter_run :focus => true
12
- config.run_all_when_everything_filtered = true
13
- # config.fixture_path = 'spec/fixtures'
14
-
15
- # config.mock_with :rr
16
- # config.before(:each) do
17
- # Project.delete_all
18
- # Category.delete_all
19
- # end
20
- end
21
-