smarter_csv 1.1.5 → 1.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.rspec +1 -2
- data/.rubocop.yml +154 -0
- data/CHANGELOG.md +364 -0
- data/CONTRIBUTORS.md +56 -0
- data/Gemfile +7 -2
- data/LICENSE.txt +21 -0
- data/README.md +44 -441
- data/Rakefile +39 -19
- data/TO_DO_v2.md +14 -0
- data/docs/_introduction.md +56 -0
- data/docs/basic_api.md +157 -0
- data/docs/batch_processing.md +68 -0
- data/docs/data_transformations.md +50 -0
- data/docs/examples.md +75 -0
- data/docs/header_transformations.md +113 -0
- data/docs/header_validations.md +36 -0
- data/docs/options.md +98 -0
- data/docs/row_col_sep.md +104 -0
- data/docs/value_converters.md +68 -0
- data/ext/smarter_csv/extconf.rb +14 -0
- data/ext/smarter_csv/smarter_csv.c +97 -0
- data/lib/smarter_csv/auto_detection.rb +78 -0
- data/lib/smarter_csv/errors.rb +16 -0
- data/lib/smarter_csv/file_io.rb +50 -0
- data/lib/smarter_csv/hash_transformations.rb +91 -0
- data/lib/smarter_csv/header_transformations.rb +63 -0
- data/lib/smarter_csv/header_validations.rb +34 -0
- data/lib/smarter_csv/headers.rb +68 -0
- data/lib/smarter_csv/options.rb +95 -0
- data/lib/smarter_csv/parser.rb +90 -0
- data/lib/smarter_csv/reader.rb +243 -0
- data/lib/smarter_csv/version.rb +3 -1
- data/lib/smarter_csv/writer.rb +116 -0
- data/lib/smarter_csv.rb +91 -3
- data/smarter_csv.gemspec +43 -20
- metadata +122 -137
- data/.gitignore +0 -8
- data/.travis.yml +0 -19
- data/lib/extensions/hash.rb +0 -7
- data/lib/smarter_csv/smarter_csv.rb +0 -281
- data/spec/fixtures/basic.csv +0 -8
- data/spec/fixtures/binary.csv +0 -1
- data/spec/fixtures/carriage_returns_n.csv +0 -18
- data/spec/fixtures/carriage_returns_quoted.csv +0 -3
- data/spec/fixtures/carriage_returns_r.csv +0 -1
- data/spec/fixtures/carriage_returns_rn.csv +0 -18
- data/spec/fixtures/chunk_cornercase.csv +0 -10
- data/spec/fixtures/empty.csv +0 -5
- data/spec/fixtures/line_endings_n.csv +0 -4
- data/spec/fixtures/line_endings_r.csv +0 -1
- data/spec/fixtures/line_endings_rn.csv +0 -4
- data/spec/fixtures/lots_of_columns.csv +0 -2
- data/spec/fixtures/malformed.csv +0 -3
- data/spec/fixtures/malformed_header.csv +0 -3
- data/spec/fixtures/money.csv +0 -3
- data/spec/fixtures/no_header.csv +0 -7
- data/spec/fixtures/numeric.csv +0 -5
- data/spec/fixtures/pets.csv +0 -5
- data/spec/fixtures/quoted.csv +0 -5
- data/spec/fixtures/separator.csv +0 -4
- data/spec/fixtures/skip_lines.csv +0 -8
- data/spec/fixtures/valid_unicode.csv +0 -5
- data/spec/fixtures/with_dashes.csv +0 -8
- data/spec/fixtures/with_dates.csv +0 -4
- data/spec/smarter_csv/binary_file2_spec.rb +0 -24
- data/spec/smarter_csv/binary_file_spec.rb +0 -22
- data/spec/smarter_csv/carriage_return_spec.rb +0 -170
- data/spec/smarter_csv/chunked_reading_spec.rb +0 -14
- data/spec/smarter_csv/close_file_spec.rb +0 -15
- data/spec/smarter_csv/column_separator_spec.rb +0 -11
- data/spec/smarter_csv/convert_values_to_numeric_spec.rb +0 -48
- data/spec/smarter_csv/extenstions_spec.rb +0 -17
- data/spec/smarter_csv/header_transformation_spec.rb +0 -21
- data/spec/smarter_csv/keep_headers_spec.rb +0 -24
- data/spec/smarter_csv/key_mapping_spec.rb +0 -25
- data/spec/smarter_csv/line_ending_spec.rb +0 -43
- data/spec/smarter_csv/load_basic_spec.rb +0 -20
- data/spec/smarter_csv/malformed_spec.rb +0 -21
- data/spec/smarter_csv/no_header_spec.rb +0 -24
- data/spec/smarter_csv/not_downcase_header_spec.rb +0 -24
- data/spec/smarter_csv/quoted_spec.rb +0 -23
- data/spec/smarter_csv/remove_empty_values_spec.rb +0 -13
- data/spec/smarter_csv/remove_keys_from_hashes_spec.rb +0 -25
- data/spec/smarter_csv/remove_not_mapped_keys_spec.rb +0 -35
- data/spec/smarter_csv/remove_values_matching_spec.rb +0 -26
- data/spec/smarter_csv/remove_zero_values_spec.rb +0 -25
- data/spec/smarter_csv/skip_lines_spec.rb +0 -29
- data/spec/smarter_csv/strings_as_keys_spec.rb +0 -24
- data/spec/smarter_csv/strip_chars_from_headers_spec.rb +0 -24
- data/spec/smarter_csv/valid_unicode_spec.rb +0 -94
- data/spec/smarter_csv/value_converters_spec.rb +0 -52
- data/spec/spec/spec_helper.rb +0 -17
- data/spec/spec.opts +0 -2
- data/spec/spec_helper.rb +0 -21
@@ -1,29 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'loads_csv_file_skipping_lines' do
|
7
|
-
options = {skip_lines: 3}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/skip_lines.csv", options)
|
9
|
-
data.size.should == 4
|
10
|
-
|
11
|
-
data.each do |item|
|
12
|
-
item.keys.each do |key|
|
13
|
-
[:first_name,:last_name,:dogs,:cats,:birds,:fish].should include(key)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
it 'loads_csv_with_user_defined_headers' do
|
19
|
-
options = {:skip_lines => 3, :headers_in_file => true, :user_provided_headers => [:a,:b,:c,:d,:e,:f]}
|
20
|
-
data = SmarterCSV.process("#{fixture_path}/skip_lines.csv", options)
|
21
|
-
data.size.should == 4
|
22
|
-
|
23
|
-
data.each do |item|
|
24
|
-
item.keys.each do |key|
|
25
|
-
[:a,:b,:c,:d,:e,:f].should include( key )
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'use_strings_as_keys' do
|
7
|
-
options = {:strings_as_keys => true}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
9
|
-
data.size.should == 5
|
10
|
-
# all the keys should be symbols
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == String}}
|
12
|
-
|
13
|
-
data.each do |item|
|
14
|
-
item.keys.each do |key|
|
15
|
-
["first_name", "last_name", "dogs", "cats", "birds", "fish"].should include( key )
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
data.each do |h|
|
20
|
-
h.size.should <= 6
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
it 'strip_whitespace_from_headers' do
|
7
|
-
options = {:strip_chars_from_headers => ' '}
|
8
|
-
data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
|
9
|
-
data.size.should == 5
|
10
|
-
# all the keys should be symbols
|
11
|
-
data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
|
12
|
-
|
13
|
-
data.each do |item|
|
14
|
-
item.keys.each do |key|
|
15
|
-
[:firstname, :lastname, :dogs, :cats, :birds, :fish].should include( key )
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
data.each do |h|
|
20
|
-
h.size.should <= 6
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
@@ -1,94 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
describe 'be_able_to' do
|
6
|
-
|
7
|
-
|
8
|
-
it 'loads file with unicode strings' do
|
9
|
-
options = {}
|
10
|
-
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
11
|
-
data.flatten.size.should == 4
|
12
|
-
data[0][:artist].should eq 'Кино'
|
13
|
-
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
14
|
-
data[0][:album].should eq 'Группа Крови'
|
15
|
-
data[0][:label].should eq 'Moroz Records'
|
16
|
-
data[0][:year].should eq 1998
|
17
|
-
|
18
|
-
data[0].should eq data[1]
|
19
|
-
|
20
|
-
data[2][:artist].should eq 'Rammstein'
|
21
|
-
data[2][:track].should eq 'Frühling in Paris'
|
22
|
-
data[2][:album].should eq 'Liebe ist für alle da'
|
23
|
-
data[2][:label].should eq 'Vagrant'
|
24
|
-
data[2][:year].should eq 2009
|
25
|
-
|
26
|
-
data[2].should eq data[3]
|
27
|
-
end
|
28
|
-
|
29
|
-
it 'loads file with unicode strings, when forcing utf8' do
|
30
|
-
options = {:force_utf8 => true}
|
31
|
-
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
32
|
-
data.flatten.size.should == 4
|
33
|
-
data[0][:artist].should eq 'Кино'
|
34
|
-
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
35
|
-
data[0][:album].should eq 'Группа Крови'
|
36
|
-
data[0][:label].should eq 'Moroz Records'
|
37
|
-
data[0][:year].should eq 1998
|
38
|
-
|
39
|
-
data[0].should eq data[1]
|
40
|
-
|
41
|
-
data[2][:artist].should eq 'Rammstein'
|
42
|
-
data[2][:track].should eq 'Frühling in Paris'
|
43
|
-
data[2][:album].should eq 'Liebe ist für alle da'
|
44
|
-
data[2][:label].should eq 'Vagrant'
|
45
|
-
data[2][:year].should eq 2009
|
46
|
-
|
47
|
-
data[2].should eq data[3]
|
48
|
-
end
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
it 'loads file with unicode strings, when loading from binary input' do
|
53
|
-
options = {:file_encoding => 'binary'}
|
54
|
-
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
55
|
-
data.flatten.size.should == 4
|
56
|
-
data[0][:artist].should eq 'Кино'
|
57
|
-
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
58
|
-
data[0][:album].should eq 'Группа Крови'
|
59
|
-
data[0][:label].should eq 'Moroz Records'
|
60
|
-
data[0][:year].should eq 1998
|
61
|
-
|
62
|
-
data[0].should eq data[1]
|
63
|
-
|
64
|
-
data[2][:artist].should eq 'Rammstein'
|
65
|
-
data[2][:track].should eq 'Frühling in Paris'
|
66
|
-
data[2][:album].should eq 'Liebe ist für alle da'
|
67
|
-
data[2][:label].should eq 'Vagrant'
|
68
|
-
data[2][:year].should eq 2009
|
69
|
-
|
70
|
-
data[2].should eq data[3]
|
71
|
-
end
|
72
|
-
|
73
|
-
it 'loads file with unicode strings, when forcing utf8 with binary input' do
|
74
|
-
options = {:file_encoding => 'binary', :force_utf8 => true}
|
75
|
-
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
76
|
-
data.flatten.size.should == 4
|
77
|
-
data[0][:artist].should eq 'Кино'
|
78
|
-
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
79
|
-
data[0][:album].should eq 'Группа Крови'
|
80
|
-
data[0][:label].should eq 'Moroz Records'
|
81
|
-
data[0][:year].should eq 1998
|
82
|
-
|
83
|
-
data[0].should eq data[1]
|
84
|
-
|
85
|
-
data[2][:artist].should eq 'Rammstein'
|
86
|
-
data[2][:track].should eq 'Frühling in Paris'
|
87
|
-
data[2][:album].should eq 'Liebe ist für alle da'
|
88
|
-
data[2][:label].should eq 'Vagrant'
|
89
|
-
data[2][:year].should eq 2009
|
90
|
-
|
91
|
-
data[2].should eq data[3]
|
92
|
-
end
|
93
|
-
|
94
|
-
end
|
@@ -1,52 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
fixture_path = 'spec/fixtures'
|
4
|
-
|
5
|
-
require 'date'
|
6
|
-
class DateConverter
|
7
|
-
def self.convert(value)
|
8
|
-
Date.strptime( value, '%m/%d/%Y')
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
class CurrencyConverter
|
13
|
-
def self.convert(value)
|
14
|
-
value.sub(/[$]/,'').to_f # would be nice to add a computed column :currency => '€'
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
describe 'be_able_to' do
|
19
|
-
it 'convert date values into Date instances' do
|
20
|
-
options = {:value_converters => {:date => DateConverter}}
|
21
|
-
data = SmarterCSV.process("#{fixture_path}/with_dates.csv", options)
|
22
|
-
data.flatten.size.should == 3
|
23
|
-
data[0][:date].class.should eq Date
|
24
|
-
data[0][:date].to_s.should eq "1998-10-30"
|
25
|
-
data[1][:date].to_s.should eq "2011-02-01"
|
26
|
-
data[2][:date].to_s.should eq "2013-01-09"
|
27
|
-
end
|
28
|
-
|
29
|
-
it 'converts dollar prices into float values' do
|
30
|
-
options = {:value_converters => {:price => CurrencyConverter}}
|
31
|
-
data = SmarterCSV.process("#{fixture_path}/money.csv", options)
|
32
|
-
data.flatten.size.should == 2
|
33
|
-
data[0][:price].class.should eq Float
|
34
|
-
data[0][:price].should eq 9.99
|
35
|
-
data[1][:price].should eq 14.99
|
36
|
-
end
|
37
|
-
|
38
|
-
it 'convert can use multiple value converters' do
|
39
|
-
options = {:value_converters => {:date => DateConverter, :price => CurrencyConverter}}
|
40
|
-
data = SmarterCSV.process("#{fixture_path}/with_dates.csv", options)
|
41
|
-
data.flatten.size.should == 3
|
42
|
-
data[0][:date].class.should eq Date
|
43
|
-
data[0][:date].to_s.should eq "1998-10-30"
|
44
|
-
data[1][:date].to_s.should eq "2011-02-01"
|
45
|
-
data[2][:date].to_s.should eq "2013-01-09"
|
46
|
-
|
47
|
-
data[0][:price].class.should eq Float
|
48
|
-
data[0][:price].should eq 44.50
|
49
|
-
data[1][:price].should eq 15.0
|
50
|
-
data[2][:price].should eq 0.11
|
51
|
-
end
|
52
|
-
end
|
data/spec/spec/spec_helper.rb
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
# This file was generated by the `rspec --init` command. Conventionally, all
|
2
|
-
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
3
|
-
# Require this file using `require "spec_helper"` to ensure that it is only
|
4
|
-
# loaded once.
|
5
|
-
#
|
6
|
-
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
7
|
-
RSpec.configure do |config|
|
8
|
-
config.treat_symbols_as_metadata_keys_with_true_values = true
|
9
|
-
config.run_all_when_everything_filtered = true
|
10
|
-
config.filter_run :focus
|
11
|
-
|
12
|
-
# Run specs in random order to surface order dependencies. If you find an
|
13
|
-
# order dependency and want to debug it, you can fix the order by providing
|
14
|
-
# the seed, which is printed after each run.
|
15
|
-
# --seed 1234
|
16
|
-
config.order = 'random'
|
17
|
-
end
|
data/spec/spec.opts
DELETED
data/spec/spec_helper.rb
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
require 'rubygems'
|
2
|
-
require 'bundler/setup'
|
3
|
-
|
4
|
-
Bundler.require(:default)
|
5
|
-
|
6
|
-
require 'smarter_csv'
|
7
|
-
|
8
|
-
|
9
|
-
RSpec.configure do |config|
|
10
|
-
config.treat_symbols_as_metadata_keys_with_true_values = true
|
11
|
-
config.filter_run :focus => true
|
12
|
-
config.run_all_when_everything_filtered = true
|
13
|
-
# config.fixture_path = 'spec/fixtures'
|
14
|
-
|
15
|
-
# config.mock_with :rr
|
16
|
-
# config.before(:each) do
|
17
|
-
# Project.delete_all
|
18
|
-
# Category.delete_all
|
19
|
-
# end
|
20
|
-
end
|
21
|
-
|