smarter_csv 1.1.3 → 1.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -0
- data/lib/extensions/hash.rb +1 -3
- data/lib/smarter_csv/smarter_csv.rb +2 -3
- data/lib/smarter_csv/version.rb +1 -1
- data/spec/fixtures/valid_unicode.csv +5 -0
- data/spec/smarter_csv/extenstions_spec.rb +17 -0
- data/spec/smarter_csv/quoted_spec.rb +11 -1
- data/spec/smarter_csv/valid_unicode_spec.rb +94 -0
- metadata +8 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: cb1216b85e197c77005a95ab4c3bc46896b7719f
|
|
4
|
+
data.tar.gz: 7539e858a39825e5fac1dc27e51f53f1e3f20c2c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2669d2f524e138bdcd8e9ded254a8dee1996589d56eabe4ac2f4480be7ebc88c1360600d26e109d3eba7e1e91075aa52629663ef4fd32489a7fd7e809f8b587c
|
|
7
|
+
data.tar.gz: 39cf42229ab96f15e860472ea138e04ba18c6c717e0860eaa87e9fd2e0c8ca516a8070ee6333f96a6cbc8a40662b509fef492eaefc72b95ecb1ccf5d8c1b1faa
|
data/README.md
CHANGED
|
@@ -293,6 +293,9 @@ Planned in the next releases:
|
|
|
293
293
|
|
|
294
294
|
## Changes
|
|
295
295
|
|
|
296
|
+
#### 1.1.4 (2017-01-16)
|
|
297
|
+
* fixing UTF-8 related bug which was introduced in 1.1.2 (thank to Tirdad C.)
|
|
298
|
+
|
|
296
299
|
#### 1.1.3 (2016-12-30)
|
|
297
300
|
* added warning when options indicate UTF-8 processing, but input filehandle is not opened with r:UTF-8 option
|
|
298
301
|
|
|
@@ -445,6 +448,7 @@ And a special thanks to those who contributed pull requests:
|
|
|
445
448
|
* [Jordan Graft](https://github.com/jordangraft)
|
|
446
449
|
* [Michael](https://github.com/polycarpou)
|
|
447
450
|
* [Kevin Coleman](https://github.com/KevinColemanInc)
|
|
451
|
+
* [Tirdad C.](https://github.com/tridadc)
|
|
448
452
|
|
|
449
453
|
|
|
450
454
|
## Contributing
|
data/lib/extensions/hash.rb
CHANGED
|
@@ -40,7 +40,7 @@ module SmarterCSV
|
|
|
40
40
|
# process the header line in the CSV file..
|
|
41
41
|
# the first line of a CSV file contains the header .. it might be commented out, so we need to read it anyhow
|
|
42
42
|
header = f.readline.sub(options[:comment_regexp],'').chomp(options[:row_sep])
|
|
43
|
-
header = header.
|
|
43
|
+
header = header.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence]) if options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
|
|
44
44
|
|
|
45
45
|
file_line_count += 1
|
|
46
46
|
csv_line_count += 1
|
|
@@ -104,10 +104,9 @@ module SmarterCSV
|
|
|
104
104
|
# now on to processing all the rest of the lines in the CSV file:
|
|
105
105
|
while ! f.eof? # we can't use f.readlines() here, because this would read the whole file into memory at once, and eof => true
|
|
106
106
|
line = f.readline # read one line.. this uses the input_record_separator $/ which we set previously!
|
|
107
|
-
line = line.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') if options[:force_utf8]
|
|
108
107
|
|
|
109
108
|
# replace invalid byte sequence in UTF-8 with question mark to avoid errors
|
|
110
|
-
line = line.
|
|
109
|
+
line = line.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence]) if options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
|
|
111
110
|
|
|
112
111
|
file_line_count += 1
|
|
113
112
|
csv_line_count += 1
|
data/lib/smarter_csv/version.rb
CHANGED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"Artist","Track","Album","Label","Year"
|
|
2
|
+
Кино,"Мама, мы все сошли с ума",Группа Крови,Moroz Records,1998
|
|
3
|
+
"Кино","Мама, мы все сошли с ума","Группа Крови","Moroz Records",1998
|
|
4
|
+
Rammstein,Frühling in Paris,Liebe ist für alle da,Vagrant,2009
|
|
5
|
+
"Rammstein","Frühling in Paris","Liebe ist für alle da","Vagrant",2009
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
describe "Hash.zip" do
|
|
4
|
+
it "constructs a new Hash from two Arrays" do
|
|
5
|
+
Hash.zip(["a", "b"], [1, 2]).should == { "a" => 1, "b" => 2 }
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
it "constructs an empty Hash if given no keys" do
|
|
9
|
+
Hash.zip([], []).should == {}
|
|
10
|
+
Hash.zip([], [1]).should == {}
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
it "uses nil values if there are more keys than values" do
|
|
14
|
+
Hash.zip(["a"], []).should == { "a" => nil }
|
|
15
|
+
Hash.zip(["a", "b"], [1]).should == { "a" => 1, "b" => nil }
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -3,11 +3,21 @@ require 'spec_helper'
|
|
|
3
3
|
fixture_path = 'spec/fixtures'
|
|
4
4
|
|
|
5
5
|
describe 'be_able_to' do
|
|
6
|
-
|
|
6
|
+
|
|
7
|
+
it 'loads_file_with_quoted_fields' do
|
|
7
8
|
options = {}
|
|
8
9
|
data = SmarterCSV.process("#{fixture_path}/quoted.csv", options)
|
|
9
10
|
data.flatten.size.should == 4
|
|
11
|
+
data[1][:model].should eq 'Venture "Extended Edition"'
|
|
10
12
|
data[1][:description].should be_nil
|
|
13
|
+
data[2][:model].should eq 'Venture "Extended Edition, Very Large"'
|
|
11
14
|
data[2][:description].should be_nil
|
|
15
|
+
data.each do |h|
|
|
16
|
+
h[:year].class.should eq Fixnum
|
|
17
|
+
h[:make].should_not be_nil
|
|
18
|
+
h[:model].should_not be_nil
|
|
19
|
+
h[:price].class.should eq Float
|
|
20
|
+
end
|
|
12
21
|
end
|
|
22
|
+
|
|
13
23
|
end
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
fixture_path = 'spec/fixtures'
|
|
4
|
+
|
|
5
|
+
describe 'be_able_to' do
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
it 'loads file with unicode strings' do
|
|
9
|
+
options = {}
|
|
10
|
+
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
|
11
|
+
data.flatten.size.should == 4
|
|
12
|
+
data[0][:artist].should eq 'Кино'
|
|
13
|
+
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
|
14
|
+
data[0][:album].should eq 'Группа Крови'
|
|
15
|
+
data[0][:label].should eq 'Moroz Records'
|
|
16
|
+
data[0][:year].should eq 1998
|
|
17
|
+
|
|
18
|
+
data[0].should eq data[1]
|
|
19
|
+
|
|
20
|
+
data[2][:artist].should eq 'Rammstein'
|
|
21
|
+
data[2][:track].should eq 'Frühling in Paris'
|
|
22
|
+
data[2][:album].should eq 'Liebe ist für alle da'
|
|
23
|
+
data[2][:label].should eq 'Vagrant'
|
|
24
|
+
data[2][:year].should eq 2009
|
|
25
|
+
|
|
26
|
+
data[2].should eq data[3]
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
it 'loads file with unicode strings, when forcing utf8' do
|
|
30
|
+
options = {:force_utf8 => true}
|
|
31
|
+
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
|
32
|
+
data.flatten.size.should == 4
|
|
33
|
+
data[0][:artist].should eq 'Кино'
|
|
34
|
+
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
|
35
|
+
data[0][:album].should eq 'Группа Крови'
|
|
36
|
+
data[0][:label].should eq 'Moroz Records'
|
|
37
|
+
data[0][:year].should eq 1998
|
|
38
|
+
|
|
39
|
+
data[0].should eq data[1]
|
|
40
|
+
|
|
41
|
+
data[2][:artist].should eq 'Rammstein'
|
|
42
|
+
data[2][:track].should eq 'Frühling in Paris'
|
|
43
|
+
data[2][:album].should eq 'Liebe ist für alle da'
|
|
44
|
+
data[2][:label].should eq 'Vagrant'
|
|
45
|
+
data[2][:year].should eq 2009
|
|
46
|
+
|
|
47
|
+
data[2].should eq data[3]
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
it 'loads file with unicode strings, when loading from binary input' do
|
|
53
|
+
options = {:file_encoding => 'binary'}
|
|
54
|
+
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
|
55
|
+
data.flatten.size.should == 4
|
|
56
|
+
data[0][:artist].should eq 'Кино'
|
|
57
|
+
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
|
58
|
+
data[0][:album].should eq 'Группа Крови'
|
|
59
|
+
data[0][:label].should eq 'Moroz Records'
|
|
60
|
+
data[0][:year].should eq 1998
|
|
61
|
+
|
|
62
|
+
data[0].should eq data[1]
|
|
63
|
+
|
|
64
|
+
data[2][:artist].should eq 'Rammstein'
|
|
65
|
+
data[2][:track].should eq 'Frühling in Paris'
|
|
66
|
+
data[2][:album].should eq 'Liebe ist für alle da'
|
|
67
|
+
data[2][:label].should eq 'Vagrant'
|
|
68
|
+
data[2][:year].should eq 2009
|
|
69
|
+
|
|
70
|
+
data[2].should eq data[3]
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
it 'loads file with unicode strings, when forcing utf8 with binary input' do
|
|
74
|
+
options = {:file_encoding => 'binary', :force_utf8 => true}
|
|
75
|
+
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
|
76
|
+
data.flatten.size.should == 4
|
|
77
|
+
data[0][:artist].should eq 'Кино'
|
|
78
|
+
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
|
79
|
+
data[0][:album].should eq 'Группа Крови'
|
|
80
|
+
data[0][:label].should eq 'Moroz Records'
|
|
81
|
+
data[0][:year].should eq 1998
|
|
82
|
+
|
|
83
|
+
data[0].should eq data[1]
|
|
84
|
+
|
|
85
|
+
data[2][:artist].should eq 'Rammstein'
|
|
86
|
+
data[2][:track].should eq 'Frühling in Paris'
|
|
87
|
+
data[2][:album].should eq 'Liebe ist für alle da'
|
|
88
|
+
data[2][:label].should eq 'Vagrant'
|
|
89
|
+
data[2][:year].should eq 2009
|
|
90
|
+
|
|
91
|
+
data[2].should eq data[3]
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: smarter_csv
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.1.
|
|
4
|
+
version: 1.1.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- |
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date:
|
|
12
|
+
date: 2017-01-17 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: rspec
|
|
@@ -68,6 +68,7 @@ files:
|
|
|
68
68
|
- spec/fixtures/quoted.csv
|
|
69
69
|
- spec/fixtures/separator.csv
|
|
70
70
|
- spec/fixtures/skip_lines.csv
|
|
71
|
+
- spec/fixtures/valid_unicode.csv
|
|
71
72
|
- spec/fixtures/with_dashes.csv
|
|
72
73
|
- spec/fixtures/with_dates.csv
|
|
73
74
|
- spec/smarter_csv/binary_file2_spec.rb
|
|
@@ -77,6 +78,7 @@ files:
|
|
|
77
78
|
- spec/smarter_csv/close_file_spec.rb
|
|
78
79
|
- spec/smarter_csv/column_separator_spec.rb
|
|
79
80
|
- spec/smarter_csv/convert_values_to_numeric_spec.rb
|
|
81
|
+
- spec/smarter_csv/extenstions_spec.rb
|
|
80
82
|
- spec/smarter_csv/header_transformation_spec.rb
|
|
81
83
|
- spec/smarter_csv/keep_headers_spec.rb
|
|
82
84
|
- spec/smarter_csv/key_mapping_spec.rb
|
|
@@ -94,6 +96,7 @@ files:
|
|
|
94
96
|
- spec/smarter_csv/skip_lines_spec.rb
|
|
95
97
|
- spec/smarter_csv/strings_as_keys_spec.rb
|
|
96
98
|
- spec/smarter_csv/strip_chars_from_headers_spec.rb
|
|
99
|
+
- spec/smarter_csv/valid_unicode_spec.rb
|
|
97
100
|
- spec/smarter_csv/value_converters_spec.rb
|
|
98
101
|
- spec/spec.opts
|
|
99
102
|
- spec/spec/spec_helper.rb
|
|
@@ -147,6 +150,7 @@ test_files:
|
|
|
147
150
|
- spec/fixtures/quoted.csv
|
|
148
151
|
- spec/fixtures/separator.csv
|
|
149
152
|
- spec/fixtures/skip_lines.csv
|
|
153
|
+
- spec/fixtures/valid_unicode.csv
|
|
150
154
|
- spec/fixtures/with_dashes.csv
|
|
151
155
|
- spec/fixtures/with_dates.csv
|
|
152
156
|
- spec/smarter_csv/binary_file2_spec.rb
|
|
@@ -156,6 +160,7 @@ test_files:
|
|
|
156
160
|
- spec/smarter_csv/close_file_spec.rb
|
|
157
161
|
- spec/smarter_csv/column_separator_spec.rb
|
|
158
162
|
- spec/smarter_csv/convert_values_to_numeric_spec.rb
|
|
163
|
+
- spec/smarter_csv/extenstions_spec.rb
|
|
159
164
|
- spec/smarter_csv/header_transformation_spec.rb
|
|
160
165
|
- spec/smarter_csv/keep_headers_spec.rb
|
|
161
166
|
- spec/smarter_csv/key_mapping_spec.rb
|
|
@@ -173,6 +178,7 @@ test_files:
|
|
|
173
178
|
- spec/smarter_csv/skip_lines_spec.rb
|
|
174
179
|
- spec/smarter_csv/strings_as_keys_spec.rb
|
|
175
180
|
- spec/smarter_csv/strip_chars_from_headers_spec.rb
|
|
181
|
+
- spec/smarter_csv/valid_unicode_spec.rb
|
|
176
182
|
- spec/smarter_csv/value_converters_spec.rb
|
|
177
183
|
- spec/spec.opts
|
|
178
184
|
- spec/spec/spec_helper.rb
|