smarter_csv 1.1.3 → 1.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +4 -0
- data/lib/extensions/hash.rb +1 -3
- data/lib/smarter_csv/smarter_csv.rb +2 -3
- data/lib/smarter_csv/version.rb +1 -1
- data/spec/fixtures/valid_unicode.csv +5 -0
- data/spec/smarter_csv/extenstions_spec.rb +17 -0
- data/spec/smarter_csv/quoted_spec.rb +11 -1
- data/spec/smarter_csv/valid_unicode_spec.rb +94 -0
- metadata +8 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: cb1216b85e197c77005a95ab4c3bc46896b7719f
|
|
4
|
+
data.tar.gz: 7539e858a39825e5fac1dc27e51f53f1e3f20c2c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2669d2f524e138bdcd8e9ded254a8dee1996589d56eabe4ac2f4480be7ebc88c1360600d26e109d3eba7e1e91075aa52629663ef4fd32489a7fd7e809f8b587c
|
|
7
|
+
data.tar.gz: 39cf42229ab96f15e860472ea138e04ba18c6c717e0860eaa87e9fd2e0c8ca516a8070ee6333f96a6cbc8a40662b509fef492eaefc72b95ecb1ccf5d8c1b1faa
|
data/README.md
CHANGED
|
@@ -293,6 +293,9 @@ Planned in the next releases:
|
|
|
293
293
|
|
|
294
294
|
## Changes
|
|
295
295
|
|
|
296
|
+
#### 1.1.4 (2017-01-16)
|
|
297
|
+
* fixing UTF-8 related bug which was introduced in 1.1.2 (thank to Tirdad C.)
|
|
298
|
+
|
|
296
299
|
#### 1.1.3 (2016-12-30)
|
|
297
300
|
* added warning when options indicate UTF-8 processing, but input filehandle is not opened with r:UTF-8 option
|
|
298
301
|
|
|
@@ -445,6 +448,7 @@ And a special thanks to those who contributed pull requests:
|
|
|
445
448
|
* [Jordan Graft](https://github.com/jordangraft)
|
|
446
449
|
* [Michael](https://github.com/polycarpou)
|
|
447
450
|
* [Kevin Coleman](https://github.com/KevinColemanInc)
|
|
451
|
+
* [Tirdad C.](https://github.com/tridadc)
|
|
448
452
|
|
|
449
453
|
|
|
450
454
|
## Contributing
|
data/lib/extensions/hash.rb
CHANGED
|
@@ -40,7 +40,7 @@ module SmarterCSV
|
|
|
40
40
|
# process the header line in the CSV file..
|
|
41
41
|
# the first line of a CSV file contains the header .. it might be commented out, so we need to read it anyhow
|
|
42
42
|
header = f.readline.sub(options[:comment_regexp],'').chomp(options[:row_sep])
|
|
43
|
-
header = header.
|
|
43
|
+
header = header.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence]) if options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
|
|
44
44
|
|
|
45
45
|
file_line_count += 1
|
|
46
46
|
csv_line_count += 1
|
|
@@ -104,10 +104,9 @@ module SmarterCSV
|
|
|
104
104
|
# now on to processing all the rest of the lines in the CSV file:
|
|
105
105
|
while ! f.eof? # we can't use f.readlines() here, because this would read the whole file into memory at once, and eof => true
|
|
106
106
|
line = f.readline # read one line.. this uses the input_record_separator $/ which we set previously!
|
|
107
|
-
line = line.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') if options[:force_utf8]
|
|
108
107
|
|
|
109
108
|
# replace invalid byte sequence in UTF-8 with question mark to avoid errors
|
|
110
|
-
line = line.
|
|
109
|
+
line = line.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence]) if options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
|
|
111
110
|
|
|
112
111
|
file_line_count += 1
|
|
113
112
|
csv_line_count += 1
|
data/lib/smarter_csv/version.rb
CHANGED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"Artist","Track","Album","Label","Year"
|
|
2
|
+
Кино,"Мама, мы все сошли с ума",Группа Крови,Moroz Records,1998
|
|
3
|
+
"Кино","Мама, мы все сошли с ума","Группа Крови","Moroz Records",1998
|
|
4
|
+
Rammstein,Frühling in Paris,Liebe ist für alle da,Vagrant,2009
|
|
5
|
+
"Rammstein","Frühling in Paris","Liebe ist für alle da","Vagrant",2009
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
describe "Hash.zip" do
|
|
4
|
+
it "constructs a new Hash from two Arrays" do
|
|
5
|
+
Hash.zip(["a", "b"], [1, 2]).should == { "a" => 1, "b" => 2 }
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
it "constructs an empty Hash if given no keys" do
|
|
9
|
+
Hash.zip([], []).should == {}
|
|
10
|
+
Hash.zip([], [1]).should == {}
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
it "uses nil values if there are more keys than values" do
|
|
14
|
+
Hash.zip(["a"], []).should == { "a" => nil }
|
|
15
|
+
Hash.zip(["a", "b"], [1]).should == { "a" => 1, "b" => nil }
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -3,11 +3,21 @@ require 'spec_helper'
|
|
|
3
3
|
fixture_path = 'spec/fixtures'
|
|
4
4
|
|
|
5
5
|
describe 'be_able_to' do
|
|
6
|
-
|
|
6
|
+
|
|
7
|
+
it 'loads_file_with_quoted_fields' do
|
|
7
8
|
options = {}
|
|
8
9
|
data = SmarterCSV.process("#{fixture_path}/quoted.csv", options)
|
|
9
10
|
data.flatten.size.should == 4
|
|
11
|
+
data[1][:model].should eq 'Venture "Extended Edition"'
|
|
10
12
|
data[1][:description].should be_nil
|
|
13
|
+
data[2][:model].should eq 'Venture "Extended Edition, Very Large"'
|
|
11
14
|
data[2][:description].should be_nil
|
|
15
|
+
data.each do |h|
|
|
16
|
+
h[:year].class.should eq Fixnum
|
|
17
|
+
h[:make].should_not be_nil
|
|
18
|
+
h[:model].should_not be_nil
|
|
19
|
+
h[:price].class.should eq Float
|
|
20
|
+
end
|
|
12
21
|
end
|
|
22
|
+
|
|
13
23
|
end
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
fixture_path = 'spec/fixtures'
|
|
4
|
+
|
|
5
|
+
describe 'be_able_to' do
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
it 'loads file with unicode strings' do
|
|
9
|
+
options = {}
|
|
10
|
+
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
|
11
|
+
data.flatten.size.should == 4
|
|
12
|
+
data[0][:artist].should eq 'Кино'
|
|
13
|
+
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
|
14
|
+
data[0][:album].should eq 'Группа Крови'
|
|
15
|
+
data[0][:label].should eq 'Moroz Records'
|
|
16
|
+
data[0][:year].should eq 1998
|
|
17
|
+
|
|
18
|
+
data[0].should eq data[1]
|
|
19
|
+
|
|
20
|
+
data[2][:artist].should eq 'Rammstein'
|
|
21
|
+
data[2][:track].should eq 'Frühling in Paris'
|
|
22
|
+
data[2][:album].should eq 'Liebe ist für alle da'
|
|
23
|
+
data[2][:label].should eq 'Vagrant'
|
|
24
|
+
data[2][:year].should eq 2009
|
|
25
|
+
|
|
26
|
+
data[2].should eq data[3]
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
it 'loads file with unicode strings, when forcing utf8' do
|
|
30
|
+
options = {:force_utf8 => true}
|
|
31
|
+
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
|
32
|
+
data.flatten.size.should == 4
|
|
33
|
+
data[0][:artist].should eq 'Кино'
|
|
34
|
+
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
|
35
|
+
data[0][:album].should eq 'Группа Крови'
|
|
36
|
+
data[0][:label].should eq 'Moroz Records'
|
|
37
|
+
data[0][:year].should eq 1998
|
|
38
|
+
|
|
39
|
+
data[0].should eq data[1]
|
|
40
|
+
|
|
41
|
+
data[2][:artist].should eq 'Rammstein'
|
|
42
|
+
data[2][:track].should eq 'Frühling in Paris'
|
|
43
|
+
data[2][:album].should eq 'Liebe ist für alle da'
|
|
44
|
+
data[2][:label].should eq 'Vagrant'
|
|
45
|
+
data[2][:year].should eq 2009
|
|
46
|
+
|
|
47
|
+
data[2].should eq data[3]
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
it 'loads file with unicode strings, when loading from binary input' do
|
|
53
|
+
options = {:file_encoding => 'binary'}
|
|
54
|
+
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
|
55
|
+
data.flatten.size.should == 4
|
|
56
|
+
data[0][:artist].should eq 'Кино'
|
|
57
|
+
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
|
58
|
+
data[0][:album].should eq 'Группа Крови'
|
|
59
|
+
data[0][:label].should eq 'Moroz Records'
|
|
60
|
+
data[0][:year].should eq 1998
|
|
61
|
+
|
|
62
|
+
data[0].should eq data[1]
|
|
63
|
+
|
|
64
|
+
data[2][:artist].should eq 'Rammstein'
|
|
65
|
+
data[2][:track].should eq 'Frühling in Paris'
|
|
66
|
+
data[2][:album].should eq 'Liebe ist für alle da'
|
|
67
|
+
data[2][:label].should eq 'Vagrant'
|
|
68
|
+
data[2][:year].should eq 2009
|
|
69
|
+
|
|
70
|
+
data[2].should eq data[3]
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
it 'loads file with unicode strings, when forcing utf8 with binary input' do
|
|
74
|
+
options = {:file_encoding => 'binary', :force_utf8 => true}
|
|
75
|
+
data = SmarterCSV.process("#{fixture_path}/valid_unicode.csv", options)
|
|
76
|
+
data.flatten.size.should == 4
|
|
77
|
+
data[0][:artist].should eq 'Кино'
|
|
78
|
+
data[0][:track].should eq 'Мама, мы все сошли с ума'
|
|
79
|
+
data[0][:album].should eq 'Группа Крови'
|
|
80
|
+
data[0][:label].should eq 'Moroz Records'
|
|
81
|
+
data[0][:year].should eq 1998
|
|
82
|
+
|
|
83
|
+
data[0].should eq data[1]
|
|
84
|
+
|
|
85
|
+
data[2][:artist].should eq 'Rammstein'
|
|
86
|
+
data[2][:track].should eq 'Frühling in Paris'
|
|
87
|
+
data[2][:album].should eq 'Liebe ist für alle da'
|
|
88
|
+
data[2][:label].should eq 'Vagrant'
|
|
89
|
+
data[2][:year].should eq 2009
|
|
90
|
+
|
|
91
|
+
data[2].should eq data[3]
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: smarter_csv
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.1.
|
|
4
|
+
version: 1.1.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- |
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date:
|
|
12
|
+
date: 2017-01-17 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: rspec
|
|
@@ -68,6 +68,7 @@ files:
|
|
|
68
68
|
- spec/fixtures/quoted.csv
|
|
69
69
|
- spec/fixtures/separator.csv
|
|
70
70
|
- spec/fixtures/skip_lines.csv
|
|
71
|
+
- spec/fixtures/valid_unicode.csv
|
|
71
72
|
- spec/fixtures/with_dashes.csv
|
|
72
73
|
- spec/fixtures/with_dates.csv
|
|
73
74
|
- spec/smarter_csv/binary_file2_spec.rb
|
|
@@ -77,6 +78,7 @@ files:
|
|
|
77
78
|
- spec/smarter_csv/close_file_spec.rb
|
|
78
79
|
- spec/smarter_csv/column_separator_spec.rb
|
|
79
80
|
- spec/smarter_csv/convert_values_to_numeric_spec.rb
|
|
81
|
+
- spec/smarter_csv/extenstions_spec.rb
|
|
80
82
|
- spec/smarter_csv/header_transformation_spec.rb
|
|
81
83
|
- spec/smarter_csv/keep_headers_spec.rb
|
|
82
84
|
- spec/smarter_csv/key_mapping_spec.rb
|
|
@@ -94,6 +96,7 @@ files:
|
|
|
94
96
|
- spec/smarter_csv/skip_lines_spec.rb
|
|
95
97
|
- spec/smarter_csv/strings_as_keys_spec.rb
|
|
96
98
|
- spec/smarter_csv/strip_chars_from_headers_spec.rb
|
|
99
|
+
- spec/smarter_csv/valid_unicode_spec.rb
|
|
97
100
|
- spec/smarter_csv/value_converters_spec.rb
|
|
98
101
|
- spec/spec.opts
|
|
99
102
|
- spec/spec/spec_helper.rb
|
|
@@ -147,6 +150,7 @@ test_files:
|
|
|
147
150
|
- spec/fixtures/quoted.csv
|
|
148
151
|
- spec/fixtures/separator.csv
|
|
149
152
|
- spec/fixtures/skip_lines.csv
|
|
153
|
+
- spec/fixtures/valid_unicode.csv
|
|
150
154
|
- spec/fixtures/with_dashes.csv
|
|
151
155
|
- spec/fixtures/with_dates.csv
|
|
152
156
|
- spec/smarter_csv/binary_file2_spec.rb
|
|
@@ -156,6 +160,7 @@ test_files:
|
|
|
156
160
|
- spec/smarter_csv/close_file_spec.rb
|
|
157
161
|
- spec/smarter_csv/column_separator_spec.rb
|
|
158
162
|
- spec/smarter_csv/convert_values_to_numeric_spec.rb
|
|
163
|
+
- spec/smarter_csv/extenstions_spec.rb
|
|
159
164
|
- spec/smarter_csv/header_transformation_spec.rb
|
|
160
165
|
- spec/smarter_csv/keep_headers_spec.rb
|
|
161
166
|
- spec/smarter_csv/key_mapping_spec.rb
|
|
@@ -173,6 +178,7 @@ test_files:
|
|
|
173
178
|
- spec/smarter_csv/skip_lines_spec.rb
|
|
174
179
|
- spec/smarter_csv/strings_as_keys_spec.rb
|
|
175
180
|
- spec/smarter_csv/strip_chars_from_headers_spec.rb
|
|
181
|
+
- spec/smarter_csv/valid_unicode_spec.rb
|
|
176
182
|
- spec/smarter_csv/value_converters_spec.rb
|
|
177
183
|
- spec/spec.opts
|
|
178
184
|
- spec/spec/spec_helper.rb
|