iostreams 0.16.0 → 0.16.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/io_streams/io_streams.rb +9 -4
- data/lib/io_streams/line/reader.rb +7 -5
- data/lib/io_streams/version.rb +1 -1
- data/test/encode_reader_test.rb +3 -3
- data/test/io_streams_test.rb +24 -1
- data/test/line_reader_test.rb +12 -6
- data/test/row_reader_test.rb +4 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 71dc5d69019340a1bcaf15bb7b79c46d136caee788c78ababcb1adbec55ccaae
|
4
|
+
data.tar.gz: 215a9d22d27fa102529a54b1886d54caad022289cb7bc62f0311e799fd2ead2a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 534218bfa084bcb43af55192210e08de829044ee316aa4043c9f06b0e97a6b484b4eb666821327f4fb79877383d10819f58570d7194420a00682a6e3e860d437
|
7
|
+
data.tar.gz: 0e8445aabb3cc8ae7a7f36859c12a724e8b4db966e97d45589ddd3ed1cc9fd30e9c2d2c29c321ee35ca2c4487f72004242ee2aa7d8425ac2061a792434e9fc66
|
@@ -316,8 +316,7 @@ module IOStreams
|
|
316
316
|
|
317
317
|
# Iterate over a file / stream returning each record/line one at a time.
|
318
318
|
def self.line_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
|
319
|
-
return yield(file_name_or_io) if file_name_or_io.is_a?(IOStreams::Line::Reader) ||
|
320
|
-
file_name_or_io.is_a?(Array)
|
319
|
+
return yield(file_name_or_io) if file_name_or_io.is_a?(IOStreams::Line::Reader) || file_name_or_io.is_a?(Array)
|
321
320
|
|
322
321
|
reader(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace) do |io|
|
323
322
|
IOStreams::Line::Reader.open(io, **args, &block)
|
@@ -427,8 +426,14 @@ module IOStreams
|
|
427
426
|
|
428
427
|
streams = streams_for_file_name(file_name) if streams.nil? && file_name
|
429
428
|
|
430
|
-
# Shortcut for when it is already a stream
|
431
|
-
|
429
|
+
# Shortcut for when it is already a stream
|
430
|
+
if !file_name_or_io.is_a?(String) && (streams.nil? || streams.empty?)
|
431
|
+
if encoding || encode_cleaner || encode_replace
|
432
|
+
return IOStreams::Encode::Reader.open(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace, &block)
|
433
|
+
else
|
434
|
+
return block.call(file_name_or_io)
|
435
|
+
end
|
436
|
+
end
|
432
437
|
|
433
438
|
if streams.nil?
|
434
439
|
streams = file_name_or_io.is_a?(String) ? streams_for_file_name(file_name_or_io) : [nil]
|
@@ -61,7 +61,7 @@ module IOStreams
|
|
61
61
|
# Auto-detect windows/linux line endings if not supplied. \n or \r\n
|
62
62
|
@delimiter = delimiter || auto_detect_line_endings
|
63
63
|
|
64
|
-
|
64
|
+
if @buffer
|
65
65
|
# Change the delimiters encoding to match that of the input stream
|
66
66
|
@delimiter = @delimiter.encode(@buffer.encoding)
|
67
67
|
@delimiter_size = @delimiter.size
|
@@ -73,7 +73,10 @@ module IOStreams
|
|
73
73
|
# Note:
|
74
74
|
# * The line delimiter is _not_ returned.
|
75
75
|
def each
|
76
|
-
|
76
|
+
until eof?
|
77
|
+
line = readline
|
78
|
+
yield(line) unless line.nil?
|
79
|
+
end
|
77
80
|
line_count
|
78
81
|
end
|
79
82
|
|
@@ -88,6 +91,7 @@ module IOStreams
|
|
88
91
|
if index
|
89
92
|
data = @buffer.slice(0, index)
|
90
93
|
@buffer = @buffer.slice(index + @delimiter_size, @buffer.size)
|
94
|
+
@line_count += 1
|
91
95
|
elsif @eof && @buffer.empty?
|
92
96
|
data = nil
|
93
97
|
@buffer = nil
|
@@ -95,9 +99,9 @@ module IOStreams
|
|
95
99
|
# Last line without delimiter
|
96
100
|
data = @buffer
|
97
101
|
@buffer = nil
|
102
|
+
@line_count += 1
|
98
103
|
end
|
99
104
|
|
100
|
-
@line_count += 1
|
101
105
|
data
|
102
106
|
end
|
103
107
|
|
@@ -130,8 +134,6 @@ module IOStreams
|
|
130
134
|
if block.nil?
|
131
135
|
@eof = true
|
132
136
|
return false
|
133
|
-
elsif block.size < @buffer_size
|
134
|
-
@eof = true
|
135
137
|
end
|
136
138
|
|
137
139
|
if @buffer
|
data/lib/io_streams/version.rb
CHANGED
data/test/encode_reader_test.rb
CHANGED
@@ -11,7 +11,7 @@ class EncodeReaderTest < Minitest::Test
|
|
11
11
|
end
|
12
12
|
|
13
13
|
let :cleansed_data do
|
14
|
-
bad_data.gsub("\xE9".b, '
|
14
|
+
bad_data.gsub("\xE9".b, '')
|
15
15
|
end
|
16
16
|
|
17
17
|
let :stripped_data do
|
@@ -32,7 +32,7 @@ class EncodeReaderTest < Minitest::Test
|
|
32
32
|
it 'strips invalid characters' do
|
33
33
|
input = StringIO.new(bad_data)
|
34
34
|
data =
|
35
|
-
IOStreams::Encode::Reader.open(input, encoding: 'UTF-8', encode_replace: '
|
35
|
+
IOStreams::Encode::Reader.open(input, encoding: 'UTF-8', encode_replace: '') do |io|
|
36
36
|
io.read
|
37
37
|
end
|
38
38
|
assert_equal cleansed_data, data
|
@@ -43,7 +43,7 @@ class EncodeReaderTest < Minitest::Test
|
|
43
43
|
it 'strips non-printable characters' do
|
44
44
|
input = StringIO.new(bad_data)
|
45
45
|
data =
|
46
|
-
IOStreams::Encode::Reader.open(input, encoding: 'UTF-8', encode_cleaner: :printable, encode_replace: '
|
46
|
+
IOStreams::Encode::Reader.open(input, encoding: 'UTF-8', encode_cleaner: :printable, encode_replace: '') do |io|
|
47
47
|
io.read
|
48
48
|
end
|
49
49
|
assert_equal stripped_data, data
|
data/test/io_streams_test.rb
CHANGED
@@ -18,6 +18,18 @@ class IOStreamsTest < Minitest::Test
|
|
18
18
|
temp_file.path
|
19
19
|
end
|
20
20
|
|
21
|
+
let :bad_data do
|
22
|
+
[
|
23
|
+
"New M\xE9xico,NE".b,
|
24
|
+
'good line',
|
25
|
+
"New M\xE9xico,\x07SF".b
|
26
|
+
].join("\n").encode('BINARY')
|
27
|
+
end
|
28
|
+
|
29
|
+
let :stripped_data do
|
30
|
+
bad_data.gsub("\xE9".b, '').gsub("\x07", '')
|
31
|
+
end
|
32
|
+
|
21
33
|
after do
|
22
34
|
temp_file.delete
|
23
35
|
end
|
@@ -86,8 +98,19 @@ class IOStreamsTest < Minitest::Test
|
|
86
98
|
describe '.each_line' do
|
87
99
|
it 'returns a line at a time' do
|
88
100
|
lines = []
|
89
|
-
IOStreams.each_line(source_file_name) { |line| lines << line }
|
101
|
+
count = IOStreams.each_line(source_file_name) { |line| lines << line }
|
90
102
|
assert_equal data.lines.map(&:strip), lines
|
103
|
+
assert_equal data.lines.count, count
|
104
|
+
end
|
105
|
+
|
106
|
+
it 'strips non-printable characters' do
|
107
|
+
input = StringIO.new(bad_data)
|
108
|
+
lines = []
|
109
|
+
count = IOStreams.each_line(input, encoding: 'UTF-8', encode_cleaner: :printable, encode_replace: '') do |line|
|
110
|
+
lines << line
|
111
|
+
end
|
112
|
+
assert_equal stripped_data.lines.map(&:strip), lines
|
113
|
+
assert_equal stripped_data.lines.count, count
|
91
114
|
end
|
92
115
|
end
|
93
116
|
|
data/test/line_reader_test.rb
CHANGED
@@ -19,39 +19,43 @@ class LineReaderTest < Minitest::Test
|
|
19
19
|
describe '#each' do
|
20
20
|
it 'each_line file' do
|
21
21
|
lines = []
|
22
|
-
IOStreams::Line::Reader.open(file_name) do |io|
|
22
|
+
count = IOStreams::Line::Reader.open(file_name) do |io|
|
23
23
|
io.each { |line| lines << line }
|
24
24
|
end
|
25
25
|
assert_equal data, lines
|
26
|
+
assert_equal data.size, count
|
26
27
|
end
|
27
28
|
|
28
29
|
it 'each_line stream' do
|
29
30
|
lines = []
|
30
|
-
File.open(file_name) do |file|
|
31
|
+
count = File.open(file_name) do |file|
|
31
32
|
IOStreams::Line::Reader.open(file) do |io|
|
32
33
|
io.each { |line| lines << line }
|
33
34
|
end
|
34
35
|
end
|
35
36
|
assert_equal data, lines
|
37
|
+
assert_equal data.size, count
|
36
38
|
end
|
37
39
|
|
38
40
|
["\r\n", "\n", "\r"].each do |delimiter|
|
39
41
|
it "autodetect delimiter: #{delimiter.inspect}" do
|
40
42
|
lines = []
|
41
43
|
stream = StringIO.new(data.join(delimiter))
|
42
|
-
IOStreams::Line::Reader.open(stream, buffer_size: 15) do |io|
|
44
|
+
count = IOStreams::Line::Reader.open(stream, buffer_size: 15) do |io|
|
43
45
|
io.each { |line| lines << line }
|
44
46
|
end
|
45
47
|
assert_equal data, lines
|
48
|
+
assert_equal data.size, count
|
46
49
|
end
|
47
50
|
|
48
51
|
it "single read autodetect delimiter: #{delimiter.inspect}" do
|
49
52
|
lines = []
|
50
53
|
stream = StringIO.new(data.join(delimiter))
|
51
|
-
IOStreams::Line::Reader.open(stream) do |io|
|
54
|
+
count = IOStreams::Line::Reader.open(stream) do |io|
|
52
55
|
io.each { |line| lines << line }
|
53
56
|
end
|
54
57
|
assert_equal data, lines
|
58
|
+
assert_equal data.size, count
|
55
59
|
end
|
56
60
|
end
|
57
61
|
|
@@ -59,10 +63,11 @@ class LineReaderTest < Minitest::Test
|
|
59
63
|
it "reads delimited #{delimiter.inspect}" do
|
60
64
|
lines = []
|
61
65
|
stream = StringIO.new(data.join(delimiter))
|
62
|
-
IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
|
66
|
+
count = IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
|
63
67
|
io.each { |line| lines << line }
|
64
68
|
end
|
65
69
|
assert_equal data, lines
|
70
|
+
assert_equal data.size, count
|
66
71
|
end
|
67
72
|
end
|
68
73
|
|
@@ -70,10 +75,11 @@ class LineReaderTest < Minitest::Test
|
|
70
75
|
delimiter = "\x01"
|
71
76
|
lines = []
|
72
77
|
stream = StringIO.new(data.join(delimiter).encode('ASCII-8BIT'))
|
73
|
-
IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
|
78
|
+
count = IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
|
74
79
|
io.each { |line| lines << line }
|
75
80
|
end
|
76
81
|
assert_equal data, lines
|
82
|
+
assert_equal data.size, count
|
77
83
|
end
|
78
84
|
|
79
85
|
describe '#readline' do
|
data/test/row_reader_test.rb
CHANGED
@@ -13,20 +13,22 @@ class RowReaderTest < Minitest::Test
|
|
13
13
|
describe '.open' do
|
14
14
|
it 'file' do
|
15
15
|
rows = []
|
16
|
-
IOStreams::Row::Reader.open(file_name) do |io|
|
16
|
+
count = IOStreams::Row::Reader.open(file_name) do |io|
|
17
17
|
io.each { |row| rows << row }
|
18
18
|
end
|
19
19
|
assert_equal expected, rows
|
20
|
+
assert_equal expected.size, count
|
20
21
|
end
|
21
22
|
|
22
23
|
it 'stream' do
|
23
24
|
rows = []
|
24
|
-
IOStreams.line_reader(file_name) do |file|
|
25
|
+
count = IOStreams.line_reader(file_name) do |file|
|
25
26
|
IOStreams::Row::Reader.open(file) do |io|
|
26
27
|
io.each { |row| rows << row }
|
27
28
|
end
|
28
29
|
end
|
29
30
|
assert_equal expected, rows
|
31
|
+
assert_equal expected.size, count
|
30
32
|
end
|
31
33
|
end
|
32
34
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: iostreams
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.16.
|
4
|
+
version: 0.16.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Reid Morrison
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-11-
|
11
|
+
date: 2018-11-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|