iostreams 0.16.0 → 0.16.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9392af579f83fb26f1ea2cfeefae497362c1232ff1226adb4220cc54805b93c2
-  data.tar.gz: 8db9dd03113c32ebe44b54610509dc86626f44deb6900fae65ac7cdd34a3abee
+  metadata.gz: 71dc5d69019340a1bcaf15bb7b79c46d136caee788c78ababcb1adbec55ccaae
+  data.tar.gz: 215a9d22d27fa102529a54b1886d54caad022289cb7bc62f0311e799fd2ead2a
 SHA512:
-  metadata.gz: 2810231db04f7a98086bb47a7657adb7888dc3f16e03f48e9824170bf36a8df65ca006cfbcaca7360e42f609f5444be189fa24ba1ae4610c5c727f3c18c6bb85
-  data.tar.gz: 2e247fb89ad090d96cd6e0b20aaf9afd0b71cbbef7dc6cb7e84bd60182c51c6ce9cb461dfcdb4d35d8e5b0cdfb8822667cb35f42a8e54ed6d67b4575d02bffdb
+  metadata.gz: 534218bfa084bcb43af55192210e08de829044ee316aa4043c9f06b0e97a6b484b4eb666821327f4fb79877383d10819f58570d7194420a00682a6e3e860d437
+  data.tar.gz: 0e8445aabb3cc8ae7a7f36859c12a724e8b4db966e97d45589ddd3ed1cc9fd30e9c2d2c29c321ee35ca2c4487f72004242ee2aa7d8425ac2061a792434e9fc66

data/lib/io_streams/io_streams.rb

@@ -316,8 +316,7 @@ module IOStreams
 
   # Iterate over a file / stream returning each record/line one at a time.
   def self.line_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
-    return yield(file_name_or_io) if file_name_or_io.is_a?(IOStreams::Line::Reader) ||
-      file_name_or_io.is_a?(Array)
+    return yield(file_name_or_io) if file_name_or_io.is_a?(IOStreams::Line::Reader) || file_name_or_io.is_a?(Array)
 
     reader(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace) do |io|
       IOStreams::Line::Reader.open(io, **args, &block)
@@ -427,8 +426,14 @@ module IOStreams
 
     streams = streams_for_file_name(file_name) if streams.nil? && file_name
 
-    # Shortcut for when it is already a stream and no further streams need to be applied.
-    return block.call(file_name_or_io) if !file_name_or_io.is_a?(String) && (streams.nil? || streams.empty?)
+    # Shortcut for when it is already a stream
+    if !file_name_or_io.is_a?(String) && (streams.nil? || streams.empty?)
+      if encoding || encode_cleaner || encode_replace
+        return IOStreams::Encode::Reader.open(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace, &block)
+      else
+        return block.call(file_name_or_io)
+      end
+    end
 
     if streams.nil?
       streams = file_name_or_io.is_a?(String) ? streams_for_file_name(file_name_or_io) : [nil]

data/lib/io_streams/line/reader.rb

@@ -61,7 +61,7 @@ module IOStreams
         # Auto-detect windows/linux line endings if not supplied. \n or \r\n
         @delimiter = delimiter || auto_detect_line_endings
 
-        unless eof?
+        if @buffer
           # Change the delimiters encoding to match that of the input stream
           @delimiter      = @delimiter.encode(@buffer.encoding)
           @delimiter_size = @delimiter.size
@@ -73,7 +73,10 @@ module IOStreams
       # Note:
       # * The line delimiter is _not_ returned.
       def each
-        yield(readline) until eof?
+        until eof?
+          line = readline
+          yield(line) unless line.nil?
+        end
         line_count
       end
 
@@ -88,6 +91,7 @@ module IOStreams
         if index
           data    = @buffer.slice(0, index)
           @buffer = @buffer.slice(index + @delimiter_size, @buffer.size)
+          @line_count += 1
         elsif @eof && @buffer.empty?
           data    = nil
           @buffer = nil
@@ -95,9 +99,9 @@ module IOStreams
           # Last line without delimiter
           data    = @buffer
           @buffer = nil
+          @line_count += 1
         end
 
-        @line_count += 1
         data
       end
 
@@ -130,8 +134,6 @@ module IOStreams
         if block.nil?
           @eof = true
           return false
-        elsif block.size < @buffer_size
-          @eof = true
         end
 
         if @buffer

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = '0.16.0'
+  VERSION = '0.16.1'
 end

data/test/encode_reader_test.rb

@@ -11,7 +11,7 @@ class EncodeReaderTest < Minitest::Test
     end
 
     let :cleansed_data do
-      bad_data.gsub("\xE9".b, '?')
+      bad_data.gsub("\xE9".b, '')
     end
 
     let :stripped_data do
@@ -32,7 +32,7 @@ class EncodeReaderTest < Minitest::Test
         it 'strips invalid characters' do
           input = StringIO.new(bad_data)
           data  =
-            IOStreams::Encode::Reader.open(input, encoding: 'UTF-8', encode_replace: '?') do |io|
+            IOStreams::Encode::Reader.open(input, encoding: 'UTF-8', encode_replace: '') do |io|
               io.read
             end
           assert_equal cleansed_data, data
@@ -43,7 +43,7 @@ class EncodeReaderTest < Minitest::Test
         it 'strips non-printable characters' do
           input = StringIO.new(bad_data)
           data  =
-            IOStreams::Encode::Reader.open(input, encoding: 'UTF-8', encode_cleaner: :printable, encode_replace: '?') do |io|
+            IOStreams::Encode::Reader.open(input, encoding: 'UTF-8', encode_cleaner: :printable, encode_replace: '') do |io|
               io.read
             end
           assert_equal stripped_data, data

data/test/io_streams_test.rb

@@ -18,6 +18,18 @@ class IOStreamsTest < Minitest::Test
       temp_file.path
     end
 
+    let :bad_data do
+      [
+        "New M\xE9xico,NE".b,
+        'good line',
+        "New M\xE9xico,\x07SF".b
+      ].join("\n").encode('BINARY')
+    end
+
+    let :stripped_data do
+      bad_data.gsub("\xE9".b, '').gsub("\x07", '')
+    end
+
     after do
       temp_file.delete
     end
@@ -86,8 +98,19 @@ class IOStreamsTest < Minitest::Test
     describe '.each_line' do
       it 'returns a line at a time' do
         lines = []
-        IOStreams.each_line(source_file_name) { |line| lines << line }
+        count = IOStreams.each_line(source_file_name) { |line| lines << line }
         assert_equal data.lines.map(&:strip), lines
+        assert_equal data.lines.count, count
+      end
+
+      it 'strips non-printable characters' do
+        input = StringIO.new(bad_data)
+        lines = []
+        count = IOStreams.each_line(input, encoding: 'UTF-8', encode_cleaner: :printable, encode_replace: '') do |line|
+          lines << line
+        end
+        assert_equal stripped_data.lines.map(&:strip), lines
+        assert_equal stripped_data.lines.count, count
       end
     end
 

data/test/line_reader_test.rb

@@ -19,39 +19,43 @@ class LineReaderTest < Minitest::Test
     describe '#each' do
       it 'each_line file' do
         lines = []
-        IOStreams::Line::Reader.open(file_name) do |io|
+        count = IOStreams::Line::Reader.open(file_name) do |io|
           io.each { |line| lines << line }
         end
         assert_equal data, lines
+        assert_equal data.size, count
       end
 
       it 'each_line stream' do
         lines = []
-        File.open(file_name) do |file|
+        count = File.open(file_name) do |file|
           IOStreams::Line::Reader.open(file) do |io|
             io.each { |line| lines << line }
           end
         end
         assert_equal data, lines
+        assert_equal data.size, count
       end
 
       ["\r\n", "\n", "\r"].each do |delimiter|
         it "autodetect delimiter: #{delimiter.inspect}" do
           lines  = []
           stream = StringIO.new(data.join(delimiter))
-          IOStreams::Line::Reader.open(stream, buffer_size: 15) do |io|
+          count = IOStreams::Line::Reader.open(stream, buffer_size: 15) do |io|
             io.each { |line| lines << line }
           end
           assert_equal data, lines
+          assert_equal data.size, count
         end
 
         it "single read autodetect delimiter: #{delimiter.inspect}" do
           lines  = []
           stream = StringIO.new(data.join(delimiter))
-          IOStreams::Line::Reader.open(stream) do |io|
+          count = IOStreams::Line::Reader.open(stream) do |io|
             io.each { |line| lines << line }
           end
           assert_equal data, lines
+          assert_equal data.size, count
         end
       end
 
@@ -59,10 +63,11 @@ class LineReaderTest < Minitest::Test
         it "reads delimited #{delimiter.inspect}" do
           lines  = []
           stream = StringIO.new(data.join(delimiter))
-          IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
+          count = IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
             io.each { |line| lines << line }
           end
           assert_equal data, lines
+          assert_equal data.size, count
         end
       end
 
@@ -70,10 +75,11 @@ class LineReaderTest < Minitest::Test
         delimiter = "\x01"
         lines     = []
         stream    = StringIO.new(data.join(delimiter).encode('ASCII-8BIT'))
-        IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
+        count = IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
           io.each { |line| lines << line }
         end
         assert_equal data, lines
+        assert_equal data.size, count
       end
 
       describe '#readline' do

data/test/row_reader_test.rb

@@ -13,20 +13,22 @@ class RowReaderTest < Minitest::Test
     describe '.open' do
       it 'file' do
         rows = []
-        IOStreams::Row::Reader.open(file_name) do |io|
+        count = IOStreams::Row::Reader.open(file_name) do |io|
           io.each { |row| rows << row }
         end
         assert_equal expected, rows
+        assert_equal expected.size, count
       end
 
       it 'stream' do
         rows = []
-        IOStreams.line_reader(file_name) do |file|
+        count = IOStreams.line_reader(file_name) do |file|
           IOStreams::Row::Reader.open(file) do |io|
             io.each { |row| rows << row }
           end
         end
         assert_equal expected, rows
+        assert_equal expected.size, count
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iostreams
 version: !ruby/object:Gem::Version
-  version: 0.16.0
+  version: 0.16.1
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-14 00:00:00.000000000 Z
+date: 2018-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby