iostreams 0.15.0 → 0.16.0

data/lib/iostreams.rb

@@ -1,6 +1,8 @@
 require 'io_streams/version'
 #@formatter:off
 module IOStreams
+  autoload :Errors,  'io_streams/errors'
+
   module Bzip2
     autoload :Reader, 'io_streams/bzip2/reader'
     autoload :Writer, 'io_streams/bzip2/writer'
@@ -27,6 +29,10 @@ module IOStreams
     autoload :Writer, 'io_streams/zip/writer'
   end
 
+  module Encode
+    autoload :Reader, 'io_streams/encode/reader'
+    autoload :Writer, 'io_streams/encode/writer'
+  end
   module Line
     autoload :Reader, 'io_streams/line/reader'
     autoload :Writer, 'io_streams/line/writer'

data/test/encode_reader_test.rb

@@ -0,0 +1,54 @@
+require_relative 'test_helper'
+
+class EncodeReaderTest < Minitest::Test
+  describe IOStreams::Encode::Reader do
+    let :bad_data do
+      [
+        "New M\xE9xico,NE".b,
+        'good line',
+        "New M\xE9xico,\x07SF".b
+      ].join("\n").encode('BINARY')
+    end
+
+    let :cleansed_data do
+      bad_data.gsub("\xE9".b, '?')
+    end
+
+    let :stripped_data do
+      cleansed_data.gsub("\x07", '')
+    end
+
+    describe '#read' do
+      describe 'replacement' do
+        it 'does not strip invalid characters' do
+          input = StringIO.new(bad_data)
+          IOStreams::Encode::Reader.open(input, encoding: 'UTF-8') do |io|
+            assert_raises ::Encoding::UndefinedConversionError do
+              io.read.encoding
+            end
+          end
+        end
+
+        it 'strips invalid characters' do
+          input = StringIO.new(bad_data)
+          data  =
+            IOStreams::Encode::Reader.open(input, encoding: 'UTF-8', encode_replace: '?') do |io|
+              io.read
+            end
+          assert_equal cleansed_data, data
+        end
+      end
+
+      describe 'printable' do
+        it 'strips non-printable characters' do
+          input = StringIO.new(bad_data)
+          data  =
+            IOStreams::Encode::Reader.open(input, encoding: 'UTF-8', encode_cleaner: :printable, encode_replace: '?') do |io|
+              io.read
+            end
+          assert_equal stripped_data, data
+        end
+      end
+    end
+  end
+end

data/test/encode_writer_test.rb

@@ -0,0 +1,82 @@
+require_relative 'test_helper'
+
+class EncodeWriterTest < Minitest::Test
+  describe IOStreams::Encode::Writer do
+    let :bad_data do
+      [
+        "New M\xE9xico,NE".b,
+        'good line',
+        "New M\xE9xico,\x07SF".b
+      ].join("\n").encode('BINARY')
+    end
+
+    let :cleansed_data do
+      bad_data.gsub("\xE9".b, '?')
+    end
+
+    let :stripped_data do
+      cleansed_data.gsub("\x07", '')
+    end
+
+    describe '#<<' do
+      it 'file' do
+        temp_file = Tempfile.new('rocket_job')
+        file_name = temp_file.to_path
+        IOStreams::Encode::Writer.open(file_name, encoding: 'ASCII-8BIT') do |io|
+          io << bad_data
+        end
+        result = File.read(file_name, mode: 'rb')
+        assert_equal bad_data, result
+      end
+
+      it 'stream' do
+        io = StringIO.new(''.b)
+        IOStreams::Encode::Writer.open(io, encoding: 'ASCII-8BIT') do |encoded|
+          encoded << bad_data
+        end
+        assert_equal 'ASCII-8BIT', io.string.encoding.to_s
+        assert_equal bad_data, io.string
+      end
+
+      it 'stream as utf-8' do
+        io = StringIO.new('')
+        assert_raises Encoding::UndefinedConversionError do
+          IOStreams::Encode::Writer.open(io, encoding: 'UTF-8') do |encoded|
+            encoded << bad_data
+          end
+        end
+      end
+
+      it 'stream as utf-8 with replacement' do
+        io = StringIO.new('')
+        IOStreams::Encode::Writer.open(io, encoding: 'UTF-8', encode_replace: '?') do |encoded|
+          encoded << bad_data
+        end
+        assert_equal 'UTF-8', io.string.encoding.to_s
+        assert_equal cleansed_data, io.string
+      end
+
+      it 'stream as utf-8 with replacement and printable cleansing' do
+        io = StringIO.new('')
+        IOStreams::Encode::Writer.open(io, encoding: 'UTF-8', encode_replace: '?', encode_cleaner: :printable) do |encoded|
+          encoded << bad_data
+        end
+        assert_equal 'UTF-8', io.string.encoding.to_s
+        assert_equal stripped_data, io.string
+      end
+    end
+
+    describe '.write' do
+      it 'returns byte count' do
+        io_string = StringIO.new(''.b)
+        count     = 0
+        IOStreams::Encode::Writer.open(io_string, encoding: 'ASCII-8BIT') do |io|
+          count += io.write(bad_data)
+        end
+        assert_equal bad_data, io_string.string
+        assert_equal bad_data.size, count
+      end
+    end
+
+  end
+end

data/test/io_streams_test.rb

@@ -91,70 +91,5 @@ class IOStreamsTest < Minitest::Test
       end
     end
 
-    describe '.reader' do
-      # IOStreams.reader('abc.csv') do |io|
-      #   p data while (data = io.read(128))
-      # end
-    end
-
-    describe '.each_line' do
-      # IOStreams.each_line('abc.csv') do |line|
-      #   puts line
-      # end
-    end
-
-    describe '.each_row' do
-      # IOStreams.each_row('abc.csv') do |array|
-      #   p array
-      # end
-    end
-
-    describe '.each_record' do
-      # IOStreams.each_record('abc.csv') do |hash|
-      #   p hash
-      # end
-
-      # array = [
-      #   'name, address, zip_code',
-      #   'Jack, Down Under, 12345'
-      # ]
-      # IOStreams.each_record(array) do |hash|
-      #   p hash
-      # end
-    end
-
-    describe '.writer' do
-      # IOStreams.writer('abc.csv') do |io|
-      #   io.write('This')
-      #   io.write(' is ')
-      #   io.write(" one line\n")
-      # end
-    end
-
-    describe '.line_writer' do
-      # IOStreams.line_writer('abc.csv') do |file|
-      #   file << 'these'
-      #   file << 'are'
-      #   file << 'all'
-      #   file << 'separate'
-      #   file << 'lines'
-      # end
-    end
-
-    describe '.row_writer' do
-      # IOStreams.row_writer('abc.csv') do |io|
-      #   io << %w[name address zip_code]
-      #   io << %w[Jack There 1234]
-      #   io << ['Joe', 'Over There somewhere', 1234]
-      # end
-    end
-
-    describe '.record_writer' do
-      # IOStreams.record_writer('abc.csv') do |stream|
-      #   stream << {name: 'Jack', address: 'There', zip_code: 1234}
-      #   stream << {name: 'Joe', address: 'Over There somewhere', zip_code: 1234}
-      # end
-    end
-
   end
 end

data/test/line_reader_test.rb

@@ -16,40 +16,6 @@ class LineReaderTest < Minitest::Test
       data
     end
 
-    describe '#initialize' do
-      it 'does not strip invalid characters' do
-        bad_lines = [
-          "New M\xE9xico,NE",
-          'good line',
-          "New M\xE9xico,SF"
-        ]
-        input     = StringIO.new(bad_lines.join("\n"))
-        lines     = []
-        IOStreams::Line::Reader.open(input) do |io|
-          assert_equal false, io.strip_non_printable
-          assert_raises ArgumentError do
-            io.each { |line| lines << line }
-          end
-        end
-      end
-
-      it 'strips invalid characters' do
-        bad_lines   = [
-          "New M\xE9xico,NE",
-          'good line',
-          "New M\xE9xico,SF"
-        ]
-        fixed_lines = bad_lines.collect { |line| line.force_encoding('BINARY').gsub(/[^[:print:]|\r|\n]/, '') }
-        input       = StringIO.new(bad_lines.join("\n"))
-        lines       = []
-        IOStreams::Line::Reader.open(input, strip_non_printable: true) do |io|
-          assert_equal true, io.strip_non_printable
-          io.each { |line| lines << line }
-        end
-        assert_equal fixed_lines, lines
-      end
-    end
-
     describe '#each' do
       it 'each_line file' do
         lines = []
@@ -69,7 +35,7 @@ class LineReaderTest < Minitest::Test
         assert_equal data, lines
       end
 
-      ["\r\n", "\n\r", "\n", "\r"].each do |delimiter|
+      ["\r\n", "\n", "\r"].each do |delimiter|
         it "autodetect delimiter: #{delimiter.inspect}" do
           lines  = []
           stream = StringIO.new(data.join(delimiter))
@@ -78,6 +44,15 @@ class LineReaderTest < Minitest::Test
           end
           assert_equal data, lines
         end
+
+        it "single read autodetect delimiter: #{delimiter.inspect}" do
+          lines  = []
+          stream = StringIO.new(data.join(delimiter))
+          IOStreams::Line::Reader.open(stream) do |io|
+            io.each { |line| lines << line }
+          end
+          assert_equal data, lines
+        end
       end
 
       ['@', 'BLAH'].each do |delimiter|
@@ -94,12 +69,180 @@ class LineReaderTest < Minitest::Test
       it 'reads binary delimited' do
         delimiter = "\x01"
         lines     = []
-        stream    = StringIO.new(data.join(delimiter))
-        IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter, encoding: IOStreams::BINARY_ENCODING) do |io|
+        stream    = StringIO.new(data.join(delimiter).encode('ASCII-8BIT'))
+        IOStreams::Line::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
           io.each { |line| lines << line }
         end
         assert_equal data, lines
       end
+
+      describe '#readline' do
+        let(:short_line) { '0123456789' }
+        let(:longer_line) { 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' }
+        let(:delimiter) { "\r\n" }
+
+        it 'reads delimiter in first block, no delimiter at end' do
+          data        = [short_line, longer_line].join(delimiter)
+          buffer_size = short_line.length + delimiter.size + (longer_line.size / 2)
+
+          stream = StringIO.new(data)
+          IOStreams::Line::Reader.open(stream, buffer_size: buffer_size) do |io|
+            refute io.eof?
+            assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
+
+            assert_equal short_line, io.readline
+            assert_equal longer_line, io.readline
+
+            assert io.eof?
+            assert_nil io.readline
+          end
+        end
+
+        it 'reads delimiter in second block, no delimiter at end' do
+          data        = [longer_line, short_line, short_line].join(delimiter)
+          buffer_size = (longer_line.length + delimiter.size + 5) / 2
+
+          stream = StringIO.new(data)
+          IOStreams::Line::Reader.open(stream, buffer_size: buffer_size) do |io|
+            refute io.eof?
+            assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
+            assert_equal longer_line, io.readline
+            assert_equal short_line, io.readline
+            assert_equal short_line, io.readline
+            assert io.eof?
+            assert_nil io.readline
+          end
+        end
+
+        it 'reads delimiter split across first and second blocks' do
+          data        = [longer_line, short_line, short_line].join(delimiter)
+          buffer_size = longer_line.length + 1
+
+          stream = StringIO.new(data)
+          IOStreams::Line::Reader.open(stream, buffer_size: buffer_size) do |io|
+            refute io.eof?
+            assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
+            assert_equal longer_line, io.readline
+            assert_equal short_line, io.readline
+            assert_equal short_line, io.readline
+            assert io.eof?
+            assert_nil io.readline
+          end
+        end
+
+        it 'reads file with no matching delimiter' do
+          delimiter   = '@'
+          data        = [longer_line, short_line, longer_line].join(delimiter) + delimiter
+          buffer_size = longer_line.length + 1
+
+          stream = StringIO.new(data)
+          IOStreams::Line::Reader.open(stream, buffer_size: buffer_size) do |io|
+            refute io.eof?
+            assert_equal "\n", io.delimiter, -> { io.delimiter.ai }
+            assert_equal data, io.readline
+            assert io.eof?
+            assert_nil io.readline
+          end
+        end
+
+        it 'reads small file with no matching delimiter' do
+          data        = short_line
+          buffer_size = short_line.length + 100
+
+          stream = StringIO.new(data)
+          IOStreams::Line::Reader.open(stream, buffer_size: buffer_size) do |io|
+            refute io.eof?
+            assert_equal "\n", io.delimiter, -> { io.delimiter.ai }
+            assert_equal short_line, io.readline
+            assert io.eof?
+            assert_nil io.readline
+          end
+        end
+
+        it 'reads last line with the delimiter as the last character' do
+          delimiter   = '@'
+          data        = [longer_line, short_line, longer_line].join(delimiter) + delimiter
+          buffer_size = longer_line.length + 1
+
+          stream = StringIO.new(data)
+          IOStreams::Line::Reader.open(stream, buffer_size: buffer_size, delimiter: delimiter) do |io|
+            refute io.eof?
+            assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
+            assert_equal longer_line, io.readline
+            assert_equal short_line, io.readline
+            assert_equal longer_line, io.readline
+            assert_nil io.readline
+            assert io.eof?
+          end
+        end
+
+        it 'reads last line with the multi-byte delimiter as the last bytes' do
+          data        = [longer_line, short_line, longer_line].join(delimiter) + delimiter
+          buffer_size = longer_line.length + 1
+
+          stream = StringIO.new(data)
+          IOStreams::Line::Reader.open(stream, buffer_size: buffer_size) do |io|
+            refute io.eof?
+            assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
+            assert_equal longer_line, io.readline
+            assert_equal short_line, io.readline
+            assert_equal longer_line, io.readline
+            assert_nil io.readline
+            assert io.eof?
+          end
+        end
+
+        describe 'read 1 char at a time' do
+          let(:buffer_size) { 1 }
+
+          it 'delimiter at the end' do
+            data = [longer_line, short_line, longer_line].join(delimiter) + delimiter
+
+            stream = StringIO.new(data)
+            IOStreams::Line::Reader.open(stream, buffer_size: buffer_size) do |io|
+              refute io.eof?
+              assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
+              assert_equal longer_line, io.readline
+              assert_equal short_line, io.readline
+              assert_equal longer_line, io.readline
+              assert_nil io.readline
+              assert io.eof?
+            end
+          end
+
+          it 'no delimiter at the end' do
+            data = [longer_line, short_line, longer_line].join(delimiter)
+
+            stream = StringIO.new(data)
+            IOStreams::Line::Reader.open(stream, buffer_size: buffer_size) do |io|
+              refute io.eof?
+              assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
+              assert_equal longer_line, io.readline
+              assert_equal short_line, io.readline
+              assert_equal longer_line, io.readline
+              assert_nil io.readline
+              assert io.eof?
+            end
+          end
+        end
+
+        it 'reads empty file' do
+          stream = StringIO.new
+          IOStreams::Line::Reader.open(stream) do |io|
+            assert io.eof?
+          end
+        end
+
+        it 'prevents denial of service' do
+          data   = 'a' * IOStreams::Line::Reader::MAX_BLOCKS_MULTIPLIER + 'a'
+          stream = StringIO.new(data)
+          assert_raises IOStreams::Errors::DelimiterNotFound do
+            IOStreams::Line::Reader.open(stream, buffer_size: 1) do |io|
+            end
+          end
+
+        end
+      end
     end
   end
 end