iostreams 0.20.3 → 1.0.0.beta

data/test/record_reader_test.rb

@@ -23,7 +23,7 @@ class RecordReaderTest < Minitest::Test
     describe '#each' do
       it 'csv file' do
         records = []
-        IOStreams::Record::Reader.open(file_name, cleanse_header: false) do |io|
+        IOStreams::Record::Reader.file(file_name, cleanse_header: false) do |io|
           io.each { |row| records << row }
         end
         assert_equal expected, records
@@ -31,7 +31,7 @@ class RecordReaderTest < Minitest::Test
 
       it 'json file' do
         records = []
-        IOStreams::Record::Reader.open(json_file_name, cleanse_header: false) do |input|
+        IOStreams::Record::Reader.file(json_file_name, cleanse_header: false, format: :json) do |input|
           input.each { |row| records << row }
         end
         assert_equal expected, records
@@ -39,8 +39,8 @@ class RecordReaderTest < Minitest::Test
 
       it 'stream' do
         rows = []
-        IOStreams.line_reader(file_name) do |file|
-          IOStreams::Record::Reader.open(file, cleanse_header: false) do |io|
+        IOStreams::Line::Reader.file(file_name) do |file|
+          IOStreams::Record::Reader.stream(file, cleanse_header: false) do |io|
             io.each { |row| rows << row }
           end
         end
@@ -50,7 +50,7 @@ class RecordReaderTest < Minitest::Test
 
     describe '#collect' do
       it 'json file' do
-        records = IOStreams::Record::Reader.open(json_file_name) do |input|
+        records = IOStreams::Record::Reader.file(json_file_name, format: :json) do |input|
           input.collect { |record| record["state"] }
         end
         assert_equal expected.collect { |record| record["state"] }, records

data/test/record_writer_test.rb

@@ -43,7 +43,7 @@ class RecordWriterTest < Minitest::Test
 
     describe '#<<' do
       it 'file' do
-        IOStreams::Record::Writer.open(file_name) do |io|
+        IOStreams::Record::Writer.file(file_name) do |io|
           inputs.each { |hash| io << hash }
         end
         result = File.read(file_name)
@@ -51,7 +51,7 @@ class RecordWriterTest < Minitest::Test
       end
 
       it 'json file' do
-        IOStreams::Record::Writer.open(file_name, file_name: 'abc.json') do |io|
+        IOStreams::Record::Writer.file(file_name, file_name: 'abc.json') do |io|
           inputs.each { |hash| io << hash }
         end
         result = File.read(file_name)
@@ -60,8 +60,8 @@ class RecordWriterTest < Minitest::Test
 
       it 'stream' do
         io_string = StringIO.new
-        IOStreams::Line::Writer.open(io_string) do |io|
-          IOStreams::Record::Writer.open(io) do |stream|
+        IOStreams::Line::Writer.stream(io_string) do |io|
+          IOStreams::Record::Writer.stream(io) do |stream|
             inputs.each { |row| stream << row }
           end
         end

data/test/row_reader_test.rb

@@ -12,8 +12,8 @@ class RowReaderTest < Minitest::Test
 
     describe '.open' do
       it 'file' do
-        rows = []
-        count = IOStreams::Row::Reader.open(file_name) do |io|
+        rows  = []
+        count = IOStreams::Row::Reader.file(file_name) do |io|
           io.each { |row| rows << row }
         end
         assert_equal expected, rows
@@ -21,9 +21,9 @@ class RowReaderTest < Minitest::Test
       end
 
       it 'stream' do
-        rows = []
-        count = IOStreams.line_reader(file_name) do |file|
-          IOStreams::Row::Reader.open(file) do |io|
+        rows  = []
+        count = IOStreams::Line::Reader.file(file_name) do |file|
+          IOStreams::Row::Reader.stream(file) do |io|
             io.each { |row| rows << row }
           end
         end

data/test/row_writer_test.rb

@@ -27,19 +27,19 @@ class RowWriterTest < Minitest::Test
       temp_file.delete
     end
 
-    describe '.open' do
+    describe '.stream' do
       it 'file' do
-        IOStreams::Row::Writer.open(file_name) do |io|
+        IOStreams::Row::Writer.file(file_name) do |io|
           csv_rows.each { |array| io << array }
         end
-        result = File.read(file_name)
+        result = ::File.read(file_name)
         assert_equal raw_csv_data, result
       end
 
-      it 'stream' do
+      it 'streams' do
         io_string = StringIO.new
-        IOStreams::Line::Writer.open(io_string) do |io|
-          IOStreams::Row::Writer.open(io) do |stream|
+        IOStreams::Line::Writer.stream(io_string) do |io|
+          IOStreams::Row::Writer.stream(io) do |stream|
             csv_rows.each { |array| stream << array }
           end
         end

data/test/stream_test.rb

@@ -0,0 +1,116 @@
+require_relative 'test_helper'
+
+class StreamTest < Minitest::Test
+  describe IOStreams::Stream do
+    let :source_file_name do
+      File.join(__dir__, 'files', 'text.txt')
+    end
+
+    let :data do
+      File.read(source_file_name)
+    end
+
+    let :bad_data do
+      [
+        "New M\xE9xico,NE".b,
+        'good line',
+        "New M\xE9xico,\x07SF".b
+      ].join("\n").encode('BINARY')
+    end
+
+    let :stripped_data do
+      bad_data.gsub("\xE9".b, '').gsub("\x07", '')
+    end
+
+    let :multiple_zip_file_name do
+      File.join(File.dirname(__FILE__), 'files', 'multiple_files.zip')
+    end
+
+    let :zip_gz_file_name do
+      File.join(File.dirname(__FILE__), 'files', 'text.zip.gz')
+    end
+
+    let :contents_test_txt do
+      File.read(File.join(File.dirname(__FILE__), 'files', 'text.txt'))
+    end
+
+    let :contents_test_json do
+      File.read(File.join(File.dirname(__FILE__), 'files', 'test.json'))
+    end
+
+    let(:string_io) { StringIO.new(data) }
+    let(:stream) { IOStreams::Stream.new(string_io) }
+
+    describe '.reader' do
+      it 'reads a zip file' do
+        File.open(multiple_zip_file_name, 'rb') do |io|
+          result = IOStreams::Stream.new(io).
+            file_name(multiple_zip_file_name).
+            option(:zip, entry_file_name: 'test.json').
+            reader { |io| io.read }
+          assert_equal contents_test_json, result
+        end
+      end
+
+      it 'reads a zip file from within a gz file' do
+        File.open(zip_gz_file_name, 'rb') do |io|
+          result = IOStreams::Stream.new(io).
+            file_name(zip_gz_file_name).
+            reader { |io| io.read }
+          assert_equal contents_test_txt, result
+        end
+      end
+    end
+
+    describe '.line_reader' do
+    end
+
+    describe '.row_reader' do
+    end
+
+    describe '.record_reader' do
+    end
+
+    describe '.each_line' do
+      it 'returns a line at a time' do
+        lines = []
+        stream.stream(:none)
+        count = stream.each_line { |line| lines << line }
+        assert_equal data.lines.map(&:strip), lines
+        assert_equal data.lines.count, count
+      end
+
+      it 'strips non-printable characters' do
+        input  = StringIO.new(bad_data)
+        lines  = []
+        stream = IOStreams::Stream.new(input)
+        stream.stream(:encode, encoding: 'UTF-8', cleaner: :printable, replace: '')
+        count = stream.each_line { |line| lines << line }
+        assert_equal stripped_data.lines.map(&:strip), lines
+        assert_equal stripped_data.lines.count, count
+      end
+    end
+
+    describe '.each_row' do
+    end
+
+    describe '.each_record' do
+    end
+
+    describe '.writer' do
+    end
+
+    describe '.writer' do
+    end
+
+    describe '.line_writer' do
+    end
+
+    describe '.row_writer' do
+    end
+
+    describe '.record_writer' do
+    end
+
+  end
+end

data/test/streams_test.rb

@@ -0,0 +1,255 @@
+require_relative 'test_helper'
+
+class StreamsTest < Minitest::Test
+  describe IOStreams::Streams do
+    let(:file_name) { 'my/path/abc.bcd.xlsx.zip.gz.pgp' }
+    let(:streams) { IOStreams::Streams.new(file_name) }
+
+    describe '#option' do
+      it 'adds one option' do
+        streams.option(:pgp, passphrase: 'unlock-me')
+        assert_equal({pgp: {passphrase: 'unlock-me'}}, streams.options)
+      end
+
+      it 'adds options in order' do
+        streams.option(:pgp, passphrase: 'unlock-me')
+        streams.option(:enc, compress: false)
+        assert_equal({pgp: {passphrase: 'unlock-me'}, enc: {compress: false}}, streams.options)
+      end
+
+      it 'will not add an option if a stream was already set' do
+        streams.stream(:pgp, passphrase: 'unlock-me')
+        assert_raises ArgumentError do
+          streams.option(:pgp, passphrase: 'unlock-me')
+        end
+      end
+
+      it 'will not add an invalid option' do
+        assert_raises ArgumentError do
+          streams.option(:blah, value: 23)
+        end
+      end
+
+      describe 'with no file_name' do
+        let(:file_name) { nil }
+
+        it 'prevents options being set' do
+          assert_raises ArgumentError do
+            streams.option(:pgp, passphrase: 'unlock-me')
+          end
+        end
+      end
+    end
+
+    describe '#stream' do
+      it 'adds one stream' do
+        streams.stream(:pgp, passphrase: 'unlock-me')
+        assert_equal({pgp: {passphrase: 'unlock-me'}}, streams.streams)
+      end
+
+      it 'adds streams in order' do
+        streams.stream(:pgp, passphrase: 'unlock-me')
+        streams.stream(:enc, compress: false)
+        assert_equal({pgp: {passphrase: 'unlock-me'}, enc: {compress: false}}, streams.streams)
+      end
+
+      it 'will not add a stream if an option was already set' do
+        streams.option(:pgp, passphrase: 'unlock-me')
+        assert_raises ArgumentError do
+          streams.stream(:pgp, passphrase: 'unlock-me')
+        end
+      end
+
+      it 'will not add an invalid stream' do
+        assert_raises ArgumentError do
+          streams.stream(:blah, value: 23)
+        end
+      end
+    end
+
+    describe '#reader' do
+      it 'directly calls block for an empty stream' do
+        string_io = StringIO.new
+        value     = nil
+        streams.stream(:none)
+        streams.reader(string_io) do |io|
+          assert_equal io, string_io
+          value = 32
+        end
+        assert_equal 32, value
+      end
+
+      it 'returns the reader' do
+        string_io = StringIO.new
+        streams.stream(:zip)
+        streams.reader(string_io) do |io|
+          assert io.is_a?(::Zip::InputStream), io
+        end
+      end
+
+      it 'returns the last reader' do
+        string_io = StringIO.new
+        streams.stream(:encode)
+        streams.stream(:zip)
+        streams.reader(string_io) do |io|
+          assert io.is_a?(IOStreams::Encode::Reader), io
+        end
+      end
+    end
+
+    describe '#writer' do
+      it 'directly calls block for an empty stream' do
+        string_io = StringIO.new
+        value     = nil
+        streams.stream(:none)
+        streams.writer(string_io) do |io|
+          assert_equal io, string_io
+          value = 32
+        end
+        assert_equal 32, value
+      end
+
+      it 'returns the reader' do
+        string_io = StringIO.new
+        streams.stream(:zip)
+        streams.writer(string_io) do |io|
+          assert io.is_a?(::Zip::OutputStream), io
+        end
+      end
+
+      it 'returns the last reader' do
+        string_io = StringIO.new
+        streams.stream(:encode)
+        streams.stream(:zip)
+        streams.writer(string_io) do |io|
+          assert io.is_a?(IOStreams::Encode::Writer), io
+        end
+      end
+    end
+
+    # Internal methods
+
+    describe '#class_for_stream' do
+      it 'xlsx' do
+        assert_equal IOStreams::Xlsx::Reader, streams.send(:class_for_stream, :reader, :xlsx)
+      end
+
+      it 'gzip' do
+        assert_equal IOStreams::Gzip::Writer, streams.send(:class_for_stream, :writer, :gzip)
+      end
+
+      it 'unknown' do
+        assert_raises ArgumentError do
+          streams.send(:class_for_stream, :reader, :unknown)
+        end
+      end
+    end
+
+    describe '#parse_extensions' do
+      it 'single stream' do
+        streams = IOStreams::Streams.new('my/path/abc.xlsx')
+        assert_equal %i[xlsx], streams.send(:parse_extensions)
+      end
+
+      it 'empty' do
+        streams = IOStreams::Streams.new('my/path/abc.csv')
+        assert_equal [], streams.send(:parse_extensions)
+      end
+
+      it 'handles multiple extensions' do
+        assert_equal %i[xlsx zip gz pgp], streams.send(:parse_extensions)
+      end
+
+      describe 'case-insensitive' do
+        let(:file_name) { 'a.XlsX.GzIp' }
+
+        it 'is case-insensitive' do
+          assert_equal %i[xlsx gzip], streams.send(:parse_extensions)
+        end
+      end
+    end
+
+    describe '#pipeline' do
+      it 'with stream and file name' do
+        expected = {enc: {compress: false}}
+        streams.stream(:enc, compress: false)
+        assert_equal expected, streams.pipeline
+      end
+
+      it 'no file name, streams, or options' do
+        expected = {}
+        streams  = IOStreams::Streams.new
+        assert_equal expected, streams.pipeline
+      end
+
+      it 'file name without options' do
+        expected = {:xlsx => {}, :zip => {}, :gz => {}, :pgp => {}}
+        assert_equal expected, streams.pipeline
+      end
+
+      it 'file name with encode option' do
+        expected = {encode: {encoding: 'BINARY'}, :xlsx => {}, :zip => {}, :gz => {}, :pgp => {}}
+        streams.option(:encode, encoding: 'BINARY')
+        assert_equal expected, streams.pipeline
+      end
+
+      it 'file name with option' do
+        expected = {:xlsx => {}, :zip => {}, :gz => {}, :pgp => {passphrase: 'unlock-me'}}
+        streams.option(:pgp, passphrase: 'unlock-me')
+        assert_equal expected, streams.pipeline
+      end
+    end
+
+    describe '#execute' do
+      it 'directly calls block for an empty stream' do
+        string_io = StringIO.new
+        value     = nil
+        streams.send(:execute, :writer, {}, string_io) do |io|
+          assert_equal io, string_io
+          value = 32
+        end
+        assert_equal 32, value
+      end
+
+      it 'calls last block in one element stream' do
+        pipeline  = {simple: {arg: 'first'}}
+        string_io = StringIO.new
+        streams.send(:execute, :writer, pipeline, string_io) { |io| io.write('last') }
+        assert_equal 'first>last', string_io.string
+      end
+
+      it 'chains blocks in 2 element stream' do
+        pipeline  = {simple: {arg: 'first'}, simple2: {arg: 'second'}}
+        string_io = StringIO.new
+        streams.send(:execute, :writer, pipeline, string_io) { |io| io.write('last') }
+        assert_equal 'second>first>last', string_io.string
+      end
+
+      it 'chains blocks in 3 element stream' do
+        pipeline  = {simple: {arg: 'first'}, simple2: {arg: 'second'}, simple3: {arg: 'third'}}
+        string_io = StringIO.new
+        streams.send(:execute, :writer, pipeline, string_io) { |io| io.write('last') }
+        assert_equal 'third>second>first>last', string_io.string
+      end
+    end
+
+    class SimpleStream
+      def self.stream(io, **args)
+        yield new(io, **args)
+      end
+
+      def initialize(io, arg:)
+        @io  = io
+        @arg = arg
+      end
+
+      def write(data)
+        @io.write("#{@arg}>#{data}")
+      end
+    end
+
+    IOStreams.register_extension(:simple, nil, SimpleStream)
+    IOStreams.register_extension(:simple2, nil, SimpleStream)
+    IOStreams.register_extension(:simple3, nil, SimpleStream)
+  end
+end