iostreams 0.14.0 → 0.15.0

data/test/pgp_writer_test.rb

@@ -1,92 +1,93 @@
 require_relative 'test_helper'
 
-module Streams
-  class PgpWriterTest < Minitest::Test
-    describe IOStreams::Pgp::Writer do
-      before do
-        file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
-        @data     = File.read(file_name)
+class PgpWriterTest < Minitest::Test
+  describe IOStreams::Pgp::Writer do
+    let :temp_file do
+      Tempfile.new('iostreams')
+    end
 
-        @temp_file = Tempfile.new('iostreams')
-        @file_name = @temp_file.to_path
-      end
+    let :file_name do
+      temp_file.path
+    end
 
-      after do
-        @temp_file.delete if @temp_file
-      end
+    let :decrypted do
+      file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
+      File.read(file_name)
+    end
 
-      describe '.open' do
-        it 'writes encrypted text file' do
-          IOStreams::Pgp::Writer.open(@file_name, recipient: 'receiver@example.org', binary: false) do |io|
-            io.write(@data)
-          end
+    after do
+      temp_file.delete
+    end
 
-          result = IOStreams::Pgp::Reader.open(@file_name, passphrase: 'receiver_passphrase', binary: false) { |file| file.read }
-          assert_equal @data, result
+    describe '.open' do
+      it 'writes encrypted text file' do
+        IOStreams::Pgp::Writer.open(file_name, recipient: 'receiver@example.org', binary: false) do |io|
+          io.write(decrypted)
         end
 
-        it 'writes encrypted binary file' do
-          binary_file_name = File.join(File.dirname(__FILE__), 'files', 'spreadsheet.xlsx')
-          binary_data      = File.open(binary_file_name, 'rb') { |file| file.read }
+        result = IOStreams::Pgp::Reader.open(file_name, passphrase: 'receiver_passphrase', binary: false) { |file| file.read }
+        assert_equal decrypted, result
+      end
 
-          File.open(binary_file_name, 'rb') do |input|
-            IOStreams::Pgp::Writer.open(@file_name, recipient: 'receiver@example.org') do |output|
-              IOStreams.copy(input, output, 65535)
-            end
-          end
+      it 'writes encrypted binary file' do
+        binary_file_name = File.join(File.dirname(__FILE__), 'files', 'spreadsheet.xlsx')
+        binary_data      = File.open(binary_file_name, 'rb') { |file| file.read }
 
-          result = IOStreams::Pgp::Reader.open(@file_name, passphrase: 'receiver_passphrase') { |file| file.read }
-          assert_equal binary_data, result
+        File.open(binary_file_name, 'rb') do |input|
+          IOStreams::Pgp::Writer.open(file_name, recipient: 'receiver@example.org') do |output|
+            IOStreams.copy(input, output)
+          end
         end
 
-        it 'writes and signs encrypted file' do
-          IOStreams::Pgp::Writer.open(@file_name, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
-            io.write(@data)
-          end
+        result = IOStreams::Pgp::Reader.open(file_name, passphrase: 'receiver_passphrase') { |file| file.read }
+        assert_equal binary_data, result
+      end
 
-          result = IOStreams::Pgp::Reader.open(@file_name, passphrase: 'receiver_passphrase') { |file| file.read }
-          assert_equal @data, result
+      it 'writes and signs encrypted file' do
+        IOStreams::Pgp::Writer.open(file_name, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
+          io.write(decrypted)
         end
 
-        it 'fails with bad signer passphrase' do
-          skip 'GnuPG v2.1 and above passes when it should not' if IOStreams::Pgp.pgp_version.to_f >= 2.1
-          assert_raises IOStreams::Pgp::Failure do
-            IOStreams::Pgp::Writer.open(@file_name, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'BAD') do |io|
-              io.write(@data)
-            end
+        result = IOStreams::Pgp::Reader.open(file_name, passphrase: 'receiver_passphrase') { |file| file.read }
+        assert_equal decrypted, result
+      end
+
+      it 'fails with bad signer passphrase' do
+        skip 'GnuPG v2.1 and above passes when it should not' if IOStreams::Pgp.pgp_version.to_f >= 2.1
+        assert_raises IOStreams::Pgp::Failure do
+          IOStreams::Pgp::Writer.open(file_name, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'BAD') do |io|
+            io.write(decrypted)
           end
         end
+      end
 
-        it 'fails with bad recipient' do
-          assert_raises IOStreams::Pgp::Failure do
-            IOStreams::Pgp::Writer.open(@file_name, recipient: 'BAD@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
-              io.write(@data)
-              # Allow process to terminate
-              sleep 1
-              io.write(@data)
-            end
+      it 'fails with bad recipient' do
+        assert_raises IOStreams::Pgp::Failure do
+          IOStreams::Pgp::Writer.open(file_name, recipient: 'BAD@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
+            io.write(decrypted)
+            # Allow process to terminate
+            sleep 1
+            io.write(decrypted)
           end
         end
+      end
 
-        it 'fails with bad signer' do
-          assert_raises IOStreams::Pgp::Failure do
-            IOStreams::Pgp::Writer.open(@file_name, recipient: 'receiver@example.org', signer: 'BAD@example.org', signer_passphrase: 'sender_passphrase') do |io|
-              io.write(@data)
-            end
+      it 'fails with bad signer' do
+        assert_raises IOStreams::Pgp::Failure do
+          IOStreams::Pgp::Writer.open(file_name, recipient: 'receiver@example.org', signer: 'BAD@example.org', signer_passphrase: 'sender_passphrase') do |io|
+            io.write(decrypted)
           end
         end
+      end
 
-        it 'fails with stream output' do
-          string_io = StringIO.new
-          assert_raises NotImplementedError do
-            IOStreams::Pgp::Writer.open(string_io, recipient: 'receiver@example.org') do |io|
-              io.write(@data)
-            end
+      it 'fails with stream output' do
+        string_io = StringIO.new
+        assert_raises NotImplementedError do
+          IOStreams::Pgp::Writer.open(string_io, recipient: 'receiver@example.org') do |io|
+            io.write(decrypted)
           end
         end
-
       end
-
     end
   end
 end

data/test/record_reader_test.rb

@@ -0,0 +1,61 @@
+require_relative 'test_helper'
+
+class RecordReaderTest < Minitest::Test
+  describe IOStreams::Record::Reader do
+    let :file_name do
+      File.join(File.dirname(__FILE__), 'files', 'test.csv')
+    end
+
+    let :json_file_name do
+      File.join(File.dirname(__FILE__), 'files', 'test.json')
+    end
+
+    let :csv_rows do
+      CSV.read(file_name)
+    end
+
+    let :expected do
+      rows   = csv_rows.dup
+      header = rows.shift
+      rows.collect { |row| header.zip(row).to_h }
+    end
+
+    describe '#each' do
+      it 'csv file' do
+        records = []
+        IOStreams::Record::Reader.open(file_name, cleanse_header: false) do |io|
+          io.each { |row| records << row }
+        end
+        assert_equal expected, records
+      end
+
+      it 'json file' do
+        records = []
+        IOStreams::Record::Reader.open(json_file_name, cleanse_header: false) do |input|
+          input.each { |row| records << row }
+        end
+        assert_equal expected, records
+      end
+
+      it 'stream' do
+        rows = []
+        IOStreams.line_reader(file_name) do |file|
+          IOStreams::Record::Reader.open(file, cleanse_header: false) do |io|
+            io.each { |row| rows << row }
+          end
+        end
+        assert_equal expected, rows
+      end
+    end
+
+    describe '#collect' do
+      it 'json file' do
+        records = IOStreams::Record::Reader.open(json_file_name) do |input|
+          input.collect { |record| record["state"] }
+        end
+        assert_equal expected.collect { |record| record["state"] }, records
+      end
+    end
+
+  end
+end

data/test/record_writer_test.rb

@@ -0,0 +1,73 @@
+require_relative 'test_helper'
+require 'csv'
+
+class RecordWriterTest < Minitest::Test
+  describe IOStreams::Record::Writer do
+    let :csv_file_name do
+      File.join(File.dirname(__FILE__), 'files', 'test.csv')
+    end
+
+    let :json_file_name do
+      File.join(File.dirname(__FILE__), 'files', 'test.json')
+    end
+
+    let :raw_csv_data do
+      File.read(csv_file_name)
+    end
+
+    let :raw_json_data do
+      File.read(json_file_name)
+    end
+
+    let :csv_rows do
+      CSV.read(csv_file_name)
+    end
+
+    let :inputs do
+      rows   = csv_rows.dup
+      header = rows.shift
+      rows.collect { |row| header.zip(row).to_h }
+    end
+
+    let :temp_file do
+      Tempfile.new('iostreams')
+    end
+
+    let :file_name do
+      temp_file.path
+    end
+
+    after do
+      temp_file.delete
+    end
+
+    describe '#<<' do
+      it 'file' do
+        IOStreams::Record::Writer.open(file_name) do |io|
+          inputs.each { |hash| io << hash }
+        end
+        result = File.read(file_name)
+        assert_equal raw_csv_data, result
+      end
+
+      it 'json file' do
+        IOStreams::Record::Writer.open(file_name, file_name: 'abc.json') do |io|
+          inputs.each { |hash| io << hash }
+        end
+        result = File.read(file_name)
+        assert_equal raw_json_data, result
+      end
+
+      it 'stream' do
+        io_string = StringIO.new
+        IOStreams::Line::Writer.open(io_string) do |io|
+          IOStreams::Record::Writer.open(io) do |stream|
+            inputs.each { |row| stream << row }
+          end
+        end
+        assert_equal raw_csv_data, io_string.string
+      end
+    end
+
+  end
+end

data/test/row_reader_test.rb

@@ -0,0 +1,34 @@
+require_relative 'test_helper'
+
+class RowReaderTest < Minitest::Test
+  describe IOStreams::Row::Reader do
+    let :file_name do
+      File.join(File.dirname(__FILE__), 'files', 'test.csv')
+    end
+
+    let :expected do
+      CSV.read(file_name)
+    end
+
+    describe '.open' do
+      it 'file' do
+        rows = []
+        IOStreams::Row::Reader.open(file_name) do |io|
+          io.each { |row| rows << row }
+        end
+        assert_equal expected, rows
+      end
+
+      it 'stream' do
+        rows = []
+        IOStreams.line_reader(file_name) do |file|
+          IOStreams::Row::Reader.open(file) do |io|
+            io.each { |row| rows << row }
+          end
+        end
+        assert_equal expected, rows
+      end
+    end
+
+  end
+end

data/test/row_writer_test.rb

@@ -0,0 +1,51 @@
+require_relative 'test_helper'
+require 'csv'
+
+class RowWriterTest < Minitest::Test
+  describe IOStreams::Row::Writer do
+    let :csv_file_name do
+      File.join(File.dirname(__FILE__), 'files', 'test.csv')
+    end
+
+    let :raw_csv_data do
+      File.read(csv_file_name)
+    end
+
+    let :csv_rows do
+      CSV.read(csv_file_name)
+    end
+
+    let :temp_file do
+      Tempfile.new('iostreams')
+    end
+
+    let :file_name do
+      temp_file.path
+    end
+
+    after do
+      temp_file.delete
+    end
+
+    describe '.open' do
+      it 'file' do
+        IOStreams::Row::Writer.open(file_name) do |io|
+          csv_rows.each { |array| io << array }
+        end
+        result = File.read(file_name)
+        assert_equal raw_csv_data, result
+      end
+
+      it 'stream' do
+        io_string = StringIO.new
+        IOStreams::Line::Writer.open(io_string) do |io|
+          IOStreams::Row::Writer.open(io) do |stream|
+            csv_rows.each { |array| stream << array }
+          end
+        end
+        assert_equal raw_csv_data, io_string.string
+      end
+    end
+
+  end
+end

data/test/tabular_test.rb

@@ -0,0 +1,184 @@
+require_relative 'test_helper'
+
+class TabularTest < Minitest::Test
+  describe IOStreams::Tabular do
+    let :format do
+      :csv
+    end
+
+    let :tabular do
+      IOStreams::Tabular.new(columns: ['first_field', 'second', 'third'], format: format)
+    end
+
+    describe '#parse_header' do
+      it 'parses and sets the csv header' do
+        tabular = IOStreams::Tabular.new(format: :csv)
+        header  = tabular.parse_header('first field,Second,thirD')
+        assert_equal ['first field', 'Second', 'thirD'], header
+        assert_equal header, tabular.header.columns
+      end
+    end
+
+    describe '#cleanse_header!' do
+      describe 'cleanses' do
+        it 'a csv header' do
+          tabular = IOStreams::Tabular.new(columns: ['first field', 'Second', 'thirD'])
+          header  = tabular.cleanse_header!
+          assert_equal ['first_field', "second", "third"], header
+          assert_equal header, tabular.header.columns
+        end
+
+        it 'white listed snake cased alphanumeric columns' do
+          tabular         = IOStreams::Tabular.new(
+            columns:         ['Ard Vark', 'password', 'robot version', '$$$'],
+            allowed_columns: %w( ard_vark robot_version )
+          )
+          expected_header = ['ard_vark', nil, 'robot_version', nil]
+          cleansed_header = tabular.cleanse_header!
+          assert_equal(expected_header, cleansed_header)
+        end
+      end
+
+      describe 'allowed_columns' do
+        before do
+          @allowed_columns = ['first', 'second', 'third', 'fourth', 'fifth']
+        end
+
+        it 'passes' do
+          tabular = IOStreams::Tabular.new(columns: ['   first ', 'Second', 'thirD   '], allowed_columns: @allowed_columns)
+          header  = tabular.cleanse_header!
+          assert_equal ['first', 'second', 'third'], header
+          assert_equal header, tabular.header.columns
+          assert_equal @allowed_columns, tabular.header.allowed_columns
+        end
+
+        it 'nils columns not in the whitelist' do
+          tabular = IOStreams::Tabular.new(columns: ['   first ', 'Unknown Column', 'thirD   '], allowed_columns: @allowed_columns)
+          header  = tabular.cleanse_header!
+          assert_equal ['first', nil, 'third'], header
+        end
+
+        it 'raises exception for columns not in the whitelist' do
+          tabular = IOStreams::Tabular.new(columns: ['   first ', 'Unknown Column', 'thirD   '], allowed_columns: @allowed_columns, skip_unknown: false)
+          exc     = assert_raises IOStreams::Tabular::Errors::InvalidHeader do
+            tabular.cleanse_header!
+          end
+          assert_equal 'Unknown columns after cleansing: Unknown Column', exc.message
+        end
+
+        it 'raises exception missing required columns' do
+          required = ['first', 'second', 'fifth']
+          tabular  = IOStreams::Tabular.new(columns: ['   first ', 'Second', 'thirD   '], allowed_columns: @allowed_columns, required_columns: required)
+          exc      = assert_raises IOStreams::Tabular::Errors::InvalidHeader do
+            tabular.cleanse_header!
+          end
+          assert_equal 'Missing columns after cleansing: fifth', exc.message
+        end
+
+        it 'raises exception when no columns left' do
+          tabular = IOStreams::Tabular.new(columns: ['one', 'two', 'three'], allowed_columns: @allowed_columns)
+          exc     = assert_raises IOStreams::Tabular::Errors::InvalidHeader do
+            tabular.cleanse_header!
+          end
+          assert_equal 'All columns are unknown after cleansing: one,two,three', exc.message
+        end
+      end
+    end
+
+    describe '#record_parse' do
+      describe ':array format' do
+        let :format do
+          :array
+        end
+
+        it 'renders' do
+          assert hash = tabular.record_parse([1, 2, 3])
+          assert_equal({'first_field' => 1, 'second' => 2, 'third' => 3}, hash)
+        end
+      end
+
+      it 'format :csv' do
+        assert hash = tabular.record_parse('1,2,3')
+        assert_equal({'first_field' => '1', 'second' => '2', 'third' => '3'}, hash)
+      end
+
+      describe ':hash format' do
+        let :format do
+          :hash
+        end
+
+        it 'renders' do
+          assert hash = tabular.record_parse('first_field' => 1, 'second' => 2, 'third' => 3)
+          assert_equal({'first_field' => 1, 'second' => 2, 'third' => 3}, hash)
+        end
+      end
+
+      describe ':json format' do
+        let :format do
+          :json
+        end
+
+        it 'renders' do
+          assert hash = tabular.record_parse('{"first_field":1,"second":2,"third":3}')
+          assert_equal({'first_field' => 1, 'second' => 2, 'third' => 3}, hash)
+        end
+      end
+
+      describe ':psv format' do
+        let :format do
+          :psv
+        end
+
+        it 'renders' do
+          assert hash = tabular.record_parse('1|2|3')
+          assert_equal({'first_field' => '1', 'second' => '2', 'third' => '3'}, hash)
+        end
+      end
+
+      it 'skips columns not in the whitelist' do
+        tabular.header.allowed_columns = ['first', 'second', 'third', 'fourth', 'fifth']
+        tabular.cleanse_header!
+        assert hash = tabular.record_parse('1,2,3')
+        assert_equal({'second' => '2', 'third' => '3'}, hash)
+      end
+
+      it 'handles missing values' do
+        assert hash = tabular.record_parse('1,2')
+        assert_equal({'first_field' => '1', 'second' => '2', 'third' => nil}, hash)
+      end
+    end
+
+    describe '#render' do
+      it 'renders an array of values' do
+        assert csv_string = tabular.render([5, 6, 9])
+        assert_equal '5,6,9', csv_string
+      end
+
+      it 'renders a hash' do
+        assert csv_string = tabular.render({'third' => '3', 'first_field' => '1'})
+        assert_equal '1,,3', csv_string
+      end
+
+      it 'renders a hash including nil and boolean' do
+        assert csv_string = tabular.render({'third' => true, 'first_field' => false, 'second' => nil})
+        assert_equal 'false,,true', csv_string
+      end
+
+      describe ':psv format' do
+        let :format do
+          :psv
+        end
+
+        it 'renders psv nil and boolean' do
+          assert psv_string = tabular.render({'third' => true, 'first_field' => false, 'second' => nil})
+          assert_equal 'false||true', psv_string
+        end
+
+        it 'renders psv numeric and pipe data' do
+          assert psv_string = tabular.render({'third' => 23, 'first_field' => 'a|b|c', 'second' => '|'})
+          assert_equal 'a:b:c|:|23', psv_string
+        end
+      end
+    end
+  end
+end