iostreams 1.2.1 → 1.6.2

data/test/files/unclosed_quote_test2.csv

@@ -0,0 +1,3 @@
+name, description, zip
+Jack,Firstn"ame is Jack,234567
+John,Firstname is John,234568

data/test/io_streams_test.rb

@@ -16,7 +16,7 @@ module IOStreams
       end
 
       let :json_file_name do
-        "/tmp/io_streams/abc.json"
+        "/tmp/iostreams_abc.json"
       end
 
       describe ".root" do
@@ -90,7 +90,7 @@ module IOStreams
         it "hash reader detects json format from file name" do
           ::File.open(json_file_name, "wb") { |file| file.write(expected_json) }
           rows = []
-          path = IOStreams.path("/tmp/io_streams/abc.json")
+          path = IOStreams.path(json_file_name)
           path.each(:hash) do |row|
             rows << row
           end

data/test/line_reader_test.rb

@@ -14,6 +14,14 @@ class LineReaderTest < Minitest::Test
       File.join(File.dirname(__FILE__), "files", "unclosed_quote_test.csv")
     end
 
+    let :unclosed_quote_file2 do
+      File.join(File.dirname(__FILE__), "files", "unclosed_quote_test2.csv")
+    end
+
+    let :unclosed_quote_large_file do
+      File.join(File.dirname(__FILE__), "files", "unclosed_quote_large_test.csv")
+    end
+
     let :data do
       data = []
       File.open(file_name, "rt") do |file|
@@ -51,13 +59,31 @@ class LineReaderTest < Minitest::Test
           assert_equal 4, lines.count
         end
 
-        it "raises error for unclosed quote" do
-          assert_raises(RuntimeError) do
+        it "raises error for unbalanced quotes" do
+          exc = assert_raises(IOStreams::Errors::MalformedDataError) do
             IOStreams::Line::Reader.file(unclosed_quote_file, embedded_within: '"') do |io|
-              io.each do |line|
-              end
+              io.each { |line| }
+            end
+          end
+          assert_includes exc.message, "Unbalanced delimited field, delimiter:"
+        end
+
+        it "raises error for unclosed quote" do
+          exc = assert_raises(IOStreams::Errors::MalformedDataError) do
+            IOStreams::Line::Reader.file(unclosed_quote_file2, embedded_within: '"') do |io|
+              io.each { |line| }
+            end
+          end
+          assert_includes exc.message, "Unbalanced delimited field, delimiter:"
+        end
+
+        it "raises error for unclosed quote before eof" do
+          exc = assert_raises(IOStreams::Errors::MalformedDataError) do
+            IOStreams::Line::Reader.file(unclosed_quote_large_file, embedded_within: '"', buffer_size: 20) do |io|
+              io.each { |line| }
             end
           end
+          assert_includes exc.message, "Unbalanced delimited field, delimiter:"
         end
       end
     end

data/test/paths/file_test.rb

@@ -171,7 +171,7 @@ module Paths
 
       describe "reader" do
         it "reads file" do
-          assert_equal data, file_path.reader(&:read)
+          assert_equal data, file_path.read
         end
       end
 

data/test/paths/s3_test.rb

@@ -73,7 +73,7 @@ module Paths
 
       describe "#reader" do
         it "reads" do
-          assert_equal raw, existing_path.reader(&:read)
+          assert_equal raw, existing_path.read
         end
       end
 
@@ -89,7 +89,7 @@ module Paths
 
       describe "#writer" do
         it "writes" do
-          assert_equal raw.size, write_path.writer { |io| io.write(raw) }
+          assert_equal(raw.size, write_path.writer { |io| io.write(raw) })
           assert write_path.exist?
           assert_equal raw, write_path.read
         end
@@ -138,7 +138,7 @@ module Paths
 
         it "returns all the children under a sub-dir" do
           write_raw_data
-          expected = abd_file_names.collect { |file_name| each_root.join(file_name) }
+          expected = %w[abd/test1.txt abd/test5.file].collect { |file_name| each_root.join(file_name) }
           assert_equal expected.collect(&:to_s).sort, each_root.children("abd/*").collect(&:to_s).sort
         end
 

data/test/paths/sftp_test.rb

@@ -42,7 +42,7 @@ module Paths
 
       describe "#reader" do
         it "reads" do
-          assert_equal raw, existing_path.reader(&:read)
+          assert_equal raw, existing_path.read
         end
 
         it "fails when the file does not exist" do
@@ -60,7 +60,7 @@ module Paths
 
       describe "#writer" do
         it "writes" do
-          assert_equal raw.size, write_path.writer { |io| io.write(raw) }
+          assert_equal(raw.size, write_path.writer { |io| io.write(raw) })
           assert_equal raw, write_path.read
         end
 
@@ -77,7 +77,7 @@ module Paths
 
           it "writes" do
             skip "No identity file env var set: SFTP_IDENTITY_FILE" unless ENV["SFTP_IDENTITY_FILE"]
-            assert_equal raw.size, write_path.writer { |io| io.write(raw) }
+            assert_equal(raw.size, write_path.writer { |io| io.write(raw) })
             assert_equal raw, write_path.read
           end
         end
@@ -90,7 +90,7 @@ module Paths
 
           it "writes" do
             skip "No identity file env var set: SFTP_IDENTITY_FILE" unless ENV["SFTP_IDENTITY_FILE"]
-            assert_equal raw.size, write_path.writer { |io| io.write(raw) }
+            assert_equal(raw.size, write_path.writer { |io| io.write(raw) })
             assert_equal raw, write_path.read
           end
         end

data/test/pgp_test.rb

@@ -70,7 +70,7 @@ class PgpTest < Minitest::Test
         refute IOStreams::Pgp.delete_keys(email: "random@iostreams.net", public: true, private: true)
       end
 
-      it "deletes existing keys" do
+      it "deletes existing keys with specified email" do
         generated_key_id
         # There is a timing issue with creating and then deleting keys.
         # Call list_keys again to give GnuPGP time.
@@ -78,7 +78,16 @@ class PgpTest < Minitest::Test
         assert IOStreams::Pgp.delete_keys(email: email, public: true, private: true)
       end
 
-      it "deletes just the private key" do
+      it "deletes existing keys with specified key_id" do
+        generated_key_id
+
+        # There is a timing issue with creating and then deleting keys.
+        # Call list_keys again to give GnuPGP time.
+        IOStreams::Pgp.list_keys(key_id: generated_key_id, private: true)
+        assert IOStreams::Pgp.delete_keys(key_id: generated_key_id, public: true, private: true)
+      end
+
+      it "deletes just the private key with specified email" do
         generated_key_id
         # There is a timing issue with creating and then deleting keys.
         # Call list_keys again to give GnuPGP time.
@@ -87,6 +96,16 @@ class PgpTest < Minitest::Test
         refute IOStreams::Pgp.key?(key_id: generated_key_id, private: true)
         assert IOStreams::Pgp.key?(key_id: generated_key_id, private: false)
       end
+
+      it "deletes just the private key with specified key_id" do
+        generated_key_id
+        # There is a timing issue with creating and then deleting keys.
+        # Call list_keys again to give GnuPGP time.
+        IOStreams::Pgp.list_keys(key_id: generated_key_id, private: true)
+        assert IOStreams::Pgp.delete_keys(key_id: generated_key_id, public: false, private: true)
+        refute IOStreams::Pgp.key?(key_id: generated_key_id, private: true)
+        assert IOStreams::Pgp.key?(key_id: generated_key_id, private: false)
+      end
     end
 
     describe ".export" do
@@ -113,7 +132,7 @@ class PgpTest < Minitest::Test
         IOStreams::Pgp.list_keys(email: email)
       end
 
-      it "lists public keys" do
+      it "lists public keys for email" do
         assert keys = IOStreams::Pgp.list_keys(email: email)
         assert_equal 1, keys.size
         assert key = keys.first
@@ -130,7 +149,24 @@ class PgpTest < Minitest::Test
         assert_equal "ultimate", key[:trust] if (ver.to_f >= 2) && (maint >= 30)
       end
 
-      it "lists private keys" do
+      it "lists public keys for key_id" do
+        assert keys = IOStreams::Pgp.list_keys(key_id: generated_key_id)
+        assert_equal 1, keys.size
+        assert key = keys.first
+
+        assert_equal Date.today, key[:date]
+        assert_equal email, key[:email]
+        assert_includes key[:key_id], generated_key_id
+        assert_equal 1024, key[:key_length]
+        assert_includes %w[R rsa], key[:key_type]
+        assert_equal user_name, key[:name]
+        refute key[:private], key
+        ver   = IOStreams::Pgp.pgp_version
+        maint = ver.split(".").last.to_i
+        assert_equal "ultimate", key[:trust] if (ver.to_f >= 2) && (maint >= 30)
+      end
+
+      it "lists private keys for email" do
         assert keys = IOStreams::Pgp.list_keys(email: email, private: true)
         assert_equal 1, keys.size
         assert key = keys.first
@@ -143,6 +179,20 @@ class PgpTest < Minitest::Test
         assert_equal user_name, key[:name]
         assert key[:private], key
       end
+
+      it "lists private keys for key_id" do
+        assert keys = IOStreams::Pgp.list_keys(key_id: generated_key_id, private: true)
+        assert_equal 1, keys.size
+        assert key = keys.first
+
+        assert_equal Date.today, key[:date]
+        assert_equal email, key[:email]
+        assert_includes key[:key_id], generated_key_id
+        assert_equal 1024, key[:key_length]
+        assert_includes %w[R rsa], key[:key_type]
+        assert_equal user_name, key[:name]
+        assert key[:private], key
+      end
     end
 
     describe ".key_info" do

data/test/pgp_writer_test.rb

@@ -53,9 +53,9 @@ class PgpWriterTest < Minitest::Test
       end
 
       it "supports multiple recipients" do
-          IOStreams::Pgp::Writer.file(file_name, recipient: %w[receiver@example.org receiver2@example.org], signer: "sender@example.org", signer_passphrase: "sender_passphrase") do |io|
-            io.write(decrypted)
-          end
+        IOStreams::Pgp::Writer.file(file_name, recipient: %w[receiver@example.org receiver2@example.org], signer: "sender@example.org", signer_passphrase: "sender_passphrase") do |io|
+          io.write(decrypted)
+        end
 
         result = IOStreams::Pgp::Reader.file(file_name, passphrase: "receiver_passphrase", &:read)
         assert_equal decrypted, result

data/test/stream_test.rb

@@ -45,9 +45,9 @@ class StreamTest < Minitest::Test
       it "reads a zip file" do
         File.open(multiple_zip_file_name, "rb") do |io|
           result = IOStreams::Stream.new(io).
-                   file_name(multiple_zip_file_name).
-                   option(:zip, entry_file_name: "test.json").
-                   read
+            file_name(multiple_zip_file_name).
+            option(:zip, entry_file_name: "test.json").
+            read
           assert_equal contents_test_json, result
         end
       end
@@ -55,8 +55,8 @@ class StreamTest < Minitest::Test
       it "reads a zip file from within a gz file" do
         File.open(zip_gz_file_name, "rb") do |io|
           result = IOStreams::Stream.new(io).
-                   file_name(zip_gz_file_name).
-                   read
+            file_name(zip_gz_file_name).
+            read
           assert_equal contents_test_txt, result
         end
       end
@@ -71,7 +71,7 @@ class StreamTest < Minitest::Test
     describe ".record_reader" do
     end
 
-    describe ".each_line" do
+    describe "#each(:line)" do
       it "returns a line at a time" do
         lines = []
         stream.stream(:none)
@@ -91,10 +91,114 @@ class StreamTest < Minitest::Test
       end
     end
 
-    describe ".each row" do
+    describe "#each(:array)" do
+      describe "csv" do
+        let :source_file_name do
+          File.join(__dir__, "files", "test.csv")
+        end
+
+        let :expected_rows do
+          rows = []
+          CSV.open(source_file_name).each { |row| rows << row }
+          rows
+        end
+
+        it "detects format from file_name" do
+          output           = []
+          stream.file_name = source_file_name
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+
+        it "honors format" do
+          output           = []
+          stream.file_name = "blah"
+          stream.format    = :csv
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+      end
+
+      describe "psv" do
+        let :source_file_name do
+          File.join(__dir__, "files", "test.psv")
+        end
+
+        let :expected_rows do
+          File.readlines(source_file_name).collect { |line| line.chomp.split("|") }
+        end
+
+        it "detects format from file_name" do
+          output           = []
+          stream.file_name = source_file_name
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+
+        it "honors format" do
+          output           = []
+          stream.file_name = "blah"
+          stream.format    = :psv
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+      end
+
+      describe "json" do
+        let :source_file_name do
+          File.join(__dir__, "files", "test.json")
+        end
+
+        let :expected_rows do
+          hash_rows = File.readlines(source_file_name).collect { |line| JSON.load(line) }
+          rows      = []
+          rows << hash_rows.first.keys
+          hash_rows.each { |hash| rows << hash.values }
+          rows
+        end
+
+        it "detects format from file_name" do
+          skip "TODO: Support reading json files as arrays"
+          output           = []
+          stream.file_name = source_file_name
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+
+        it "honors format" do
+          skip "TODO: Support reading json files as arrays"
+          output           = []
+          stream.file_name = "blah"
+          stream.format    = :json
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+      end
     end
 
-    describe ".each record" do
+    describe ".each hash" do
+      let :source_file_name do
+        File.join(__dir__, "files", "test.json")
+      end
+
+      let :expected_json do
+        File.readlines(source_file_name).collect { |line| JSON.load(line) }
+      end
+
+      it "detects format from file_name" do
+        output           = []
+        stream.file_name = source_file_name
+        stream.each(:hash) { |record| output << record }
+        assert_equal expected_json, output
+      end
+
+      it "honors format" do
+        output           = []
+        stream.file_name = "blah"
+        stream.format    = :json
+        stream.each(:hash) { |record| output << record }
+        assert_equal expected_json, output
+      end
     end
 
     describe "#writer" do
@@ -359,6 +463,24 @@ class StreamTest < Minitest::Test
           end
           assert_equal "\nHe\n\nl\n\nlo \nWorld\n\n", io.string, io.string.inspect
         end
+
+        it "honors format" do
+          io = StringIO.new
+          IOStreams::Stream.new(io).format(:psv).writer(:array) do |stream|
+            stream << %w[first_name last_name]
+            stream << %w[Jack Johnson]
+          end
+          assert_equal "first_name|last_name\nJack|Johnson\n", io.string, io.string.inspect
+        end
+
+        it "auto detects format" do
+          io = StringIO.new
+          IOStreams::Stream.new(io).file_name("abc.psv").writer(:array) do |stream|
+            stream << %w[first_name last_name]
+            stream << %w[Jack Johnson]
+          end
+          assert_equal "first_name|last_name\nJack|Johnson\n", io.string, io.string.inspect
+        end
       end
     end
 
@@ -402,6 +524,50 @@ class StreamTest < Minitest::Test
           end
           assert_equal "first_name,last_name\nJack,Johnson\n\n{:first_name=>\"Able\", :last_name=>\"Smith\"}\n\n", io.string, io.string.inspect
         end
+
+        it "honors format" do
+          io = StringIO.new
+          IOStreams::Stream.new(io).format(:json).writer(:hash) do |stream|
+            stream << {first_name: "Jack", last_name: "Johnson"}
+          end
+          assert_equal "{\"first_name\":\"Jack\",\"last_name\":\"Johnson\"}\n", io.string, io.string.inspect
+        end
+
+        it "auto detects format" do
+          io = StringIO.new
+          IOStreams::Stream.new(io).file_name("abc.json").writer(:hash) do |stream|
+            stream << {first_name: "Jack", last_name: "Johnson"}
+          end
+          assert_equal "{\"first_name\":\"Jack\",\"last_name\":\"Johnson\"}\n", io.string, io.string.inspect
+        end
+      end
+    end
+
+    describe "#format" do
+      it "detects the format from the file name" do
+        stream.file_name = "abc.json"
+        assert_equal :json, stream.format
+      end
+
+      it "is nil if the file name has no meaningful format" do
+        assert_nil stream.format
+      end
+
+      it "returns set format with no file_name" do
+        stream.format = :csv
+        assert_equal :csv, stream.format
+      end
+
+      it "returns set format with file_name" do
+        stream.file_name = "abc.json"
+        stream.format    = :csv
+        assert_equal :csv, stream.format
+      end
+
+      it "validates bad format" do
+        assert_raises ArgumentError do
+          stream.format = :blah
+        end
       end
     end
   end