iostreams 1.5.0 → 1.7.0

data/test/files/unclosed_quote_test2.csv

@@ -0,0 +1,3 @@
+name, description, zip
+Jack,Firstn"ame is Jack,234567
+John,Firstname is John,234568

data/test/line_reader_test.rb

@@ -14,6 +14,14 @@ class LineReaderTest < Minitest::Test
       File.join(File.dirname(__FILE__), "files", "unclosed_quote_test.csv")
     end
 
+    let :unclosed_quote_file2 do
+      File.join(File.dirname(__FILE__), "files", "unclosed_quote_test2.csv")
+    end
+
+    let :unclosed_quote_large_file do
+      File.join(File.dirname(__FILE__), "files", "unclosed_quote_large_test.csv")
+    end
+
     let :data do
       data = []
       File.open(file_name, "rt") do |file|
@@ -51,13 +59,31 @@ class LineReaderTest < Minitest::Test
           assert_equal 4, lines.count
         end
 
-        it "raises error for unclosed quote" do
-          assert_raises(RuntimeError) do
+        it "raises error for unbalanced quotes" do
+          exc = assert_raises(IOStreams::Errors::MalformedDataError) do
             IOStreams::Line::Reader.file(unclosed_quote_file, embedded_within: '"') do |io|
-              io.each do |line|
-              end
+              io.each { |line| }
+            end
+          end
+          assert_includes exc.message, "Unbalanced delimited field, delimiter:"
+        end
+
+        it "raises error for unclosed quote" do
+          exc = assert_raises(IOStreams::Errors::MalformedDataError) do
+            IOStreams::Line::Reader.file(unclosed_quote_file2, embedded_within: '"') do |io|
+              io.each { |line| }
+            end
+          end
+          assert_includes exc.message, "Unbalanced delimited field, delimiter:"
+        end
+
+        it "raises error for unclosed quote before eof" do
+          exc = assert_raises(IOStreams::Errors::MalformedDataError) do
+            IOStreams::Line::Reader.file(unclosed_quote_large_file, embedded_within: '"', buffer_size: 20) do |io|
+              io.each { |line| }
             end
           end
+          assert_includes exc.message, "Unbalanced delimited field, delimiter:"
         end
       end
     end

data/test/paths/file_test.rb

@@ -126,15 +126,13 @@ module Paths
 
         it "missing source file" do
           IOStreams.temp_file("iostreams_move_test", ".txt") do |temp_file|
-            begin
-              refute temp_file.exist?
-              target = temp_file.directory.join("move_test.txt")
-              assert_raises Errno::ENOENT do
-                temp_file.move_to(target)
-              end
-              refute target.exist?
-              refute temp_file.exist?
+            refute temp_file.exist?
+            target = temp_file.directory.join("move_test.txt")
+            assert_raises Errno::ENOENT do
+              temp_file.move_to(target)
             end
+            refute target.exist?
+            refute temp_file.exist?
           end
         end
 

data/test/paths/sftp_test.rb

@@ -20,7 +20,13 @@ module Paths
       let(:file_name) { File.join(File.dirname(__FILE__), "..", "files", "text file.txt") }
       let(:raw) { File.read(file_name) }
 
-      let(:root_path) { IOStreams::Paths::SFTP.new(url, username: username, password: password) }
+      let(:root_path) do
+        if ENV["SFTP_HOST_KEY"]
+          IOStreams::Paths::SFTP.new(url, username: username, password: password, ssh_options: {"HostKey" => ENV["SFTP_HOST_KEY"]})
+        else
+          IOStreams::Paths::SFTP.new(url, username: username, password: password)
+        end
+      end
 
       let :existing_path do
         path = root_path.join("test.txt")

data/test/stream_test.rb

@@ -71,7 +71,7 @@ class StreamTest < Minitest::Test
     describe ".record_reader" do
     end
 
-    describe ".each_line" do
+    describe "#each(:line)" do
       it "returns a line at a time" do
         lines = []
         stream.stream(:none)
@@ -91,10 +91,114 @@ class StreamTest < Minitest::Test
       end
     end
 
-    describe ".each row" do
+    describe "#each(:array)" do
+      describe "csv" do
+        let :source_file_name do
+          File.join(__dir__, "files", "test.csv")
+        end
+
+        let :expected_rows do
+          rows = []
+          CSV.open(source_file_name).each { |row| rows << row }
+          rows
+        end
+
+        it "detects format from file_name" do
+          output           = []
+          stream.file_name = source_file_name
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+
+        it "honors format" do
+          output           = []
+          stream.file_name = "blah"
+          stream.format    = :csv
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+      end
+
+      describe "psv" do
+        let :source_file_name do
+          File.join(__dir__, "files", "test.psv")
+        end
+
+        let :expected_rows do
+          File.readlines(source_file_name).collect { |line| line.chomp.split("|") }
+        end
+
+        it "detects format from file_name" do
+          output           = []
+          stream.file_name = source_file_name
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+
+        it "honors format" do
+          output           = []
+          stream.file_name = "blah"
+          stream.format    = :psv
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+      end
+
+      describe "json" do
+        let :source_file_name do
+          File.join(__dir__, "files", "test.json")
+        end
+
+        let :expected_rows do
+          hash_rows = File.readlines(source_file_name).collect { |line| JSON.load(line) }
+          rows      = []
+          rows << hash_rows.first.keys
+          hash_rows.each { |hash| rows << hash.values }
+          rows
+        end
+
+        it "detects format from file_name" do
+          skip "TODO: Support reading json files as arrays"
+          output           = []
+          stream.file_name = source_file_name
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+
+        it "honors format" do
+          skip "TODO: Support reading json files as arrays"
+          output           = []
+          stream.file_name = "blah"
+          stream.format    = :json
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+      end
     end
 
-    describe ".each record" do
+    describe ".each hash" do
+      let :source_file_name do
+        File.join(__dir__, "files", "test.json")
+      end
+
+      let :expected_json do
+        File.readlines(source_file_name).collect { |line| JSON.load(line) }
+      end
+
+      it "detects format from file_name" do
+        output           = []
+        stream.file_name = source_file_name
+        stream.each(:hash) { |record| output << record }
+        assert_equal expected_json, output
+      end
+
+      it "honors format" do
+        output           = []
+        stream.file_name = "blah"
+        stream.format    = :json
+        stream.each(:hash) { |record| output << record }
+        assert_equal expected_json, output
+      end
     end
 
     describe "#writer" do
@@ -359,6 +463,24 @@ class StreamTest < Minitest::Test
           end
           assert_equal "\nHe\n\nl\n\nlo \nWorld\n\n", io.string, io.string.inspect
         end
+
+        it "honors format" do
+          io = StringIO.new
+          IOStreams::Stream.new(io).format(:psv).writer(:array) do |stream|
+            stream << %w[first_name last_name]
+            stream << %w[Jack Johnson]
+          end
+          assert_equal "first_name|last_name\nJack|Johnson\n", io.string, io.string.inspect
+        end
+
+        it "auto detects format" do
+          io = StringIO.new
+          IOStreams::Stream.new(io).file_name("abc.psv").writer(:array) do |stream|
+            stream << %w[first_name last_name]
+            stream << %w[Jack Johnson]
+          end
+          assert_equal "first_name|last_name\nJack|Johnson\n", io.string, io.string.inspect
+        end
       end
     end
 
@@ -402,6 +524,50 @@ class StreamTest < Minitest::Test
           end
           assert_equal "first_name,last_name\nJack,Johnson\n\n{:first_name=>\"Able\", :last_name=>\"Smith\"}\n\n", io.string, io.string.inspect
         end
+
+        it "honors format" do
+          io = StringIO.new
+          IOStreams::Stream.new(io).format(:json).writer(:hash) do |stream|
+            stream << {first_name: "Jack", last_name: "Johnson"}
+          end
+          assert_equal "{\"first_name\":\"Jack\",\"last_name\":\"Johnson\"}\n", io.string, io.string.inspect
+        end
+
+        it "auto detects format" do
+          io = StringIO.new
+          IOStreams::Stream.new(io).file_name("abc.json").writer(:hash) do |stream|
+            stream << {first_name: "Jack", last_name: "Johnson"}
+          end
+          assert_equal "{\"first_name\":\"Jack\",\"last_name\":\"Johnson\"}\n", io.string, io.string.inspect
+        end
+      end
+    end
+
+    describe "#format" do
+      it "detects the format from the file name" do
+        stream.file_name = "abc.json"
+        assert_equal :json, stream.format
+      end
+
+      it "is nil if the file name has no meaningful format" do
+        assert_nil stream.format
+      end
+
+      it "returns set format with no file_name" do
+        stream.format = :csv
+        assert_equal :csv, stream.format
+      end
+
+      it "returns set format with file_name" do
+        stream.file_name = "abc.json"
+        stream.format    = :csv
+        assert_equal :csv, stream.format
+      end
+
+      it "validates bad format" do
+        assert_raises ArgumentError do
+          stream.format = :blah
+        end
       end
     end
   end

data/test/test_helper.rb

@@ -2,7 +2,6 @@ $LOAD_PATH.unshift File.dirname(__FILE__) + "/../lib"
 
 require "yaml"
 require "minitest/autorun"
-require "minitest/reporters"
 require "iostreams"
 require "amazing_print"
 require "symmetric-encryption"
@@ -10,8 +9,6 @@ require "symmetric-encryption"
 # Since PGP libraries use UTC for Dates
 ENV["TZ"] = "UTC"
 
-Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
-
 # Test cipher
 SymmetricEncryption.cipher = SymmetricEncryption::Cipher.new(
   cipher_name: "aes-128-cbc",

metadata

@@ -1,18 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: iostreams
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.7.0
 platform: ruby
 authors:
 - Reid Morrison
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-10 00:00:00.000000000 Z
+date: 2021-06-23 00:00:00.000000000 Z
 dependencies: []
-description: 
+description:
 email:
-- reidmo@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -77,6 +76,7 @@ files:
 - test/files/spreadsheet.xlsx
 - test/files/test.csv
 - test/files/test.json
+- test/files/test.psv
 - test/files/text file.txt
 - test/files/text.txt
 - test/files/text.txt.bz2
@@ -84,7 +84,9 @@ files:
 - test/files/text.txt.gz.zip
 - test/files/text.zip
 - test/files/text.zip.gz
+- test/files/unclosed_quote_large_test.csv
 - test/files/unclosed_quote_test.csv
+- test/files/unclosed_quote_test2.csv
 - test/gzip_reader_test.rb
 - test/gzip_writer_test.rb
 - test/io_streams_test.rb
@@ -111,11 +113,11 @@ files:
 - test/xlsx_reader_test.rb
 - test/zip_reader_test.rb
 - test/zip_writer_test.rb
-homepage: https://github.com/rocketjob/iostreams
+homepage: https://iostreams.rocketjob.io
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -123,60 +125,63 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: Input and Output streaming for Ruby.
 test_files:
-- test/pgp_reader_test.rb
-- test/deprecated_test.rb
 - test/builder_test.rb
-- test/line_reader_test.rb
-- test/xlsx_reader_test.rb
-- test/minimal_file_reader.rb
-- test/row_writer_test.rb
-- test/zip_reader_test.rb
+- test/bzip2_reader_test.rb
 - test/bzip2_writer_test.rb
+- test/deprecated_test.rb
+- test/encode_reader_test.rb
 - test/encode_writer_test.rb
-- test/gzip_writer_test.rb
-- test/stream_test.rb
-- test/paths/matcher_test.rb
-- test/paths/s3_test.rb
-- test/paths/sftp_test.rb
-- test/paths/file_test.rb
-- test/paths/http_test.rb
-- test/record_reader_test.rb
-- test/pgp_writer_test.rb
-- test/line_writer_test.rb
-- test/row_reader_test.rb
-- test/bzip2_reader_test.rb
-- test/zip_writer_test.rb
-- test/files/text.zip
-- test/files/spreadsheet.xlsx
 - test/files/embedded_lines_test.csv
+- test/files/multiple_files.zip
+- test/files/spreadsheet.xlsx
 - test/files/test.csv
 - test/files/test.json
-- test/files/text.zip.gz
-- test/files/unclosed_quote_test.csv
+- test/files/test.psv
+- test/files/text file.txt
+- test/files/text.txt
 - test/files/text.txt.bz2
-- test/files/text.txt.gz.zip
 - test/files/text.txt.gz
-- test/files/text.txt
-- test/files/multiple_files.zip
-- test/files/text file.txt
+- test/files/text.txt.gz.zip
+- test/files/text.zip
+- test/files/text.zip.gz
+- test/files/unclosed_quote_large_test.csv
+- test/files/unclosed_quote_test.csv
+- test/files/unclosed_quote_test2.csv
 - test/gzip_reader_test.rb
-- test/encode_reader_test.rb
-- test/test_helper.rb
-- test/utils_test.rb
-- test/tabular_test.rb
+- test/gzip_writer_test.rb
+- test/io_streams_test.rb
+- test/line_reader_test.rb
+- test/line_writer_test.rb
+- test/minimal_file_reader.rb
 - test/path_test.rb
+- test/paths/file_test.rb
+- test/paths/http_test.rb
+- test/paths/matcher_test.rb
+- test/paths/s3_test.rb
+- test/paths/sftp_test.rb
+- test/pgp_reader_test.rb
 - test/pgp_test.rb
-- test/io_streams_test.rb
+- test/pgp_writer_test.rb
+- test/record_reader_test.rb
 - test/record_writer_test.rb
+- test/row_reader_test.rb
+- test/row_writer_test.rb
+- test/stream_test.rb
+- test/tabular_test.rb
+- test/test_helper.rb
+- test/utils_test.rb
+- test/xlsx_reader_test.rb
+- test/zip_reader_test.rb
+- test/zip_writer_test.rb