iostreams 1.4.0 → 1.6.2

data/test/files/unclosed_quote_test2.csv

@@ -0,0 +1,3 @@
+name, description, zip
+Jack,Firstn"ame is Jack,234567
+John,Firstname is John,234568

data/test/line_reader_test.rb

@@ -14,6 +14,14 @@ class LineReaderTest < Minitest::Test
       File.join(File.dirname(__FILE__), "files", "unclosed_quote_test.csv")
     end
 
+    let :unclosed_quote_file2 do
+      File.join(File.dirname(__FILE__), "files", "unclosed_quote_test2.csv")
+    end
+
+    let :unclosed_quote_large_file do
+      File.join(File.dirname(__FILE__), "files", "unclosed_quote_large_test.csv")
+    end
+
     let :data do
       data = []
       File.open(file_name, "rt") do |file|
@@ -51,13 +59,31 @@ class LineReaderTest < Minitest::Test
           assert_equal 4, lines.count
         end
 
-        it "raises error for unclosed quote" do
-          assert_raises(RuntimeError) do
+        it "raises error for unbalanced quotes" do
+          exc = assert_raises(IOStreams::Errors::MalformedDataError) do
             IOStreams::Line::Reader.file(unclosed_quote_file, embedded_within: '"') do |io|
-              io.each do |line|
-              end
+              io.each { |line| }
+            end
+          end
+          assert_includes exc.message, "Unbalanced delimited field, delimiter:"
+        end
+
+        it "raises error for unclosed quote" do
+          exc = assert_raises(IOStreams::Errors::MalformedDataError) do
+            IOStreams::Line::Reader.file(unclosed_quote_file2, embedded_within: '"') do |io|
+              io.each { |line| }
+            end
+          end
+          assert_includes exc.message, "Unbalanced delimited field, delimiter:"
+        end
+
+        it "raises error for unclosed quote before eof" do
+          exc = assert_raises(IOStreams::Errors::MalformedDataError) do
+            IOStreams::Line::Reader.file(unclosed_quote_large_file, embedded_within: '"', buffer_size: 20) do |io|
+              io.each { |line| }
             end
           end
+          assert_includes exc.message, "Unbalanced delimited field, delimiter:"
         end
       end
     end

data/test/stream_test.rb

@@ -45,9 +45,9 @@ class StreamTest < Minitest::Test
       it "reads a zip file" do
         File.open(multiple_zip_file_name, "rb") do |io|
           result = IOStreams::Stream.new(io).
-                   file_name(multiple_zip_file_name).
-                   option(:zip, entry_file_name: "test.json").
-                   read
+            file_name(multiple_zip_file_name).
+            option(:zip, entry_file_name: "test.json").
+            read
           assert_equal contents_test_json, result
         end
       end
@@ -55,8 +55,8 @@ class StreamTest < Minitest::Test
       it "reads a zip file from within a gz file" do
         File.open(zip_gz_file_name, "rb") do |io|
           result = IOStreams::Stream.new(io).
-                   file_name(zip_gz_file_name).
-                   read
+            file_name(zip_gz_file_name).
+            read
           assert_equal contents_test_txt, result
         end
       end
@@ -71,7 +71,7 @@ class StreamTest < Minitest::Test
     describe ".record_reader" do
     end
 
-    describe ".each_line" do
+    describe "#each(:line)" do
       it "returns a line at a time" do
         lines = []
         stream.stream(:none)
@@ -91,10 +91,114 @@ class StreamTest < Minitest::Test
       end
     end
 
-    describe ".each row" do
+    describe "#each(:array)" do
+      describe "csv" do
+        let :source_file_name do
+          File.join(__dir__, "files", "test.csv")
+        end
+
+        let :expected_rows do
+          rows = []
+          CSV.open(source_file_name).each { |row| rows << row }
+          rows
+        end
+
+        it "detects format from file_name" do
+          output           = []
+          stream.file_name = source_file_name
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+
+        it "honors format" do
+          output           = []
+          stream.file_name = "blah"
+          stream.format    = :csv
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+      end
+
+      describe "psv" do
+        let :source_file_name do
+          File.join(__dir__, "files", "test.psv")
+        end
+
+        let :expected_rows do
+          File.readlines(source_file_name).collect { |line| line.chomp.split("|") }
+        end
+
+        it "detects format from file_name" do
+          output           = []
+          stream.file_name = source_file_name
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+
+        it "honors format" do
+          output           = []
+          stream.file_name = "blah"
+          stream.format    = :psv
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+      end
+
+      describe "json" do
+        let :source_file_name do
+          File.join(__dir__, "files", "test.json")
+        end
+
+        let :expected_rows do
+          hash_rows = File.readlines(source_file_name).collect { |line| JSON.load(line) }
+          rows      = []
+          rows << hash_rows.first.keys
+          hash_rows.each { |hash| rows << hash.values }
+          rows
+        end
+
+        it "detects format from file_name" do
+          skip "TODO: Support reading json files as arrays"
+          output           = []
+          stream.file_name = source_file_name
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+
+        it "honors format" do
+          skip "TODO: Support reading json files as arrays"
+          output           = []
+          stream.file_name = "blah"
+          stream.format    = :json
+          stream.each(:array) { |record| output << record }
+          assert_equal expected_rows, output
+        end
+      end
     end
 
-    describe ".each record" do
+    describe ".each hash" do
+      let :source_file_name do
+        File.join(__dir__, "files", "test.json")
+      end
+
+      let :expected_json do
+        File.readlines(source_file_name).collect { |line| JSON.load(line) }
+      end
+
+      it "detects format from file_name" do
+        output           = []
+        stream.file_name = source_file_name
+        stream.each(:hash) { |record| output << record }
+        assert_equal expected_json, output
+      end
+
+      it "honors format" do
+        output           = []
+        stream.file_name = "blah"
+        stream.format    = :json
+        stream.each(:hash) { |record| output << record }
+        assert_equal expected_json, output
+      end
     end
 
     describe "#writer" do
@@ -359,6 +463,24 @@ class StreamTest < Minitest::Test
           end
           assert_equal "\nHe\n\nl\n\nlo \nWorld\n\n", io.string, io.string.inspect
         end
+
+        it "honors format" do
+          io = StringIO.new
+          IOStreams::Stream.new(io).format(:psv).writer(:array) do |stream|
+            stream << %w[first_name last_name]
+            stream << %w[Jack Johnson]
+          end
+          assert_equal "first_name|last_name\nJack|Johnson\n", io.string, io.string.inspect
+        end
+
+        it "auto detects format" do
+          io = StringIO.new
+          IOStreams::Stream.new(io).file_name("abc.psv").writer(:array) do |stream|
+            stream << %w[first_name last_name]
+            stream << %w[Jack Johnson]
+          end
+          assert_equal "first_name|last_name\nJack|Johnson\n", io.string, io.string.inspect
+        end
       end
     end
 
@@ -402,6 +524,50 @@ class StreamTest < Minitest::Test
           end
           assert_equal "first_name,last_name\nJack,Johnson\n\n{:first_name=>\"Able\", :last_name=>\"Smith\"}\n\n", io.string, io.string.inspect
         end
+
+        it "honors format" do
+          io = StringIO.new
+          IOStreams::Stream.new(io).format(:json).writer(:hash) do |stream|
+            stream << {first_name: "Jack", last_name: "Johnson"}
+          end
+          assert_equal "{\"first_name\":\"Jack\",\"last_name\":\"Johnson\"}\n", io.string, io.string.inspect
+        end
+
+        it "auto detects format" do
+          io = StringIO.new
+          IOStreams::Stream.new(io).file_name("abc.json").writer(:hash) do |stream|
+            stream << {first_name: "Jack", last_name: "Johnson"}
+          end
+          assert_equal "{\"first_name\":\"Jack\",\"last_name\":\"Johnson\"}\n", io.string, io.string.inspect
+        end
+      end
+    end
+
+    describe "#format" do
+      it "detects the format from the file name" do
+        stream.file_name = "abc.json"
+        assert_equal :json, stream.format
+      end
+
+      it "is nil if the file name has no meaningful format" do
+        assert_nil stream.format
+      end
+
+      it "returns set format with no file_name" do
+        stream.format = :csv
+        assert_equal :csv, stream.format
+      end
+
+      it "returns set format with file_name" do
+        stream.file_name = "abc.json"
+        stream.format    = :csv
+        assert_equal :csv, stream.format
+      end
+
+      it "validates bad format" do
+        assert_raises ArgumentError do
+          stream.format = :blah
+        end
       end
     end
   end

data/test/tabular_test.rb

@@ -10,6 +10,36 @@ class TabularTest < Minitest::Test
       IOStreams::Tabular.new(columns: %w[first_field second third], format: format)
     end
 
+    let :fixed do
+      layout = [
+        {size: 23, key: :name},
+        {size: 40, key: :address},
+        {size: 2},
+        {size: 5, key: :zip, type: :integer},
+        {size: 8, key: :age, type: :integer},
+        {size: 10, key: :weight, type: :float, decimals: 2}
+      ]
+      IOStreams::Tabular.new(format: :fixed, format_options: {layout: layout})
+    end
+
+    let :fixed_with_remainder do
+      layout = [
+        {size: 23, key: :name},
+        {size: 40, key: :address},
+        {size: :remainder, key: :remainder}
+      ]
+      IOStreams::Tabular.new(format: :fixed, format_options: {layout: layout})
+    end
+
+    let :fixed_discard_remainder do
+      layout = [
+        {size: 23, key: :name},
+        {size: 40, key: :address},
+        {size: :remainder}
+      ]
+      IOStreams::Tabular.new(format: :fixed, format_options: {layout: layout})
+    end
+
     describe "#parse_header" do
       it "parses and sets the csv header" do
         tabular = IOStreams::Tabular.new(format: :csv)
@@ -136,57 +166,55 @@ class TabularTest < Minitest::Test
       end
 
       describe ":fixed format" do
-        let :tabular do
-          layout = [
-            {size: 23, key: :name},
-            {size: 40, key: :address},
-            {size: 2},
-            {size: 5, key: :zip, type: :integer},
-            {size: 8, key: :age, type: :integer},
-            {size: 10, key: :weight, type: :float, decimals: 2}
-          ]
-          IOStreams::Tabular.new(format: :fixed, format_options: {layout: layout})
-        end
-
         it "parses to hash" do
-          assert hash = tabular.record_parse("Jack                   over there                              XX34618012345670012345.01")
+          assert hash = fixed.record_parse("Jack                   over there                              XX34618012345670012345.01")
           assert_equal({name: "Jack", address: "over there", zip: 34_618, age: 1_234_567, weight: 12_345.01}, hash)
         end
 
         it "parses short string" do
           assert_raises IOStreams::Errors::InvalidLineLength do
-            tabular.record_parse("Jack                   over th")
+            fixed.record_parse("Jack                   over th")
           end
         end
 
         it "parses longer string" do
           assert_raises IOStreams::Errors::InvalidLineLength do
-            tabular.record_parse("Jack                   over there                              XX34618012345670012345.01............")
+            fixed.record_parse("Jack                   over there                              XX34618012345670012345.01............")
           end
         end
 
         it "parses zero values" do
-          assert hash = tabular.record_parse("                                                                 00000000000000000000000")
+          assert hash = fixed.record_parse("                                                                 00000000000000000000000")
           assert_equal({name: "", address: "", zip: 0, age: 0, weight: 0.0}, hash)
         end
 
         it "parses empty values" do
-          assert hash = tabular.record_parse("                                                               XX                       ")
+          assert hash = fixed.record_parse("                                                               XX                       ")
           assert_equal({name: "", address: "", zip: nil, age: nil, weight: nil}, hash)
         end
 
         it "parses blank strings" do
-          skip "TODO: Part of tabular refactor to get this working"
-          assert hash = tabular.record_parse("                                                                                        ")
+          skip "TODO: Part of fixed refactor to get this working"
+          assert hash = fixed.record_parse("                                                                                        ")
           assert_equal({name: "", address: "", zip: nil, age: nil, weight: nil}, hash)
         end
 
         it "parses nil data as nil" do
-          refute tabular.record_parse(nil)
+          refute fixed.record_parse(nil)
         end
 
         it "parses empty string as nil" do
-          refute tabular.record_parse("")
+          refute fixed.record_parse("")
+        end
+
+        it "parses remainder" do
+          hash = fixed_with_remainder.record_parse("Jack                   over there                              XX34618012345670012345.01............")
+          assert_equal({name: "Jack", address: "over there", remainder: "XX34618012345670012345.01............"}, hash)
+        end
+
+        it "discards remainder" do
+          hash = fixed_discard_remainder.record_parse("Jack                   over there                              XX34618012345670012345.01............")
+          assert_equal({name: "Jack", address: "over there"}, hash)
         end
       end
 
@@ -236,52 +264,55 @@ class TabularTest < Minitest::Test
       end
 
       describe ":fixed format" do
-        let :tabular do
-          layout = [
-            {size: 23, key: :name},
-            {size: 40, key: :address},
-            {size: 2},
-            {size: 5, key: :zip, type: :integer},
-            {size: 8, key: :age, type: :integer},
-            {size: 10, key: :weight, type: :float, decimals: 2}
-          ]
-          IOStreams::Tabular.new(format: :fixed, format_options: {layout: layout})
-        end
-
         it "renders fixed data" do
-          assert string = tabular.render(name: "Jack", address: "over there", zip: 34_618, weight: 123_456.789123, age: 21)
+          assert string = fixed.render(name: "Jack", address: "over there", zip: 34_618, weight: 123_456.789123, age: 21)
           assert_equal "Jack                   over there                                34618000000210123456.79", string
         end
 
         it "truncates long strings" do
-          assert string = tabular.render(name: "Jack ran up the beanstalk and when jack reached the top it was truncated", address: "over there", zip: 34_618)
+          assert string = fixed.render(name: "Jack ran up the beanstalk and when jack reached the top it was truncated", address: "over there", zip: 34_618)
           assert_equal "Jack ran up the beanstaover there                                34618000000000000000.00", string
         end
 
         it "when integer is too large" do
           assert_raises IOStreams::Errors::ValueTooLong do
-            tabular.render(zip: 3_461_832_653_653_265)
+            fixed.render(zip: 3_461_832_653_653_265)
           end
         end
 
         it "when float is too large" do
           assert_raises IOStreams::Errors::ValueTooLong do
-            tabular.render(weight: 3_461_832_653_653_265.234)
+            fixed.render(weight: 3_461_832_653_653_265.234)
           end
         end
 
         it "renders nil as empty string" do
-          assert string = tabular.render(zip: 34_618)
+          assert string = fixed.render(zip: 34_618)
           assert_equal "                                                                 34618000000000000000.00", string
         end
 
         it "renders boolean" do
-          assert string = tabular.render(name: true, address: false)
+          assert string = fixed.render(name: true, address: false)
           assert_equal "true                   false                                     00000000000000000000.00", string
         end
 
         it "renders no data as nil" do
-          refute tabular.render({})
+          refute fixed.render({})
+        end
+
+        it "any size last string" do
+          assert string = fixed_with_remainder.render(name: "Jack", address: "over there", remainder: "XX34618012345670012345.01............")
+          assert_equal "Jack                   over there                              XX34618012345670012345.01............", string
+        end
+
+        it "nil last string" do
+          assert string = fixed_with_remainder.render(name: "Jack", address: "over there", remainder: nil)
+          assert_equal "Jack                   over there                              ", string
+        end
+
+        it "skips last filler" do
+          assert string = fixed_discard_remainder.render(name: "Jack", address: "over there")
+          assert_equal "Jack                   over there                              ", string
         end
       end
     end