iostreams 1.11.0 → 2.0.0

data/test/files/unclosed_quote_test.csv

@@ -1,4 +0,0 @@
-name, description, zip
-"Jack","Firstn"ame is Jack","234567"
-"John","Firstname is John","234568"
-"Zack","Firstname is Zack","234568"

data/test/files/unclosed_quote_test2.csv

@@ -1,3 +0,0 @@
-name, description, zip
-Jack,Firstn"ame is Jack,234567
-John,Firstname is John,234568

data/test/gzip_reader_test.rb

@@ -1,27 +0,0 @@
-require_relative "test_helper"
-
-class GzipReaderTest < Minitest::Test
-  describe IOStreams::Gzip::Reader do
-    let :file_name do
-      File.join(File.dirname(__FILE__), "files", "text.txt.gz")
-    end
-
-    let :decompressed do
-      Zlib::GzipReader.open(file_name, &:read)
-    end
-
-    describe ".open" do
-      it "file" do
-        result = IOStreams::Gzip::Reader.file(file_name, &:read)
-        assert_equal decompressed, result
-      end
-
-      it "stream" do
-        result = File.open(file_name) do |file|
-          IOStreams::Gzip::Reader.stream(file, &:read)
-        end
-        assert_equal decompressed, result
-      end
-    end
-  end
-end

data/test/gzip_writer_test.rb

@@ -1,52 +0,0 @@
-require_relative "test_helper"
-
-class GzipWriterTest < Minitest::Test
-  describe IOStreams::Gzip::Writer do
-    let :temp_file do
-      Tempfile.new("iostreams")
-    end
-
-    let :file_name do
-      temp_file.path
-    end
-
-    let :decompressed do
-      File.read(File.join(File.dirname(__FILE__), "files", "text.txt"))
-    end
-
-    after do
-      temp_file.delete
-    end
-
-    describe ".file" do
-      it "file" do
-        result =
-          IOStreams::Gzip::Writer.file(file_name) do |io|
-            io.write(decompressed)
-            53534
-          end
-        assert_equal 53534, result
-
-        result = Zlib::GzipReader.open(file_name, &:read)
-        temp_file.delete
-        assert_equal decompressed, result
-      end
-
-      it "stream" do
-        io_string = StringIO.new("".b)
-        result    =
-          IOStreams::Gzip::Writer.stream(io_string) do |io|
-            io.write(decompressed)
-            53534
-          end
-        assert_equal 53534, result
-
-        io   = StringIO.new(io_string.string)
-        gz   = Zlib::GzipReader.new(io)
-        data = gz.read
-        gz.close
-        assert_equal decompressed, data
-      end
-    end
-  end
-end

data/test/io_streams_test.rb

@@ -1,132 +0,0 @@
-require_relative "test_helper"
-require "json"
-
-module IOStreams
-  class PathTest < Minitest::Test
-    describe IOStreams do
-      let :records do
-        [
-          {"name" => "Jack Jones", "login" => "jjones"},
-          {"name" => "Jill Smith", "login" => "jsmith"}
-        ]
-      end
-
-      let :expected_json do
-        records.collect(&:to_json).join("\n") + "\n"
-      end
-
-      let :json_file_name do
-        "/tmp/iostreams_abc.json"
-      end
-
-      describe ".root" do
-        it "return default path" do
-          path = ::File.expand_path(::File.join(__dir__, "../tmp/default"))
-          assert_equal path, IOStreams.root.to_s
-        end
-
-        it "return downloads path" do
-          path = ::File.expand_path(::File.join(__dir__, "../tmp/downloads"))
-          assert_equal path, IOStreams.root(:downloads).to_s
-        end
-      end
-
-      describe ".join" do
-        it "returns path" do
-          assert_equal IOStreams.root.to_s, IOStreams.join.to_s
-        end
-
-        it "adds path to root" do
-          assert_equal ::File.join(IOStreams.root.to_s, "test"), IOStreams.join("test").to_s
-        end
-
-        it "adds paths to root" do
-          assert_equal ::File.join(IOStreams.root.to_s, "test", "second", "third"), IOStreams.join("test", "second", "third").to_s
-        end
-
-        it "returns path and filename" do
-          path = ::File.join(IOStreams.root.to_s, "file.xls")
-          assert_equal path, IOStreams.join("file.xls").to_s
-        end
-
-        it "adds path to root and filename" do
-          path = ::File.join(IOStreams.root.to_s, "test", "file.xls")
-          assert_equal path, IOStreams.join("test", "file.xls").to_s
-        end
-
-        it "adds paths to root" do
-          path = ::File.join(IOStreams.root.to_s, "test", "second", "third", "file.xls")
-          assert_equal path, IOStreams.join("test", "second", "third", "file.xls").to_s
-        end
-
-        it "return path as sent in when full path" do
-          path = ::File.join(IOStreams.root.to_s, "file.xls")
-          assert_equal path, IOStreams.join(path).to_s
-        end
-      end
-
-      describe ".path" do
-        it "default" do
-          path = IOStreams.path("a.xyz")
-          assert path.is_a?(IOStreams::Paths::File), path
-        end
-
-        it "s3" do
-          skip "TODO"
-          IOStreams.path("s3://a.xyz")
-          assert_equal :s3, path
-        end
-
-        it "hash writer detects json format from file name" do
-          path = IOStreams.path("/tmp/io_streams/abc.json")
-          path.writer(:hash) do |io|
-            records.each { |hash| io << hash }
-          end
-          actual = path.read
-          path.delete
-          assert_equal expected_json, actual
-        end
-
-        it "hash reader detects json format from file name" do
-          ::File.open(json_file_name, "wb") { |file| file.write(expected_json) }
-          rows = []
-          path = IOStreams.path(json_file_name)
-          path.each(:hash) do |row|
-            rows << row
-          end
-          actual = rows.collect(&:to_json).join("\n") + "\n"
-          path.delete
-          assert_equal expected_json, actual
-        end
-
-        it "array writer detects json format from file name" do
-          path = IOStreams.path("/tmp/io_streams/abc.json")
-          path.writer(:array, columns: %w[name login]) do |io|
-            io << ["Jack Jones", "jjones"]
-            io << ["Jill Smith", "jsmith"]
-          end
-          actual = path.read
-          path.delete
-          assert_equal expected_json, actual
-        end
-      end
-
-      describe ".temp_file" do
-        it "returns value from block" do
-          result = IOStreams.temp_file("base", ".ext") { |_path| 257 }
-          assert_equal 257, result
-        end
-
-        it "supplies new temp file_name" do
-          path1 = nil
-          path2 = nil
-          IOStreams.temp_file("base", ".ext") { |path| path1 = path }
-          IOStreams.temp_file("base", ".ext") { |path| path2 = path }
-          refute_equal path1.to_s, path2.to_s
-          assert path1.is_a?(IOStreams::Paths::File), path1
-          assert path2.is_a?(IOStreams::Paths::File), path2
-        end
-      end
-    end
-  end
-end

data/test/line_reader_test.rb

@@ -1,325 +0,0 @@
-require_relative "test_helper"
-
-class LineReaderTest < Minitest::Test
-  describe IOStreams::Line::Reader do
-    let :file_name do
-      File.join(File.dirname(__FILE__), "files", "text.txt")
-    end
-
-    let :csv_file do
-      File.join(File.dirname(__FILE__), "files", "embedded_lines_test.csv")
-    end
-
-    let :unclosed_quote_file do
-      File.join(File.dirname(__FILE__), "files", "unclosed_quote_test.csv")
-    end
-
-    let :unclosed_quote_file2 do
-      File.join(File.dirname(__FILE__), "files", "unclosed_quote_test2.csv")
-    end
-
-    let :unclosed_quote_large_file do
-      File.join(File.dirname(__FILE__), "files", "unclosed_quote_large_test.csv")
-    end
-
-    let :data do
-      data = []
-      File.open(file_name, "rt") do |file|
-        data << file.readline.strip until file.eof?
-      end
-      data
-    end
-
-    # Test file has embedded new lines in row 2, 3 and 4
-    #
-    #  name, description, zip
-    # "\nJack","Firstname is Jack","234567"
-    # "John","Firstname\n is John","234568"
-    # "Zack","Firstname is Zack","234568\n"
-    #
-    describe "embedded_within_quotes" do
-      describe "csv file" do
-        it "fails to keep embedded lines if flag is not set" do
-          lines = []
-          IOStreams::Line::Reader.file(csv_file) do |io|
-            io.each do |line|
-              lines << line
-            end
-          end
-          assert_equal 7, lines.count
-        end
-
-        it "keeps embedded lines if flag is set" do
-          lines = []
-          IOStreams::Line::Reader.file(csv_file, embedded_within: '"') do |io|
-            io.each do |line|
-              lines << line
-            end
-          end
-          assert_equal 4, lines.count
-        end
-
-        it "raises error for unbalanced quotes" do
-          exc = assert_raises(IOStreams::Errors::MalformedDataError) do
-            IOStreams::Line::Reader.file(unclosed_quote_file, embedded_within: '"') do |io|
-              io.each { |line| }
-            end
-          end
-          assert_includes exc.message, "Unbalanced delimited field, delimiter:"
-        end
-
-        it "raises error for unclosed quote" do
-          exc = assert_raises(IOStreams::Errors::MalformedDataError) do
-            IOStreams::Line::Reader.file(unclosed_quote_file2, embedded_within: '"') do |io|
-              io.each { |line| }
-            end
-          end
-          assert_includes exc.message, "Unbalanced delimited field, delimiter:"
-        end
-
-        it "raises error for unclosed quote before eof" do
-          exc = assert_raises(IOStreams::Errors::MalformedDataError) do
-            IOStreams::Line::Reader.file(unclosed_quote_large_file, embedded_within: '"', buffer_size: 20) do |io|
-              io.each { |line| }
-            end
-          end
-          assert_includes exc.message, "Unbalanced delimited field, delimiter:"
-        end
-      end
-    end
-
-    describe "#each" do
-      it "each_line file" do
-        lines = []
-        count = IOStreams::Line::Reader.file(file_name) do |io|
-          io.each { |line| lines << line }
-        end
-        assert_equal data, lines
-        assert_equal data.size, count
-      end
-
-      it "each_line stream" do
-        lines = []
-        count = File.open(file_name) do |file|
-          IOStreams::Line::Reader.stream(file) do |io|
-            io.each { |line| lines << line }
-          end
-        end
-        assert_equal data, lines
-        assert_equal data.size, count
-      end
-
-      ["\r\n", "\n", "\r"].each do |delimiter|
-        it "autodetect delimiter: #{delimiter.inspect}" do
-          lines  = []
-          stream = StringIO.new(data.join(delimiter))
-          count  = IOStreams::Line::Reader.stream(stream, buffer_size: 15) do |io|
-            io.each { |line| lines << line }
-          end
-          assert_equal data, lines
-          assert_equal data.size, count
-        end
-
-        it "single read autodetect delimiter: #{delimiter.inspect}" do
-          lines  = []
-          stream = StringIO.new(data.join(delimiter))
-          count  = IOStreams::Line::Reader.stream(stream) do |io|
-            io.each { |line| lines << line }
-          end
-          assert_equal data, lines
-          assert_equal data.size, count
-        end
-      end
-
-      ["@", "BLAH"].each do |delimiter|
-        it "reads delimited #{delimiter.inspect}" do
-          lines  = []
-          stream = StringIO.new(data.join(delimiter))
-          count  = IOStreams::Line::Reader.stream(stream, buffer_size: 15, delimiter: delimiter) do |io|
-            io.each { |line| lines << line }
-          end
-          assert_equal data, lines
-          assert_equal data.size, count
-        end
-      end
-
-      it "reads binary delimited" do
-        delimiter = "\x01"
-        lines     = []
-        stream    = StringIO.new(data.join(delimiter).encode("ASCII-8BIT"))
-        count     = IOStreams::Line::Reader.stream(stream, buffer_size: 15, delimiter: delimiter) do |io|
-          io.each { |line| lines << line }
-        end
-        assert_equal data, lines
-        assert_equal data.size, count
-      end
-
-      describe "#readline" do
-        let(:short_line) { "0123456789" }
-        let(:longer_line) { "ABCDEFGHIJKLMNOPQRSTUVWXYZ" }
-        let(:delimiter) { "\r\n" }
-
-        it "reads delimiter in first block, no delimiter at end" do
-          data        = [short_line, longer_line].join(delimiter)
-          buffer_size = short_line.length + delimiter.size + (longer_line.size / 2)
-
-          stream = StringIO.new(data)
-          IOStreams::Line::Reader.stream(stream, buffer_size: buffer_size) do |io|
-            refute io.eof?
-            assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
-
-            assert_equal short_line, io.readline
-            assert_equal longer_line, io.readline
-
-            assert io.eof?
-            assert_nil io.readline
-          end
-        end
-
-        it "reads delimiter in second block, no delimiter at end" do
-          data        = [longer_line, short_line, short_line].join(delimiter)
-          buffer_size = (longer_line.length + delimiter.size + 5) / 2
-
-          stream = StringIO.new(data)
-          IOStreams::Line::Reader.stream(stream, buffer_size: buffer_size) do |io|
-            refute io.eof?
-            assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
-            assert_equal longer_line, io.readline
-            assert_equal short_line, io.readline
-            assert_equal short_line, io.readline
-            assert io.eof?
-            assert_nil io.readline
-          end
-        end
-
-        it "reads delimiter split across first and second blocks" do
-          data        = [longer_line, short_line, short_line].join(delimiter)
-          buffer_size = longer_line.length + 1
-
-          stream = StringIO.new(data)
-          IOStreams::Line::Reader.stream(stream, buffer_size: buffer_size) do |io|
-            refute io.eof?
-            assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
-            assert_equal longer_line, io.readline
-            assert_equal short_line, io.readline
-            assert_equal short_line, io.readline
-            assert io.eof?
-            assert_nil io.readline
-          end
-        end
-
-        it "reads file with no matching delimiter" do
-          delimiter   = "@"
-          data        = [longer_line, short_line, longer_line].join(delimiter) + delimiter
-          buffer_size = longer_line.length + 1
-
-          stream = StringIO.new(data)
-          IOStreams::Line::Reader.stream(stream, buffer_size: buffer_size) do |io|
-            refute io.eof?
-            assert_equal "\n", io.delimiter, -> { io.delimiter.ai }
-            assert_equal data, io.readline
-            assert io.eof?
-            assert_nil io.readline
-          end
-        end
-
-        it "reads small file with no matching delimiter" do
-          data        = short_line
-          buffer_size = short_line.length + 100
-
-          stream = StringIO.new(data)
-          IOStreams::Line::Reader.stream(stream, buffer_size: buffer_size) do |io|
-            refute io.eof?
-            assert_equal "\n", io.delimiter, -> { io.delimiter.ai }
-            assert_equal short_line, io.readline
-            assert io.eof?
-            assert_nil io.readline
-          end
-        end
-
-        it "reads last line with the delimiter as the last character" do
-          delimiter   = "@"
-          data        = [longer_line, short_line, longer_line].join(delimiter) + delimiter
-          buffer_size = longer_line.length + 1
-
-          stream = StringIO.new(data)
-          IOStreams::Line::Reader.stream(stream, buffer_size: buffer_size, delimiter: delimiter) do |io|
-            refute io.eof?
-            assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
-            assert_equal longer_line, io.readline
-            assert_equal short_line, io.readline
-            assert_equal longer_line, io.readline
-            assert_nil io.readline
-            assert io.eof?
-          end
-        end
-
-        it "reads last line with the multi-byte delimiter as the last bytes" do
-          data        = [longer_line, short_line, longer_line].join(delimiter) + delimiter
-          buffer_size = longer_line.length + 1
-
-          stream = StringIO.new(data)
-          IOStreams::Line::Reader.stream(stream, buffer_size: buffer_size) do |io|
-            refute io.eof?
-            assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
-            assert_equal longer_line, io.readline
-            assert_equal short_line, io.readline
-            assert_equal longer_line, io.readline
-            assert_nil io.readline
-            assert io.eof?
-          end
-        end
-
-        describe "read 1 char at a time" do
-          let(:buffer_size) { 1 }
-
-          it "delimiter at the end" do
-            data = [longer_line, short_line, longer_line].join(delimiter) + delimiter
-
-            stream = StringIO.new(data)
-            IOStreams::Line::Reader.stream(stream, buffer_size: buffer_size) do |io|
-              refute io.eof?
-              assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
-              assert_equal longer_line, io.readline
-              assert_equal short_line, io.readline
-              assert_equal longer_line, io.readline
-              assert_nil io.readline
-              assert io.eof?
-            end
-          end
-
-          it "no delimiter at the end" do
-            data = [longer_line, short_line, longer_line].join(delimiter)
-
-            stream = StringIO.new(data)
-            IOStreams::Line::Reader.stream(stream, buffer_size: buffer_size) do |io|
-              refute io.eof?
-              assert_equal delimiter, io.delimiter, -> { io.delimiter.ai }
-              assert_equal longer_line, io.readline
-              assert_equal short_line, io.readline
-              assert_equal longer_line, io.readline
-              assert_nil io.readline
-              assert io.eof?
-            end
-          end
-        end
-
-        it "reads empty file" do
-          stream = StringIO.new
-          IOStreams::Line::Reader.stream(stream) do |io|
-            assert io.eof?
-          end
-        end
-
-        it "prevents denial of service" do
-          data   = "a" * IOStreams::Line::Reader::MAX_BLOCKS_MULTIPLIER + "a"
-          stream = StringIO.new(data)
-          assert_raises IOStreams::Errors::DelimiterNotFound do
-            IOStreams::Line::Reader.stream(stream, buffer_size: 1) do |io|
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/test/line_writer_test.rb

@@ -1,59 +0,0 @@
-require_relative "test_helper"
-
-class DelimitedWriterTest < Minitest::Test
-  describe IOStreams::Line::Writer do
-    let :file_name do
-      File.join(File.dirname(__FILE__), "files", "text.txt")
-    end
-
-    let :raw do
-      File.read(file_name)
-    end
-
-    let :lines do
-      raw.lines.map(&:strip)
-    end
-
-    describe "#<<" do
-      it "file" do
-        temp_file = Tempfile.new("rocket_job")
-        file_name = temp_file.to_path
-        result    =
-          IOStreams::Line::Writer.file(file_name) do |io|
-            lines.each { |line| io << line }
-            53534
-          end
-        assert_equal 53534, result
-
-        result = File.read(file_name)
-        assert_equal raw, result
-      end
-
-      it "stream" do
-        io_string = StringIO.new
-        result    =
-          IOStreams::Line::Writer.stream(io_string) do |io|
-            lines.each { |line| io << line }
-            53534
-          end
-        assert_equal 53534, result
-        assert_equal raw, io_string.string
-      end
-    end
-
-    describe ".write" do
-      it "returns byte count" do
-        io_string = StringIO.new
-        count     = 0
-        result    =
-          IOStreams::Line::Writer.stream(io_string) do |io|
-            lines.each { |line| count += io.write(line) }
-            53534
-          end
-        assert_equal 53534, result
-        assert_equal raw, io_string.string
-        assert_equal raw.size, count
-      end
-    end
-  end
-end

data/test/minimal_file_reader.rb

@@ -1,25 +0,0 @@
-# The miminum methods that any IOStreams Reader must implement.
-class MinimalFileReader
-  def self.open(file_name)
-    io = new(file_name)
-    yield(io)
-  ensure
-    io&.close
-  end
-
-  def initialize(file_name)
-    @file = File.open(file_name)
-  end
-
-  def read(size = nil, outbuf = nil)
-    @file.read(size, outbuf)
-  end
-
-  def close
-    @file.close
-  end
-
-  def closed?
-    @file.closed
-  end
-end

data/test/path_test.rb

@@ -1,55 +0,0 @@
-require_relative "test_helper"
-
-module IOStreams
-  class PathTest < Minitest::Test
-    describe IOStreams::Path do
-      describe ".join" do
-        let(:path) { IOStreams::Path.new("some_path") }
-
-        it "returns self when no elements" do
-          assert_equal path.object_id, path.join.object_id
-        end
-
-        it "adds element to path" do
-          assert_equal ::File.join("some_path", "test"), path.join("test").to_s
-        end
-
-        it "adds paths to root" do
-          assert_equal ::File.join("some_path", "test", "second", "third"), path.join("test", "second", "third").to_s
-        end
-
-        it "returns path and filename" do
-          assert_equal ::File.join("some_path", "file.xls"), path.join("file.xls").to_s
-        end
-
-        it "adds elements to path" do
-          assert_equal ::File.join("some_path", "test", "second", "third", "file.xls"), path.join("test", "second", "third", "file.xls").to_s
-        end
-
-        it "return path as sent in when full path" do
-          assert_equal ::File.join("some_path", "test", "second", "third", "file.xls"), path.join("some_path", "test", "second", "third", "file.xls").to_s
-        end
-      end
-
-      describe "#absolute?" do
-        it "true on absolute" do
-          assert_equal true, IOStreams::Path.new("/a/b/c/d").absolute?
-        end
-
-        it "false when not absolute" do
-          assert_equal false, IOStreams::Path.new("a/b/c/d").absolute?
-        end
-      end
-
-      describe "#relatve?" do
-        it "true on relative" do
-          assert_equal true, IOStreams::Path.new("a/b/c/d").relative?
-        end
-
-        it "false on absolute" do
-          assert_equal false, IOStreams::Path.new("/a/b/c/d").relative?
-        end
-      end
-    end
-  end
-end