flat_kit 0.3.0 → 1.0.0

data/test/field_type/test_timestamp_type.rb

@@ -1,108 +0,0 @@
-require_relative '../test_helper'
-require 'set'
-
-module TestFieldType
-  class TestTimestampType < ::Minitest::Test
-    def test_time
-      assert(FlatKit::FieldType::TimestampType.matches?(Time.now))
-    end
-
-    def test_date
-      refute(FlatKit::FieldType::TimestampType.matches?(Date.today))
-    end
-
-    def test_date_time
-      refute(FlatKit::FieldType::TimestampType.matches?(DateTime.now))
-    end
-
-    def test_builtin_formats
-      stock_formats = [
-        Time.now.httpdate,
-        Time.now.utc.httpdate,
-        Time.now.iso8601,
-        Time.now.utc.iso8601,
-        Time.now.rfc2822,
-        Time.now.utc.rfc2822,
-      ]
-
-      stock_formats.each do |t|
-        assert(FlatKit::FieldType::TimestampType.matches?(t), "#{t} should match timestamp")
-        assert_instance_of(Time, FlatKit::FieldType::TimestampType.coerce(t), "#{t} should convert to timestamp")
-      end
-    end
-
-    def test_no_duplicate_formats
-      parse_formats = ::FlatKit::FieldType::TimestampType.parse_formats
-
-      assert_equal(parse_formats.size, parse_formats.sort.uniq.size)
-    end
-
-    def test_parse_formats
-      parse_formats = ::FlatKit::FieldType::TimestampType.parse_formats.dup
-      extra_formats = [
-          "%Y-%m-%dT%H:%M:%S.%N%z",
-          "%Y-%m-%d %H:%M:%S.%NZ",
-          "%Y-%m-%d %H:%M:%S.%N",
-          "%Y-%m-%dT%H:%M:%S.%3N%z",
-          "%Y-%m-%d %H:%M:%S.%3NZ",
-          "%Y-%m-%d %H:%M:%S.%3N",
-          "%Y-%m-%dT%H:%M:%S.%6N%z",
-          "%Y-%m-%d %H:%M:%S.%6NZ",
-          "%Y-%m-%d %H:%M:%S.%6N",
-          "%Y-%m-%dT%H:%M:%S.%9N%z",
-          "%Y-%m-%d %H:%M:%S.%9NZ",
-          "%Y-%m-%d %H:%M:%S.%9N",
-      ]
-      parse_formats.concat(extra_formats)
-
-      parse_formats.each do |format|
-        now = Time.now
-        str = now.strftime(format)
-
-        assert(FlatKit::FieldType::TimestampType.matches?(str), "#{str} should match timestamp")
-        coerced = FlatKit::FieldType::TimestampType.coerce(str)
-
-        assert_instance_of(Time, coerced)
-      end
-    end
-
-    def test_other_class_does_not_match
-      [ 42, Object.new, true, false ].each do |x|
-        refute(FlatKit::FieldType::TimestampType.matches?(x), "#{x} should not be date")
-      end
-    end
-
-    def test_N_number_does_not_match
-      x = "N89362"
-      refute(FlatKit::FieldType::TimestampType.matches?(x), "#{x} should not be date")
-    end
-
-    def test_coerce_passthrough_time
-      t = Time.now
-      assert_equal(t, FlatKit::FieldType::TimestampType.coerce(t))
-    end
-
-    def test_coerce_do_not_passthrough_date
-      t = Date.today
-      assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::TimestampType.coerce(t))
-    end
-
-    def test_date_coerce_passthrough_datetime
-      t = Time.now
-      assert_equal(t, FlatKit::FieldType::TimestampType.coerce(t))
-    end
-
-    def test_date_only_does_not_parse
-      t = Time.now.strftime("%Y-%m-%d")
-      assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::TimestampType.coerce(t))
-    end
-
-    def test_number_coerce_failure
-      assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::TimestampType.coerce(42))
-    end
-
-    def test_number_coerce_failure_bad_parse
-      assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::TimestampType.coerce("1234 56 78 90"))
-    end
-  end
-end

data/test/field_type/test_unknown_type.rb

@@ -1,35 +0,0 @@
-require_relative '../test_helper'
-
-module TestFieldType
-  class TestUnknownType < ::Minitest::Test
-
-    def unknown_items
-      [ 'na', 'n/a', 'unk', 'unknown']
-    end
-
-    def test_unknown_items
-      unknown_items.each do |u|
-        assert(FlatKit::FieldType::UnknownType.matches?(u), "#{u} should be unknown")
-      end
-    end
-
-    def test_other_class_does_not_match
-      [ 42, Object.new, true, false, "whatever" ].each do |x|
-        refute(FlatKit::FieldType::UnknownType.matches?(x), "#{x} should not unknown ")
-      end
-    end
-
-    def test_coerce_unknown
-      unknown_items.each do |u|
-        assert_equal(u, FlatKit::FieldType::UnknownType.coerce(u), "#{u} should be unknown")
-      end
-    end
-
-    def test_other_class_does_not_coerce
-      [ 42, Object.new, true, false, "whatever" ].each do |x|
-        assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::UnknownType.coerce(x))
-      end
-    end
-
-  end
-end

data/test/input/test_file.rb

@@ -1,73 +0,0 @@
-require_relative '../test_helper'
-
-module TestInput
-  class TestFile < ::Minitest::Test
-    def test_does_not_handle_stdin_text
-      ::FlatKit::Input::IO::STDINS.each do |e|
-        refute(::FlatKit::Input::File.handles?(e), "#{e} is not stdin text")
-      end
-    end
-
-    def test_handles_existing_file
-      test_path = "tmp/test_handles_existing_file.txt"
-      begin
-        IO.write(test_path,"test handles existing file")
-        assert(::FlatKit::Input::File.handles?(test_path))
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_only_handles_string
-      refute(::FlatKit::Input::File.handles?(Object.new))
-    end
-
-    def test_raises_error_if_not_readable
-      assert_raises(FlatKit::Error) { ::FlatKit::Input::File.new("tmp/does-not-exist") }
-    end
-
-    def test_init_from_path
-      test_path = "tmp/test_init_from_path.txt"
-      begin
-        IO.write(test_path,"nothing to see here")
-        io = ::FlatKit::Input::File.new(test_path)
-        assert_equal(test_path, io.name)
-        assert_instance_of(::File, io.io)
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_reads_from_file
-      test_path = "tmp/test_reads_from_file.txt"
-      begin
-        text = "test_reads_from_file"
-        IO.write(test_path,text)
-
-        input = ::FlatKit::Input::File.new(test_path)
-        content = input.io.read
-        assert_equal(text, content)
-
-        input.close
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_reads_from_gzfile
-      test_path = "tmp/test_reads_from_gzfile.txt.gz"
-      begin
-        text = "this is something to read"
-        system("echo '#{text}' | gzip > #{test_path}")
-
-        input = ::FlatKit::Input::File.new(test_path)
-        content = input.io.read
-        assert_equal(text + "\n", content)
-
-        input.close
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-  end
-end

data/test/input/test_io.rb

@@ -1,93 +0,0 @@
-require_relative '../test_helper'
-
-module TestInput
-  class NullIO < ::IO
-    def initialize()
-    end
-  end
-
-  class TestIO < ::Minitest::Test
-
-    def test_handles_stdin_text
-      ::FlatKit::Input::IO::STDINS.each do |e|
-        assert(::FlatKit::Input::IO.handles?(e), "#{e} is not stdin text")
-      end
-    end
-
-    def test_handles_stdin_io
-      x = $stdin
-      assert(::FlatKit::Input::IO.handles?(x), "is not stdin")
-    end
-
-    def test_handles_stringio
-      assert(::FlatKit::Input::IO.handles?(StringIO.new))
-    end
-
-    def test_does_not_handle_other
-      x = Object.new
-      refute(::FlatKit::Input::IO.handles?(x))
-    end
-
-    def test_init_from_dash
-      io = ::FlatKit::Input::IO.new("-")
-      assert_equal("<STDIN>", io.name)
-      assert_equal(::STDIN, io.io)
-    end
-
-    def test_init_from_file_object
-      test_path = "tmp/test_init_from_file_object.txt"
-      begin
-        File.open(test_path, "w+") do |f|
-          io = ::FlatKit::Input::IO.new(f)
-          assert_equal(test_path, io.name)
-          assert_instance_of(::File, io.io)
-        end
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_init_from_string_io_object
-      sio = StringIO.new
-      io = ::FlatKit::Input::IO.new(sio)
-      assert_match(/StringIO/, io.name)
-      assert_instance_of(::StringIO, io.io)
-    end
-
-    def test_init_from_io_object
-      null_io = NullIO.new
-      io = ::FlatKit::Input::IO.new(null_io)
-      assert_match(/NullIO/, io.name)
-      assert_instance_of(::TestInput::NullIO, io.io)
-    end
-
-    def test_init_from_stdin
-      io  = ::FlatKit::Input::IO.new($stdin)
-      assert_equal("<STDIN>", io.name)
-      assert_equal(::STDIN, io.io)
-    end
-
-    def test_init_from_invalid
-      assert_raises(::FlatKit::Error) { ::FlatKit::Input::IO.new(Object.new) }
-    end
-
-    def test_reads_from_io
-      test_path = "tmp/test_reads_from_io.txt"
-      begin
-        line = "This is a line to read"
-        File.open(test_path, "w+") do |f|
-          f.write(line)
-          f.rewind
-
-          io = ::FlatKit::Input::IO.new(f)
-          content = io.io.read
-          io.close
-          assert_equal(content, line)
-        end
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-  end
-end

data/test/jsonl/test_format.rb

@@ -1,22 +0,0 @@
-require_relative '../test_helper'
-
-module TestJsonl
-  class TestFormat < ::Minitest::Test
-
-    def test_handles_json
-      assert(::FlatKit::Jsonl::Format.handles?("data.json.gz"))
-    end
-
-    def test_handles_jsonl
-      assert(::FlatKit::Jsonl::Format.handles?("data.jsonl"))
-    end
-
-    def test_handles_ndjson
-      assert(::FlatKit::Jsonl::Format.handles?("log.ndjson"))
-    end
-
-    def test_does_not_handle_csv
-      refute(::FlatKit::Jsonl::Format.handles?("data.csv"))
-    end
-  end
-end

data/test/jsonl/test_reader.rb

@@ -1,49 +0,0 @@
-require_relative '../test_helper'
-
-module TestJsonl
-  class TestReader < ::Minitest::Test
-    def setup
-      @count = 20
-      @dataset = DeviceDataset.new(count: @count)
-      @compare_fields = @dataset.compare_fields
-      @test_path = "tmp/test_reads_from_io.jsonl"
-
-      File.open(@test_path, "wb") do |f|
-        f.write(@dataset.records_as_jsonl)
-      end
-    end
-
-    def teardown
-      File.unlink(@test_path) if File.exist?(@test_path)
-    end
-
-    def test_raises_error_on_invalid_source
-      assert_raises(::FlatKit::Error) {
-        ::FlatKit::Jsonl::Reader.new(source: Object.new, compare_fields: nil)
-      }
-    end
-
-    def test_reads_from_pathname
-      reader = ::FlatKit::Jsonl::Reader.new(source: @test_path, compare_fields: @compare_fields)
-      all = reader.to_a
-      assert_equal(@count, reader.count)
-      assert_equal(@count, all.size)
-    end
-
-    def test_reads_from_io
-      File.open(@test_path) do |f|
-        reader = ::FlatKit::Jsonl::Reader.new(source: f, compare_fields: @compare_fields)
-        all = reader.to_a
-        assert_equal(@count, reader.count)
-        assert_equal(@count, all.size)
-      end
-    end
-
-    def test_raises_error_on_io_error
-      s = StringIO.new
-      s.close_read
-      reader = ::FlatKit::Jsonl::Reader.new(source: s, compare_fields: @compare_fields)
-      assert_raises(::FlatKit::Error) { reader.to_a }
-    end
-  end
-end

data/test/jsonl/test_record.rb

@@ -1,61 +0,0 @@
-require_relative '../test_helper'
-require 'faker'
-require 'byebug'
-
-module TestJsonl
-  class TestRecord < ::Minitest::Test
-    def setup
-      @one_row_dataset = DeviceDataset.new(count: 1)
-      @src_record = @one_row_dataset.records.first
-      @src_json = JSON.generate(@src_record)
-      @compare_fields = @one_row_dataset.compare_fields
-    end
-
-    def test_initializes_from_data
-      record = FlatKit::Jsonl::Record.new(data: @src_json, compare_fields: @compare_fields)
-      @compare_fields.each do |k|
-        assert_equal(@src_record[k], record[k])
-      end
-    end
-
-    def test_ignores_non_compare_fields_values
-      record = FlatKit::Jsonl::Record.new(data: @src_json, compare_fields: @compare_fields)
-
-      refute(record["version"])
-    end
-
-    def test_is_sortable
-      dataset = DeviceDataset.new(count: 20)
-      fk_records = Array.new.tap do |a|
-        dataset.records.each do |r|
-          data = JSON.generate(r)
-          record = FlatKit::Jsonl::Record.new(data: data, compare_fields: @compare_fields)
-          a << record
-        end
-      end
-
-      sorted = fk_records.sort
-
-      sio = StringIO.new
-      sorted.each do |r|
-        sio.puts(r.to_s)
-      end
-
-      sorted_string = sio.string
-      assert_equal(dataset.sorted_records_as_jsonl, sorted_string)
-    end
-
-    def test_converts_to_hash
-      record = FlatKit::Jsonl::Record.new(data: @src_json, compare_fields: @compare_fields)
-      h      = record.to_hash
-
-      assert_equal(@src_record, h)
-    end
-
-    def test_converts_from_record
-      rec1 = FlatKit::Jsonl::Record.new(data: @src_json, compare_fields: @compare_fields)
-      rec2 = FlatKit::Jsonl::Record.from_record(rec1)
-      assert_equal(rec1, rec2)
-    end
-  end
-end

data/test/jsonl/test_writer.rb

@@ -1,86 +0,0 @@
-require_relative '../test_helper'
-
-module TestJsonl
-  class TestWriter < ::Minitest::Test
-    def setup
-      @count = 20
-      @dataset = DeviceDataset.new(count: @count)
-      @compare_fields = @dataset.compare_fields
-      @write_path = "tmp/test_writes_to_io.jsonl"
-      @read_path = "tmp/test_read.jsonl"
-
-      File.open(@read_path, "wb") do |f|
-        f.write(@dataset.records_as_jsonl)
-      end
-
-      @reader = ::FlatKit::Jsonl::Reader.new(source: @read_path, compare_fields: @compare_fields)
-      @records = @reader.to_a
-    end
-
-    def teardown
-      File.unlink(@write_path) if File.exist?(@write_path)
-      File.unlink(@read_path) if File.exist?(@read_path)
-    end
-
-    def test_raises_error_on_invalid_destination
-      assert_raises(::FlatKit::Error) {
-        ::FlatKit::Jsonl::Writer.new(destination: Object.new)
-      }
-    end
-
-    def test_writes_to_pathname
-      writer = ::FlatKit::Jsonl::Writer.new(destination: @write_path)
-      @records.each do |r|
-        writer.write(r)
-      end
-      writer.close
-      assert_equal(@count, writer.count)
-
-      expected = @dataset.records_as_jsonl
-      actual = IO.read(@write_path)
-      assert_equal(expected, actual)
-    end
-
-    def test_postion
-      File.open(@write_path, "w+") do |f|
-        writer = ::FlatKit::Jsonl::Writer.new(destination: f)
-
-        byte_offset = 0
-        @records.each_with_index do |r, idx|
-          record_length = r.data.bytesize
-
-          position = writer.write(r)
-
-          # make sure write stores the last_position api and returns that value
-          assert_equal(position, writer.last_position)
-
-          assert_equal(idx, position.index)
-          assert_equal(byte_offset, position.offset)
-          assert_equal(record_length, position.bytesize)
-
-          byte_offset += record_length
-
-          current_position = writer.current_position
-          assert_equal(idx+1, current_position.index)
-          assert_equal(byte_offset, current_position.offset)
-          assert_equal(0, current_position.bytesize)
-
-        end
-        writer.close
-
-        assert_equal(@count, writer.count)
-
-        expected = @dataset.records_as_jsonl
-        actual = IO.read(@write_path)
-        assert_equal(expected, actual)
-      end
-    end
-
-    def test_raises_error_on_io_error
-      s = StringIO.new
-      s.close_write
-      writer = ::FlatKit::Jsonl::Writer.new(destination: s)
-      assert_raises(::FlatKit::Error) { writer.write(@records.first) }
-    end
-  end
-end

data/test/output/test_file.rb

@@ -1,60 +0,0 @@
-require_relative '../test_helper'
-
-module TestOutput
-  class TestFile < ::Minitest::Test
-    def test_does_not_handle_stderr_text
-      ::FlatKit::Output::IO::STDERRS.each do |e|
-        refute(::FlatKit::Output::File.handles?(e), "#{e} is not stderr text")
-      end
-    end
-
-    def test_only_handles_string
-      refute(::FlatKit::Output::File.handles?(Object.new))
-    end
-
-    def test_doest_not_handles_stdout_text
-      ::FlatKit::Output::IO::STDOUTS.each do |e|
-        refute(::FlatKit::Output::File.handles?(e), "#{e} is not stdout text")
-      end
-    end
-
-    def test_init_from_path
-      test_path = "tmp/test_init_from_path.txt"
-      begin
-        io = ::FlatKit::Output::File.new(test_path)
-        assert_equal(test_path, io.name)
-        assert_instance_of(::File, io.io)
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_writes_to_file
-      test_path = "tmp/test_writes_to_file.txt"
-      begin
-        output = ::FlatKit::Output::File.new(test_path)
-        assert_equal(test_path, output.name)
-        output.io.write("test_writes_to_file output")
-        output.close
-        t = IO.read(test_path)
-        assert_equal("test_writes_to_file output", t)
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_writes_to_gzfile
-      test_path = "tmp/test_writes_to_gzfile.txt.gz"
-      begin
-        output = ::FlatKit::Output::File.new(test_path)
-        assert_equal(test_path, output.name)
-        output.io.write("test_writes_to_gzfile output")
-        output.close
-        t = %x[ gunzip -c #{test_path} ]
-        assert_equal("test_writes_to_gzfile output", t)
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-  end
-end

data/test/output/test_io.rb

@@ -1,104 +0,0 @@
-require_relative '../test_helper'
-
-module TestOutput
-  class NullIO < ::IO
-    def initialize()
-    end
-  end
-
-  class TestIO < ::Minitest::Test
-    def test_handles_stderr_text
-      ::FlatKit::Output::IO::STDERRS.each do |e|
-        assert(::FlatKit::Output::IO.handles?(e), "#{e} is not stderr text")
-      end
-    end
-
-    def test_handles_stderr_io
-      x = $stderr
-      assert(::FlatKit::Output::IO.handles?(x), "is not stderr")
-    end
-
-    def test_handles_stdout_text
-      ::FlatKit::Output::IO::STDOUTS.each do |e|
-        assert(::FlatKit::Output::IO.handles?(e), "#{e} is not stdout text")
-      end
-    end
-
-    def test_handles_stdout_io
-      x = $stderr
-      assert(::FlatKit::Output::IO.handles?(x), "is not stdout")
-    end
-
-    def test_handles_stringio
-      assert(::FlatKit::Output::IO.handles?(StringIO.new))
-    end
-
-    def test_does_not_handle_other
-      x = Object.new
-      refute(::FlatKit::Output::IO.handles?(x))
-    end
-
-    def test_init_from_dash
-      io = ::FlatKit::Output::IO.new("-")
-      assert_equal("<STDOUT>", io.name)
-      assert_equal(::STDOUT, io.io)
-    end
-
-    def test_init_from_stderr_text
-      io = ::FlatKit::Output::IO.new("stderr")
-      assert_equal("<STDERR>", io.name)
-      assert_equal(::STDERR, io.io)
-    end
-
-    def test_init_from_file_object
-      test_path = "tmp/test_init_from_file_object.txt"
-      begin
-        File.open(test_path, "w") do |f|
-          io = ::FlatKit::Output::IO.new(f)
-          assert_equal(test_path, io.name)
-          assert_instance_of(::File, io.io)
-        end
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_init_from_stdout
-      io  = ::FlatKit::Output::IO.new($stdout)
-      assert_equal("<STDOUT>", io.name)
-      assert_equal(::STDOUT, io.io)
-    end
-
-    def test_init_from_string_io_object
-      sio = StringIO.new
-      io = ::FlatKit::Output::IO.new(sio)
-      assert_match(/StringIO/, io.name)
-      assert_instance_of(::StringIO, io.io)
-    end
-
-    def test_init_from_io_object
-      null_io = NullIO.new
-      io = ::FlatKit::Output::IO.new(null_io)
-      assert_match(/NullIO/, io.name)
-      assert_instance_of(::TestOutput::NullIO, io.io)
-    end
-
-    def test_writes_to_io
-      test_path = "tmp/test_writes_to_io.txt"
-      begin
-        File.open(test_path, "w") do |f|
-          io = ::FlatKit::Output::IO.new(f)
-          assert_equal(test_path, io.name)
-          assert_instance_of(::File, io.io)
-          io.io.write("test_writes_to_io output")
-          io.close
-        end
-        t = IO.read(test_path)
-        assert_equal("test_writes_to_io output", t)
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-  end
-end