flat_kit 0.1.0

data/test/jsonl/test_format.rb

@@ -0,0 +1,22 @@
+require_relative '../test_helper'
+
+module TestJsonl
+  class TestFormat < ::Minitest::Test
+
+    def test_handles_json
+      assert(::FlatKit::Jsonl::Format.handles?("data.json.gz"))
+    end
+
+    def test_handles_jsonl
+      assert(::FlatKit::Jsonl::Format.handles?("data.jsonl"))
+    end
+
+    def test_handles_ndjson
+      assert(::FlatKit::Jsonl::Format.handles?("log.ndjson"))
+    end
+
+    def test_does_not_handle_csv
+      refute(::FlatKit::Jsonl::Format.handles?("data.csv"))
+    end
+  end
+end

data/test/jsonl/test_reader.rb

@@ -0,0 +1,49 @@
+require_relative '../test_helper'
+
+module TestJsonl
+  class TestReader < ::Minitest::Test
+    def setup
+      @count = 20
+      @dataset = DeviceDataset.new(count: @count)
+      @compare_fields = @dataset.compare_fields
+      @test_path = "tmp/test_reads_from_io.jsonl"
+
+      File.open(@test_path, "wb") do |f|
+        f.write(@dataset.records_as_jsonl)
+      end
+    end
+
+    def teardown
+      File.unlink(@test_path) if File.exist?(@test_path)
+    end
+
+    def test_raises_error_on_invalid_source
+      assert_raises(::FlatKit::Error) {
+        ::FlatKit::Jsonl::Reader.new(source: Object.new, compare_fields: nil)
+      }
+    end
+
+    def test_reads_from_pathname
+      reader = ::FlatKit::Jsonl::Reader.new(source: @test_path, compare_fields: @compare_fields)
+      all = reader.to_a
+      assert_equal(@count, reader.count)
+      assert_equal(@count, all.size)
+    end
+
+    def test_reads_from_io
+      File.open(@test_path) do |f|
+        reader = ::FlatKit::Jsonl::Reader.new(source: f, compare_fields: @compare_fields)
+        all = reader.to_a
+        assert_equal(@count, reader.count)
+        assert_equal(@count, all.size)
+      end
+    end
+
+    def test_raises_error_on_io_error
+      s = StringIO.new
+      s.close_read
+      reader = ::FlatKit::Jsonl::Reader.new(source: s, compare_fields: @compare_fields)
+      assert_raises(::FlatKit::Error) { reader.to_a }
+    end
+  end
+end

data/test/jsonl/test_record.rb

@@ -0,0 +1,61 @@
+require_relative '../test_helper'
+require 'faker'
+require 'byebug'
+
+module TestJsonl
+  class TestRecord < ::Minitest::Test
+    def setup
+      @one_row_dataset = DeviceDataset.new(count: 1)
+      @src_record = @one_row_dataset.records.first
+      @src_json = JSON.generate(@src_record)
+      @compare_fields = @one_row_dataset.compare_fields
+    end
+
+    def test_initializes_from_data
+      record = FlatKit::Jsonl::Record.new(data: @src_json, compare_fields: @compare_fields)
+      @compare_fields.each do |k|
+        assert_equal(@src_record[k], record[k])
+      end
+    end
+
+    def test_ignores_non_compare_fields_values
+      record = FlatKit::Jsonl::Record.new(data: @src_json, compare_fields: @compare_fields)
+
+      refute(record["version"])
+    end
+
+    def test_is_sortable
+      dataset = DeviceDataset.new(count: 20)
+      fk_records = Array.new.tap do |a|
+        dataset.records.each do |r|
+          data = JSON.generate(r)
+          record = FlatKit::Jsonl::Record.new(data: data, compare_fields: @compare_fields)
+          a << record
+        end
+      end
+
+      sorted = fk_records.sort
+
+      sio = StringIO.new
+      sorted.each do |r|
+        sio.puts(r.to_s)
+      end
+
+      sorted_string = sio.string
+      assert_equal(dataset.sorted_records_as_jsonl, sorted_string)
+    end
+
+    def test_converts_to_hash
+      record = FlatKit::Jsonl::Record.new(data: @src_json, compare_fields: @compare_fields)
+      h      = record.to_hash
+
+      assert_equal(@src_record, h)
+    end
+
+    def test_converts_from_record
+      rec1 = FlatKit::Jsonl::Record.new(data: @src_json, compare_fields: @compare_fields)
+      rec2 = FlatKit::Jsonl::Record.from_record(rec1)
+      assert_equal(rec1, rec2)
+    end
+  end
+end

data/test/jsonl/test_writer.rb

@@ -0,0 +1,68 @@
+require_relative '../test_helper'
+
+module TestJsonl
+  class TestWriter < ::Minitest::Test
+    def setup
+      @count = 20
+      @dataset = DeviceDataset.new(count: @count)
+      @compare_fields = @dataset.compare_fields
+      @write_path = "tmp/test_writes_to_io.jsonl"
+      @read_path = "tmp/test_read.jsonl"
+
+      File.open(@read_path, "wb") do |f|
+        f.write(@dataset.records_as_jsonl)
+      end
+
+      @reader = ::FlatKit::Jsonl::Reader.new(source: @read_path, compare_fields: @compare_fields)
+      @records = @reader.to_a
+    end
+
+    def teardown
+      File.unlink(@write_path) if File.exist?(@write_path)
+      File.unlink(@read_path) if File.exist?(@read_path)
+    end
+
+    def test_raises_error_on_invalid_destination
+      assert_raises(::FlatKit::Error) {
+        ::FlatKit::Jsonl::Writer.new(destination: Object.new)
+      }
+    end
+
+    def test_writes_to_pathname
+      writer = ::FlatKit::Jsonl::Writer.new(destination: @write_path)
+      @records.each do |r|
+        writer.write(r)
+      end
+      writer.close
+      assert_equal(@count, writer.count)
+
+      expected = @dataset.records_as_jsonl
+      actual = IO.read(@write_path)
+      assert_equal(expected, actual)
+    end
+
+    def test_writes_to_io
+      File.open(@write_path, "w+") do |f|
+        writer = ::FlatKit::Jsonl::Writer.new(destination: f)
+
+        @records.each do |r|
+          writer.write(r)
+        end
+        writer.close
+
+        assert_equal(@count, writer.count)
+
+        expected = @dataset.records_as_jsonl
+        actual = IO.read(@write_path)
+        assert_equal(expected, actual)
+      end
+    end
+
+    def test_raises_error_on_io_error
+      s = StringIO.new
+      s.close_write
+      writer = ::FlatKit::Jsonl::Writer.new(destination: s)
+      assert_raises(::FlatKit::Error) { writer.write(@records.first) }
+    end
+  end
+end

data/test/output/test_file.rb

@@ -0,0 +1,60 @@
+require_relative '../test_helper'
+
+module TestOutput
+  class TestFile < ::Minitest::Test
+    def test_does_not_handle_stderr_text
+      ::FlatKit::Output::IO::STDERRS.each do |e|
+        refute(::FlatKit::Output::File.handles?(e), "#{e} is not stderr text")
+      end
+    end
+
+    def test_only_handles_string
+      refute(::FlatKit::Output::File.handles?(Object.new))
+    end
+
+    def test_doest_not_handles_stdout_text
+      ::FlatKit::Output::IO::STDOUTS.each do |e|
+        refute(::FlatKit::Output::File.handles?(e), "#{e} is not stdout text")
+      end
+    end
+
+    def test_init_from_path
+      test_path = "tmp/test_init_from_path.txt"
+      begin
+        io = ::FlatKit::Output::File.new(test_path)
+        assert_equal(test_path, io.name)
+        assert_instance_of(::File, io.io)
+      ensure
+        File.unlink(test_path) if File.exist?(test_path)
+      end
+    end
+
+    def test_writes_to_file
+      test_path = "tmp/test_writes_to_file.txt"
+      begin
+        output = ::FlatKit::Output::File.new(test_path)
+        assert_equal(test_path, output.name)
+        output.io.write("test_writes_to_file output")
+        output.close
+        t = IO.read(test_path)
+        assert_equal("test_writes_to_file output", t)
+      ensure
+        File.unlink(test_path) if File.exist?(test_path)
+      end
+    end
+
+    def test_writes_to_gzfile
+      test_path = "tmp/test_writes_to_gzfile.txt.gz"
+      begin
+        output = ::FlatKit::Output::File.new(test_path)
+        assert_equal(test_path, output.name)
+        output.io.write("test_writes_to_gzfile output")
+        output.close
+        t = %x[ gunzip -c #{test_path} ]
+        assert_equal("test_writes_to_gzfile output", t)
+      ensure
+        File.unlink(test_path) if File.exist?(test_path)
+      end
+    end
+  end
+end

data/test/output/test_io.rb

@@ -0,0 +1,104 @@
+require_relative '../test_helper'
+
+module TestOutput
+  class NullIO < ::IO
+    def initialize()
+    end
+  end
+
+  class TestIO < ::Minitest::Test
+    def test_handles_stderr_text
+      ::FlatKit::Output::IO::STDERRS.each do |e|
+        assert(::FlatKit::Output::IO.handles?(e), "#{e} is not stderr text")
+      end
+    end
+
+    def test_handles_stderr_io
+      x = $stderr
+      assert(::FlatKit::Output::IO.handles?(x), "is not stderr")
+    end
+
+    def test_handles_stdout_text
+      ::FlatKit::Output::IO::STDOUTS.each do |e|
+        assert(::FlatKit::Output::IO.handles?(e), "#{e} is not stdout text")
+      end
+    end
+
+    def test_handles_stdout_io
+      x = $stderr
+      assert(::FlatKit::Output::IO.handles?(x), "is not stdout")
+    end
+
+    def test_handles_stringio
+      assert(::FlatKit::Output::IO.handles?(StringIO.new))
+    end
+
+    def test_does_not_handle_other
+      x = Object.new
+      refute(::FlatKit::Output::IO.handles?(x))
+    end
+
+    def test_init_from_dash
+      io = ::FlatKit::Output::IO.new("-")
+      assert_equal("<STDOUT>", io.name)
+      assert_equal(::STDOUT, io.io)
+    end
+
+    def test_init_from_stderr_text
+      io = ::FlatKit::Output::IO.new("stderr")
+      assert_equal("<STDERR>", io.name)
+      assert_equal(::STDERR, io.io)
+    end
+
+    def test_init_from_file_object
+      test_path = "tmp/test_init_from_file_object.txt"
+      begin
+        File.open(test_path, "w") do |f|
+          io = ::FlatKit::Output::IO.new(f)
+          assert_equal(test_path, io.name)
+          assert_instance_of(::File, io.io)
+        end
+      ensure
+        File.unlink(test_path) if File.exist?(test_path)
+      end
+    end
+
+    def test_init_from_stdout
+      io  = ::FlatKit::Output::IO.new($stdout)
+      assert_equal("<STDOUT>", io.name)
+      assert_equal(::STDOUT, io.io)
+    end
+
+    def test_init_from_string_io_object
+      sio = StringIO.new
+      io = ::FlatKit::Output::IO.new(sio)
+      assert_match(/StringIO/, io.name)
+      assert_instance_of(::StringIO, io.io)
+    end
+
+    def test_init_from_io_object
+      null_io = NullIO.new
+      io = ::FlatKit::Output::IO.new(null_io)
+      assert_match(/NullIO/, io.name)
+      assert_instance_of(::TestOutput::NullIO, io.io)
+    end
+
+    def test_writes_to_io
+      test_path = "tmp/test_writes_to_io.txt"
+      begin
+        File.open(test_path, "w") do |f|
+          io = ::FlatKit::Output::IO.new(f)
+          assert_equal(test_path, io.name)
+          assert_instance_of(::File, io.io)
+          io.io.write("test_writes_to_io output")
+          io.close
+        end
+        t = IO.read(test_path)
+        assert_equal("test_writes_to_io output", t)
+      ensure
+        File.unlink(test_path) if File.exist?(test_path)
+      end
+    end
+
+  end
+end

data/test/test_conversions.rb

@@ -0,0 +1,45 @@
+require 'test_helper'
+
+class TestConversions < ::Minitest::Test
+  def setup
+    @one_row_dataset = DeviceDataset.new(count: 1)
+    @src_record = @one_row_dataset.records.first
+    @csv_row = @one_row_dataset.records_as_csv_rows.first
+    @compare_fields = @one_row_dataset.compare_fields
+  end
+
+  def test_from_csv_to_json
+    xsv_record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
+    json_record = FlatKit::Jsonl::Record.from_record(xsv_record)
+
+    assert_equal(@one_row_dataset.records.first, xsv_record.to_hash)
+    assert_equal(@one_row_dataset.records.first, json_record.to_hash)
+    assert_equal(xsv_record, json_record)
+  end
+
+  def test_from_json_to_csv
+    src_json = JSON.generate(@src_record)
+    json_record = FlatKit::Jsonl::Record.new(data: src_json, compare_fields: @compare_fields)
+    xsv_record = FlatKit::Xsv::Record.from_record(json_record)
+
+    assert_equal(@one_row_dataset.records.first, xsv_record.to_hash)
+    assert_equal(@one_row_dataset.records.first, json_record.to_hash)
+    assert_equal(xsv_record, json_record)
+  end
+
+  def test_roundtrip_csv_json_csv
+    xsv_record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
+    json_record = FlatKit::Jsonl::Record.from_record(xsv_record)
+    xsv2 = FlatKit::Xsv::Record.from_record(json_record)
+
+    assert_equal(xsv_record.to_s, xsv2.to_s)
+  end
+
+  def test_roundtrip_json_csv_json
+    src_json = JSON.generate(@src_record)
+    json_record = FlatKit::Jsonl::Record.new(data: src_json, compare_fields: @compare_fields)
+    xsv_record = FlatKit::Xsv::Record.from_record(json_record)
+    json2 = FlatKit::Jsonl::Record.from_record(xsv_record)
+    assert_equal(src_json, json2.to_s)
+  end
+end

data/test/test_format.rb

@@ -0,0 +1,24 @@
+require_relative 'test_helper'
+
+class TestFormat < ::Minitest::Test
+  def test_finds_jsonl_format
+    klass = ::FlatKit::Format.for("data.json.gz")
+    assert_equal(::FlatKit::Jsonl::Format, klass)
+  end
+
+  def test_finds_xsv_format
+    klass = ::FlatKit::Format.for("data.csv.gz")
+    assert_equal(::FlatKit::Xsv::Format, klass)
+  end
+
+  def test_finds_jsonl_format_for_full_path
+    klass = ::FlatKit::Format.for("tmp/sorted/foo.jsonl")
+    assert_equal(::FlatKit::Jsonl::Format, klass)
+  end
+
+  def test_finds_jsonl_format_with_fallback
+    path = "tmp/sorted/foo.json"
+    klass = ::FlatKit::Format.for_with_fallback!(path: path, fallback: "auto")
+    assert_equal(::FlatKit::Jsonl::Format, klass)
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,26 @@
+require 'simplecov'
+SimpleCov.start if ENV['COVERAGE']
+
+require 'byebug'
+
+require 'minitest/autorun'
+require 'minitest/focus'
+require 'minitest/pride'
+
+module TestHelper
+  def scratch_dir
+    p = Pathname.new(__FILE__).parent.parent.join('tmp/testing_scratch')
+    p.mkpath
+    p
+  end
+
+  def generate_slug(length: 10)
+    SecureRandom.alphanumeric(10)
+  end
+
+  def scratch_file(prefix: "test_", slug: generate_slug, extension: ".jsonl")
+    scratch_dir.join("#{prefix}#{slug}#{extension}")
+  end
+end
+require_relative '../lib/flat_kit'
+require_relative './device_dataset'