flat_kit 0.2.0 → 1.0.0

data/test/jsonl/test_writer.rb

@@ -1,68 +0,0 @@
-require_relative '../test_helper'
-
-module TestJsonl
-  class TestWriter < ::Minitest::Test
-    def setup
-      @count = 20
-      @dataset = DeviceDataset.new(count: @count)
-      @compare_fields = @dataset.compare_fields
-      @write_path = "tmp/test_writes_to_io.jsonl"
-      @read_path = "tmp/test_read.jsonl"
-
-      File.open(@read_path, "wb") do |f|
-        f.write(@dataset.records_as_jsonl)
-      end
-
-      @reader = ::FlatKit::Jsonl::Reader.new(source: @read_path, compare_fields: @compare_fields)
-      @records = @reader.to_a
-    end
-
-    def teardown
-      File.unlink(@write_path) if File.exist?(@write_path)
-      File.unlink(@read_path) if File.exist?(@read_path)
-    end
-
-    def test_raises_error_on_invalid_destination
-      assert_raises(::FlatKit::Error) {
-        ::FlatKit::Jsonl::Writer.new(destination: Object.new)
-      }
-    end
-
-    def test_writes_to_pathname
-      writer = ::FlatKit::Jsonl::Writer.new(destination: @write_path)
-      @records.each do |r|
-        writer.write(r)
-      end
-      writer.close
-      assert_equal(@count, writer.count)
-
-      expected = @dataset.records_as_jsonl
-      actual = IO.read(@write_path)
-      assert_equal(expected, actual)
-    end
-
-    def test_writes_to_io
-      File.open(@write_path, "w+") do |f|
-        writer = ::FlatKit::Jsonl::Writer.new(destination: f)
-
-        @records.each do |r|
-          writer.write(r)
-        end
-        writer.close
-
-        assert_equal(@count, writer.count)
-
-        expected = @dataset.records_as_jsonl
-        actual = IO.read(@write_path)
-        assert_equal(expected, actual)
-      end
-    end
-
-    def test_raises_error_on_io_error
-      s = StringIO.new
-      s.close_write
-      writer = ::FlatKit::Jsonl::Writer.new(destination: s)
-      assert_raises(::FlatKit::Error) { writer.write(@records.first) }
-    end
-  end
-end

data/test/output/test_file.rb

@@ -1,60 +0,0 @@
-require_relative '../test_helper'
-
-module TestOutput
-  class TestFile < ::Minitest::Test
-    def test_does_not_handle_stderr_text
-      ::FlatKit::Output::IO::STDERRS.each do |e|
-        refute(::FlatKit::Output::File.handles?(e), "#{e} is not stderr text")
-      end
-    end
-
-    def test_only_handles_string
-      refute(::FlatKit::Output::File.handles?(Object.new))
-    end
-
-    def test_doest_not_handles_stdout_text
-      ::FlatKit::Output::IO::STDOUTS.each do |e|
-        refute(::FlatKit::Output::File.handles?(e), "#{e} is not stdout text")
-      end
-    end
-
-    def test_init_from_path
-      test_path = "tmp/test_init_from_path.txt"
-      begin
-        io = ::FlatKit::Output::File.new(test_path)
-        assert_equal(test_path, io.name)
-        assert_instance_of(::File, io.io)
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_writes_to_file
-      test_path = "tmp/test_writes_to_file.txt"
-      begin
-        output = ::FlatKit::Output::File.new(test_path)
-        assert_equal(test_path, output.name)
-        output.io.write("test_writes_to_file output")
-        output.close
-        t = IO.read(test_path)
-        assert_equal("test_writes_to_file output", t)
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_writes_to_gzfile
-      test_path = "tmp/test_writes_to_gzfile.txt.gz"
-      begin
-        output = ::FlatKit::Output::File.new(test_path)
-        assert_equal(test_path, output.name)
-        output.io.write("test_writes_to_gzfile output")
-        output.close
-        t = %x[ gunzip -c #{test_path} ]
-        assert_equal("test_writes_to_gzfile output", t)
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-  end
-end

data/test/output/test_io.rb

@@ -1,104 +0,0 @@
-require_relative '../test_helper'
-
-module TestOutput
-  class NullIO < ::IO
-    def initialize()
-    end
-  end
-
-  class TestIO < ::Minitest::Test
-    def test_handles_stderr_text
-      ::FlatKit::Output::IO::STDERRS.each do |e|
-        assert(::FlatKit::Output::IO.handles?(e), "#{e} is not stderr text")
-      end
-    end
-
-    def test_handles_stderr_io
-      x = $stderr
-      assert(::FlatKit::Output::IO.handles?(x), "is not stderr")
-    end
-
-    def test_handles_stdout_text
-      ::FlatKit::Output::IO::STDOUTS.each do |e|
-        assert(::FlatKit::Output::IO.handles?(e), "#{e} is not stdout text")
-      end
-    end
-
-    def test_handles_stdout_io
-      x = $stderr
-      assert(::FlatKit::Output::IO.handles?(x), "is not stdout")
-    end
-
-    def test_handles_stringio
-      assert(::FlatKit::Output::IO.handles?(StringIO.new))
-    end
-
-    def test_does_not_handle_other
-      x = Object.new
-      refute(::FlatKit::Output::IO.handles?(x))
-    end
-
-    def test_init_from_dash
-      io = ::FlatKit::Output::IO.new("-")
-      assert_equal("<STDOUT>", io.name)
-      assert_equal(::STDOUT, io.io)
-    end
-
-    def test_init_from_stderr_text
-      io = ::FlatKit::Output::IO.new("stderr")
-      assert_equal("<STDERR>", io.name)
-      assert_equal(::STDERR, io.io)
-    end
-
-    def test_init_from_file_object
-      test_path = "tmp/test_init_from_file_object.txt"
-      begin
-        File.open(test_path, "w") do |f|
-          io = ::FlatKit::Output::IO.new(f)
-          assert_equal(test_path, io.name)
-          assert_instance_of(::File, io.io)
-        end
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-    def test_init_from_stdout
-      io  = ::FlatKit::Output::IO.new($stdout)
-      assert_equal("<STDOUT>", io.name)
-      assert_equal(::STDOUT, io.io)
-    end
-
-    def test_init_from_string_io_object
-      sio = StringIO.new
-      io = ::FlatKit::Output::IO.new(sio)
-      assert_match(/StringIO/, io.name)
-      assert_instance_of(::StringIO, io.io)
-    end
-
-    def test_init_from_io_object
-      null_io = NullIO.new
-      io = ::FlatKit::Output::IO.new(null_io)
-      assert_match(/NullIO/, io.name)
-      assert_instance_of(::TestOutput::NullIO, io.io)
-    end
-
-    def test_writes_to_io
-      test_path = "tmp/test_writes_to_io.txt"
-      begin
-        File.open(test_path, "w") do |f|
-          io = ::FlatKit::Output::IO.new(f)
-          assert_equal(test_path, io.name)
-          assert_instance_of(::File, io.io)
-          io.io.write("test_writes_to_io output")
-          io.close
-        end
-        t = IO.read(test_path)
-        assert_equal("test_writes_to_io output", t)
-      ensure
-        File.unlink(test_path) if File.exist?(test_path)
-      end
-    end
-
-  end
-end

data/test/test_conversions.rb

@@ -1,45 +0,0 @@
-require 'test_helper'
-
-class TestConversions < ::Minitest::Test
-  def setup
-    @one_row_dataset = DeviceDataset.new(count: 1)
-    @src_record = @one_row_dataset.records.first
-    @csv_row = @one_row_dataset.records_as_csv_rows.first
-    @compare_fields = @one_row_dataset.compare_fields
-  end
-
-  def test_from_csv_to_json
-    xsv_record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
-    json_record = FlatKit::Jsonl::Record.from_record(xsv_record)
-
-    assert_equal(@one_row_dataset.records.first, xsv_record.to_hash)
-    assert_equal(@one_row_dataset.records.first, json_record.to_hash)
-    assert_equal(xsv_record, json_record)
-  end
-
-  def test_from_json_to_csv
-    src_json = JSON.generate(@src_record)
-    json_record = FlatKit::Jsonl::Record.new(data: src_json, compare_fields: @compare_fields)
-    xsv_record = FlatKit::Xsv::Record.from_record(json_record)
-
-    assert_equal(@one_row_dataset.records.first, xsv_record.to_hash)
-    assert_equal(@one_row_dataset.records.first, json_record.to_hash)
-    assert_equal(xsv_record, json_record)
-  end
-
-  def test_roundtrip_csv_json_csv
-    xsv_record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
-    json_record = FlatKit::Jsonl::Record.from_record(xsv_record)
-    xsv2 = FlatKit::Xsv::Record.from_record(json_record)
-
-    assert_equal(xsv_record.to_s, xsv2.to_s)
-  end
-
-  def test_roundtrip_json_csv_json
-    src_json = JSON.generate(@src_record)
-    json_record = FlatKit::Jsonl::Record.new(data: src_json, compare_fields: @compare_fields)
-    xsv_record = FlatKit::Xsv::Record.from_record(json_record)
-    json2 = FlatKit::Jsonl::Record.from_record(xsv_record)
-    assert_equal(src_json, json2.to_s)
-  end
-end

data/test/test_event_emitter.rb

@@ -1,72 +0,0 @@
-require 'test_helper'
-
-class TestEventEmitter < ::Minitest::Test
-  class Pub
-    include ::FlatKit::EventEmitter
-  end
-
-  class Sub
-    attr_reader :name
-    attr_reader :data
-
-    def initialize
-      @name = nil
-      @data = nil
-    end
-
-    def on_event(name:, data:)
-      @name = name
-      @data = data
-    end
-  end
-
-  class BadSub; end
-
-  def setup
-    @emitter = Pub.new
-    @receiver = Sub.new
-  end
-
-  def test_counts_no_listeners_before_adding_one
-    assert_equal(0, @emitter.count_listeners)
-  end
-
-  def test_adds_listener
-    @emitter.add_listener(@receiver)
-    assert_equal(1, @emitter.count_listeners)
-  end
-
-  def test_removes_listener
-    @emitter.add_listener(@receiver)
-    assert_equal(1, @emitter.count_listeners)
-
-    @emitter.remove_listener(@receiver)
-    assert_equal(0, @emitter.count_listeners)
-  end
-
-  def test_only_adds_an_listener_once
-    @emitter.add_listener(@receiver)
-    assert_equal(1, @emitter.count_listeners)
-
-    @emitter.add_listener(@receiver)
-    assert_equal(1, @emitter.count_listeners)
-  end
-
-  def test_verifies_reciever_responds_t_observed
-    assert_raises(::NoMethodError) { @emitter.add_listener(BadSub.new) }
-  end
-
-  def test_listeners_get_notified
-    @receiver_2 = Sub.new
-    @emitter.add_listener(@receiver)
-    @emitter.add_listener(@receiver_2)
-
-    @emitter.notify_listeners(name: :notification, data: "DATA!")
-
-    assert_equal(:notification, @receiver.name)
-    assert_equal(:notification, @receiver_2.name)
-
-    assert_equal("DATA!", @receiver.data)
-    assert_equal("DATA!", @receiver_2.data)
-  end
-end

data/test/test_format.rb

@@ -1,24 +0,0 @@
-require_relative 'test_helper'
-
-class TestFormat < ::Minitest::Test
-  def test_finds_jsonl_format
-    klass = ::FlatKit::Format.for("data.json.gz")
-    assert_equal(::FlatKit::Jsonl::Format, klass)
-  end
-
-  def test_finds_xsv_format
-    klass = ::FlatKit::Format.for("data.csv.gz")
-    assert_equal(::FlatKit::Xsv::Format, klass)
-  end
-
-  def test_finds_jsonl_format_for_full_path
-    klass = ::FlatKit::Format.for("tmp/sorted/foo.jsonl")
-    assert_equal(::FlatKit::Jsonl::Format, klass)
-  end
-
-  def test_finds_jsonl_format_with_fallback
-    path = "tmp/sorted/foo.json"
-    klass = ::FlatKit::Format.for_with_fallback!(path: path, fallback: "auto")
-    assert_equal(::FlatKit::Jsonl::Format, klass)
-  end
-end

data/test/test_helper.rb

@@ -1,26 +0,0 @@
-require 'simplecov'
-SimpleCov.start if ENV['COVERAGE']
-
-require 'byebug'
-
-require 'minitest/autorun'
-require 'minitest/focus'
-require 'minitest/pride'
-
-module TestHelper
-  def scratch_dir
-    p = Pathname.new(__FILE__).parent.parent.join('tmp/testing_scratch')
-    p.mkpath
-    p
-  end
-
-  def generate_slug(length: 10)
-    SecureRandom.alphanumeric(10)
-  end
-
-  def scratch_file(prefix: "test_", slug: generate_slug, extension: ".jsonl")
-    scratch_dir.join("#{prefix}#{slug}#{extension}")
-  end
-end
-require_relative '../lib/flat_kit'
-require_relative './device_dataset'

data/test/test_merge.rb

@@ -1,40 +0,0 @@
-require 'test_helper'
-
-class TestMerge < ::Minitest::Test
-
-  def test_can_use_use_dash_as_output
-    merge = ::FlatKit::Merge.new(inputs: [], input_fallback: "json",
-                                 output: "-", output_fallback: "json", compare_fields: [])
-    assert_match(/STDOUT/, merge.writer.output.name)
-    assert_instance_of(::FlatKit::Output::IO, merge.writer.output)
-  end
-
-  def test_can_use_a_text_path_as_output
-    test_path = "tmp/test_can_use_a_text_path_as_output.json"
-    begin
-      merge = ::FlatKit::Merge.new(output: test_path, inputs: [], input_fallback: "json",  compare_fields: [])
-      assert_equal(test_path, merge.writer.output.name)
-      assert_instance_of(::FlatKit::Output::File, merge.writer.output)
-      merge.writer.close
-    ensure
-      File.unlink(test_path) if File.exist?(test_path)
-    end
-  end
-
-  def test_can_use_a_pathname_as_output
-    test_path = Pathname.new("tmp/test_can_use_a_pathname_as_output.json")
-    begin
-      merge = ::FlatKit::Merge.new(output: test_path, inputs: [], input_fallback: "json", compare_fields: [])
-      assert_equal(test_path.to_s, merge.writer.output.name)
-      assert_instance_of(::FlatKit::Output::File, merge.writer.output)
-      merge.writer.close
-    ensure
-      test_path.unlink if test_path.exist?
-    end
-  end
-
-  def test_raises_error_if_unable_to_parse_output
-    test_path = Object.new
-    assert_raises(FlatKit::Error) { ::FlatKit::Merge.new(output: test_path, inputs: [], compare_fields: []) }
-  end
-end

data/test/test_merge_tree.rb

@@ -1,64 +0,0 @@
-require 'test_helper'
-
-class TestMergeTree < ::Minitest::Test
-  def setup
-    @dataset_count = 20
-    @records_per_dataset = 100
-    @records = []
-    @datasets = Array.new.tap do |a|
-      @dataset_count.times do
-        dd = DeviceDataset.new(count: @records_per_dataset)
-        dd.persist_sorted_records_as_jsonl
-        @records.concat(dd.records)
-        a << dd
-      end
-    end
-    @compare_fields = @datasets.first.compare_fields
-    @readers = @datasets.map { |dd|
-      ::FlatKit::Jsonl::Reader.new(source: dd.filename_sorted_jsonl, compare_fields: @compare_fields)
-    }
-  end
-
-  def teardown
-    @datasets.each do |ds|
-      ds.cleanup_files
-    end
-  end
-
-  def test_init_tree
-    tree = ::FlatKit::MergeTree.new(@readers)
-    assert_equal(20, tree.leaves.size)
-
-    assert_equal(5, tree.depth)
-
-    # 0th level should have 10 nodes - since 20 leaves
-    assert_equal(10, tree.levels[0].size)
-
-    # 1st level should have 5 nodes - since 10 nodes lower
-    assert_equal(5, tree.levels[1].size)
-
-    # 2nd level should have 3 nodes - since 5 above (and we shim in a Sentinel 
-    # node on the last internal node)
-    assert_equal(3, tree.levels[2].size)
-    assert_instance_of(::FlatKit::SentinelInternalNode, tree.levels[2].last.right)
-
-    # 3rd level should have 2 nodes
-    assert_equal(2, tree.levels[3].size)
-
-    # 4th level should have 1 nodes
-    assert_equal(1, tree.levels[4].size)
-  end
-
-  def test_merging
-    expected_records = @records.sort_by { |r| @compare_fields.map { |f| r[f] } }
-    tree = ::FlatKit::MergeTree.new(@readers)
-    actual_records = tree.to_a.map { |r| r.to_hash }
-
-    assert_equal(expected_records.size, actual_records.size)
-
-    expected_records.each_with_index do |expected, idx|
-      actual = actual_records[idx]
-      assert_equal(expected, actual)
-    end
-  end
-end

data/test/test_version.rb

@@ -1,11 +0,0 @@
-require 'test_helper'
-
-class TestVersion < ::Minitest::Test
-  def test_version_constant_match
-    assert_match(/\A\d+\.\d+\.\d+\Z/, FlatKit::VERSION)
-  end
-
-  def test_version_string_match
-    assert_match(/\A\d+\.\d+\.\d+\Z/, FlatKit::VERSION.to_s)
-  end
-end

data/test/xsv/test_format.rb

@@ -1,22 +0,0 @@
-require_relative '../test_helper'
-
-module TestXsv
-  class TestFormat < ::Minitest::Test
-
-    def test_handles_csv
-      assert(::FlatKit::Xsv::Format.handles?("csv"))
-    end
-
-    def test_handles_tsv
-      assert(::FlatKit::Xsv::Format.handles?("tsv"))
-    end
-
-    def test_handles_txt
-      assert(::FlatKit::Xsv::Format.handles?("txt"))
-    end
-
-    def test_does_not_handle_json
-      refute(::FlatKit::Xsv::Format.handles?("json"))
-    end
-  end
-end

data/test/xsv/test_reader.rb

@@ -1,61 +0,0 @@
-require_relative '../test_helper'
-
-module TestXsv
-  class TestReader < ::Minitest::Test
-    def setup
-      @count = 20
-      @dataset = DeviceDataset.new(count: @count)
-      @compare_fields = @dataset.compare_fields
-      @test_path = "tmp/test_reads_from_io.csv"
-
-      File.open(@test_path, "wb") do |f|
-        f.write(@dataset.records_as_csv)
-      end
-    end
-
-    def teardown
-      File.unlink(@test_path) if File.exist?(@test_path)
-    end
-
-    def test_fields
-      reader = ::FlatKit::Xsv::Reader.new(source: @test_path, compare_fields: @compare_fields)
-      reader.to_a
-      assert_equal(@dataset.fields, reader.fields)
-    end
-
-    def test_raises_error_on_invalid_source
-      assert_raises(::FlatKit::Error) {
-        ::FlatKit::Xsv::Reader.new(source: Object.new, compare_fields: nil)
-      }
-    end
-
-    def test_automatically_figures_out_fields_if_needed
-      reader = ::FlatKit::Xsv::Reader.new(source: @test_path)
-      reader.take(1)
-      assert_equal(@dataset.fields, reader.fields)
-    end
-
-    def test_reads_from_pathname
-      reader = ::FlatKit::Xsv::Reader.new(source: @test_path, compare_fields: @compare_fields)
-      all = reader.to_a
-      assert_equal(@count, reader.count)
-      assert_equal(@count, all.size)
-    end
-
-    def test_reads_from_io
-      File.open(@test_path) do |f|
-        reader = ::FlatKit::Xsv::Reader.new(source: f, compare_fields: @compare_fields)
-        all = reader.to_a
-        assert_equal(@count, reader.count)
-        assert_equal(@count, all.size)
-      end
-    end
-
-    def test_raises_error_on_io_error
-      s = StringIO.new
-      s.close_read
-      reader = ::FlatKit::Xsv::Reader.new(source: s, compare_fields: @compare_fields)
-      assert_raises(::FlatKit::Error) { reader.to_a }
-    end
-  end
-end

data/test/xsv/test_record.rb

@@ -1,69 +0,0 @@
-require 'test_helper'
-require 'faker'
-require 'byebug'
-
-module TestXsv
-  class TestRecord< ::Minitest::Test
-    def setup
-      @one_row_dataset = DeviceDataset.new(count: 1)
-      @csv_row = @one_row_dataset.records_as_csv_rows.first
-      @compare_fields = @one_row_dataset.compare_fields
-    end
-
-    def test_initializes_from_data
-      record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
-      original_record = @one_row_dataset.records.first
-      @compare_fields.each do |field|
-        assert_equal(original_record[field], record[field])
-      end
-    end
-
-    def test_ignores_non_compare_fields_values
-      record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
-      refute(record["version"])
-    end
-
-    def test_is_sortable
-      dataset = DeviceDataset.new(count: 20)
-      fk_records = Array.new.tap do |a|
-        dataset.records_as_csv_rows.each do |csv_row|
-          a << FlatKit::Xsv::Record.new(data: csv_row, compare_fields: @compare_fields)
-        end
-      end
-
-      sorted = fk_records.sort
-      output_text = CSV.generate('', headers: dataset.fields, write_headers: true) do |csv|
-        sorted.each do |row|
-          csv << row.data
-        end
-      end
-
-      assert_equal(output_text, dataset.sorted_records_as_csv)
-    end
-
-    def test_to_hash
-      record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
-      h = record.to_hash
-      assert_equal(@one_row_dataset.records.first, h)
-    end
-
-    def test_from_record
-      rec1 = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
-      rec2 = FlatKit::Xsv::Record.from_record(rec1)
-      assert_equal(rec1, rec2)
-    end
-
-    def test_incomplete_initialization
-      assert_raises(FlatKit::Error) {
-        FlatKit::Xsv::Record.new(data: nil, compare_fields: [])
-      }
-    end
-
-    def test_to_s_from_csv_record
-      record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
-      line = record.to_s
-      expected = @one_row_dataset.records_as_csv_rows[0].to_csv
-      assert_equal(expected, line)
-    end
-  end
-end