flat_kit 0.1.0

data/test/test_merge.rb

@@ -0,0 +1,40 @@
+require 'test_helper'
+
+class TestMerge < ::Minitest::Test
+
+  def test_can_use_use_dash_as_output
+    merge = ::FlatKit::Merge.new(inputs: [], input_fallback: "json",
+                                 output: "-", output_fallback: "json", compare_fields: [])
+    assert_match(/STDOUT/, merge.writer.output.name)
+    assert_instance_of(::FlatKit::Output::IO, merge.writer.output)
+  end
+
+  def test_can_use_a_text_path_as_output
+    test_path = "tmp/test_can_use_a_text_path_as_output.json"
+    begin
+      merge = ::FlatKit::Merge.new(output: test_path, inputs: [], input_fallback: "json",  compare_fields: [])
+      assert_equal(test_path, merge.writer.output.name)
+      assert_instance_of(::FlatKit::Output::File, merge.writer.output)
+      merge.writer.close
+    ensure
+      File.unlink(test_path) if File.exist?(test_path)
+    end
+  end
+
+  def test_can_use_a_pathname_as_output
+    test_path = Pathname.new("tmp/test_can_use_a_pathname_as_output.json")
+    begin
+      merge = ::FlatKit::Merge.new(output: test_path, inputs: [], input_fallback: "json", compare_fields: [])
+      assert_equal(test_path.to_s, merge.writer.output.name)
+      assert_instance_of(::FlatKit::Output::File, merge.writer.output)
+      merge.writer.close
+    ensure
+      test_path.unlink if test_path.exist?
+    end
+  end
+
+  def test_raises_error_if_unable_to_parse_output
+    test_path = Object.new
+    assert_raises(FlatKit::Error) { ::FlatKit::Merge.new(output: test_path, inputs: [], compare_fields: []) }
+  end
+end

data/test/test_merge_tree.rb

@@ -0,0 +1,64 @@
+require 'test_helper'
+
+class TestMergeTree < ::Minitest::Test
+  def setup
+    @dataset_count = 20
+    @records_per_dataset = 100
+    @records = []
+    @datasets = Array.new.tap do |a|
+      @dataset_count.times do
+        dd = DeviceDataset.new(count: @records_per_dataset)
+        dd.persist_sorted_records_as_jsonl
+        @records.concat(dd.records)
+        a << dd
+      end
+    end
+    @compare_fields = @datasets.first.compare_fields
+    @readers = @datasets.map { |dd|
+      ::FlatKit::Jsonl::Reader.new(source: dd.filename_sorted_jsonl, compare_fields: @compare_fields)
+    }
+  end
+
+  def teardown
+    @datasets.each do |ds|
+      ds.cleanup_files
+    end
+  end
+
+  def test_init_tree
+    tree = ::FlatKit::MergeTree.new(@readers)
+    assert_equal(20, tree.leaves.size)
+
+    assert_equal(5, tree.depth)
+
+    # 0th level should have 10 nodes - since 20 leaves
+    assert_equal(10, tree.levels[0].size)
+
+    # 1st level should have 5 nodes - since 10 nodes lower
+    assert_equal(5, tree.levels[1].size)
+
+    # 2nd level should have 3 nodes - since 5 above (and we shim in a Sentinel 
+    # node on the last internal node)
+    assert_equal(3, tree.levels[2].size)
+    assert_instance_of(::FlatKit::SentinelInternalNode, tree.levels[2].last.right)
+
+    # 3rd level should have 2 nodes
+    assert_equal(2, tree.levels[3].size)
+
+    # 4th level should have 1 nodes
+    assert_equal(1, tree.levels[4].size)
+  end
+
+  def test_merging
+    expected_records = @records.sort_by { |r| @compare_fields.map { |f| r[f] } }
+    tree = ::FlatKit::MergeTree.new(@readers)
+    actual_records = tree.to_a.map { |r| r.to_hash }
+
+    assert_equal(expected_records.size, actual_records.size)
+
+    expected_records.each_with_index do |expected, idx|
+      actual = actual_records[idx]
+      assert_equal(expected, actual)
+    end
+  end
+end

data/test/test_version.rb

@@ -0,0 +1,11 @@
+require 'test_helper'
+
+class TestVersion < ::Minitest::Test
+  def test_version_constant_match
+    assert_match(/\A\d+\.\d+\.\d+\Z/, FlatKit::VERSION)
+  end
+
+  def test_version_string_match
+    assert_match(/\A\d+\.\d+\.\d+\Z/, FlatKit::VERSION.to_s)
+  end
+end

data/test/xsv/test_format.rb

@@ -0,0 +1,22 @@
+require_relative '../test_helper'
+
+module TestXsv
+  class TestFormat < ::Minitest::Test
+
+    def test_handles_csv
+      assert(::FlatKit::Xsv::Format.handles?("csv"))
+    end
+
+    def test_handles_tsv
+      assert(::FlatKit::Xsv::Format.handles?("tsv"))
+    end
+
+    def test_handles_txt
+      assert(::FlatKit::Xsv::Format.handles?("txt"))
+    end
+
+    def test_does_not_handle_json
+      refute(::FlatKit::Xsv::Format.handles?("json"))
+    end
+  end
+end

data/test/xsv/test_reader.rb

@@ -0,0 +1,61 @@
+require_relative '../test_helper'
+
+module TestXsv
+  class TestReader < ::Minitest::Test
+    def setup
+      @count = 20
+      @dataset = DeviceDataset.new(count: @count)
+      @compare_fields = @dataset.compare_fields
+      @test_path = "tmp/test_reads_from_io.csv"
+
+      File.open(@test_path, "wb") do |f|
+        f.write(@dataset.records_as_csv)
+      end
+    end
+
+    def teardown
+      File.unlink(@test_path) if File.exist?(@test_path)
+    end
+
+    def test_fields
+      reader = ::FlatKit::Xsv::Reader.new(source: @test_path, compare_fields: @compare_fields)
+      reader.to_a
+      assert_equal(@dataset.fields, reader.fields)
+    end
+
+    def test_raises_error_on_invalid_source
+      assert_raises(::FlatKit::Error) {
+        ::FlatKit::Xsv::Reader.new(source: Object.new, compare_fields: nil)
+      }
+    end
+
+    def test_automatically_figures_out_fields_if_needed
+      reader = ::FlatKit::Xsv::Reader.new(source: @test_path)
+      reader.take(1)
+      assert_equal(@dataset.fields, reader.fields)
+    end
+
+    def test_reads_from_pathname
+      reader = ::FlatKit::Xsv::Reader.new(source: @test_path, compare_fields: @compare_fields)
+      all = reader.to_a
+      assert_equal(@count, reader.count)
+      assert_equal(@count, all.size)
+    end
+
+    def test_reads_from_io
+      File.open(@test_path) do |f|
+        reader = ::FlatKit::Xsv::Reader.new(source: f, compare_fields: @compare_fields)
+        all = reader.to_a
+        assert_equal(@count, reader.count)
+        assert_equal(@count, all.size)
+      end
+    end
+
+    def test_raises_error_on_io_error
+      s = StringIO.new
+      s.close_read
+      reader = ::FlatKit::Xsv::Reader.new(source: s, compare_fields: @compare_fields)
+      assert_raises(::FlatKit::Error) { reader.to_a }
+    end
+  end
+end

data/test/xsv/test_record.rb

@@ -0,0 +1,69 @@
+require 'test_helper'
+require 'faker'
+require 'byebug'
+
+module TestXsv
+  class TestRecord< ::Minitest::Test
+    def setup
+      @one_row_dataset = DeviceDataset.new(count: 1)
+      @csv_row = @one_row_dataset.records_as_csv_rows.first
+      @compare_fields = @one_row_dataset.compare_fields
+    end
+
+    def test_initializes_from_data
+      record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
+      original_record = @one_row_dataset.records.first
+      @compare_fields.each do |field|
+        assert_equal(original_record[field], record[field])
+      end
+    end
+
+    def test_ignores_non_compare_fields_values
+      record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
+      refute(record["version"])
+    end
+
+    def test_is_sortable
+      dataset = DeviceDataset.new(count: 20)
+      fk_records = Array.new.tap do |a|
+        dataset.records_as_csv_rows.each do |csv_row|
+          a << FlatKit::Xsv::Record.new(data: csv_row, compare_fields: @compare_fields)
+        end
+      end
+
+      sorted = fk_records.sort
+      output_text = CSV.generate('', headers: dataset.fields, write_headers: true) do |csv|
+        sorted.each do |row|
+          csv << row.data
+        end
+      end
+
+      assert_equal(output_text, dataset.sorted_records_as_csv)
+    end
+
+    def test_to_hash
+      record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
+      h = record.to_hash
+      assert_equal(@one_row_dataset.records.first, h)
+    end
+
+    def test_from_record
+      rec1 = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
+      rec2 = FlatKit::Xsv::Record.from_record(rec1)
+      assert_equal(rec1, rec2)
+    end
+
+    def test_incomplete_initialization
+      assert_raises(FlatKit::Error) {
+        FlatKit::Xsv::Record.new(data: nil, compare_fields: [])
+      }
+    end
+
+    def test_to_s_from_csv_record
+      record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
+      line = record.to_s
+      expected = @one_row_dataset.records_as_csv_rows[0].to_csv
+      assert_equal(expected, line)
+    end
+  end
+end

data/test/xsv/test_writer.rb

@@ -0,0 +1,68 @@
+require_relative '../test_helper'
+
+module TestXsv
+  class TestWriter < ::Minitest::Test
+    def setup
+      @count = 20
+      @dataset = DeviceDataset.new(count: @count)
+      @compare_fields = @dataset.compare_fields
+      @write_path = "tmp/test_writes_to_io.csv"
+      @read_path = "tmp/test_read.csv"
+
+      File.open(@read_path, "wb") do |f|
+        f.write(@dataset.records_as_csv)
+      end
+
+      @reader = ::FlatKit::Xsv::Reader.new(source: @read_path, compare_fields: @compare_fields)
+      @records = @reader.to_a
+    end
+
+    def teardown
+      File.unlink(@write_path) if File.exist?(@write_path)
+      File.unlink(@read_path) if File.exist?(@read_path)
+    end
+
+    def test_raises_error_on_invalid_destination
+      assert_raises(::FlatKit::Error) {
+        ::FlatKit::Xsv::Writer.new(destination: Object.new, fields: @reader.fields)
+      }
+    end
+
+    def test_writes_to_pathname
+      writer = ::FlatKit::Xsv::Writer.new(destination: @write_path, fields: @reader.fields)
+      @records.each do |r|
+        writer.write(r)
+      end
+      writer.close
+      assert_equal(@count, writer.count)
+
+      expected = @dataset.records_as_csv
+      actual = IO.read(@write_path)
+      assert_equal(expected, actual)
+    end
+
+    def test_writes_to_io
+      File.open(@write_path, "w+") do |f|
+        writer = ::FlatKit::Xsv::Writer.new(destination: f,fields: @reader.fields)
+
+        @records.each do |r|
+          writer.write(r)
+        end
+        writer.close
+
+        assert_equal(@count, writer.count)
+
+        expected = @dataset.records_as_csv
+        actual = IO.read(@write_path)
+        assert_equal(expected, actual)
+      end
+    end
+
+    def test_raises_error_on_io_error
+      s = StringIO.new
+      writer = ::FlatKit::Xsv::Writer.new(destination: s, fields: @reader.fields)
+      s.close_write
+      assert_raises(::FlatKit::Error) { writer.write(@records.first) }
+    end
+  end
+end

metadata

@@ -0,0 +1,237 @@
+--- !ruby/object:Gem::Specification
+name: flat_kit
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jeremy Hinegardner
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-01-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: optimist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.11'
+- !ruby/object:Gem::Dependency
+  name: minitest-focus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.3'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+description: A library and commandline program for reading, writing, indexing,  sorting,
+  and merging CSV, TSV, JSON and other flat-file formats.
+email: jeremy@copiousfreetime.org
+executables:
+- fk
+extensions: []
+extra_rdoc_files:
+- CONTRIBUTING.md
+- HISTORY.md
+- LICENSE.txt
+- Manifest.txt
+- README.md
+files:
+- CONTRIBUTING.md
+- HISTORY.md
+- LICENSE.txt
+- Manifest.txt
+- README.md
+- Rakefile
+- bin/fk
+- lib/flat_kit.rb
+- lib/flat_kit/cli.rb
+- lib/flat_kit/command.rb
+- lib/flat_kit/command/cat.rb
+- lib/flat_kit/command/merge.rb
+- lib/flat_kit/command/sort.rb
+- lib/flat_kit/descendant_tracker.rb
+- lib/flat_kit/error.rb
+- lib/flat_kit/format.rb
+- lib/flat_kit/input.rb
+- lib/flat_kit/input/file.rb
+- lib/flat_kit/input/io.rb
+- lib/flat_kit/internal_node.rb
+- lib/flat_kit/jsonl.rb
+- lib/flat_kit/jsonl/format.rb
+- lib/flat_kit/jsonl/reader.rb
+- lib/flat_kit/jsonl/record.rb
+- lib/flat_kit/jsonl/writer.rb
+- lib/flat_kit/leaf_node.rb
+- lib/flat_kit/logger.rb
+- lib/flat_kit/merge.rb
+- lib/flat_kit/merge_tree.rb
+- lib/flat_kit/output.rb
+- lib/flat_kit/output/file.rb
+- lib/flat_kit/output/io.rb
+- lib/flat_kit/reader.rb
+- lib/flat_kit/record.rb
+- lib/flat_kit/sentinel_internal_node.rb
+- lib/flat_kit/sentinel_leaf_node.rb
+- lib/flat_kit/sort.rb
+- lib/flat_kit/writer.rb
+- lib/flat_kit/xsv.rb
+- lib/flat_kit/xsv/format.rb
+- lib/flat_kit/xsv/reader.rb
+- lib/flat_kit/xsv/record.rb
+- lib/flat_kit/xsv/writer.rb
+- tasks/default.rake
+- tasks/extension.rake
+- tasks/man.rake
+- tasks/this.rb
+- test/device_dataset.rb
+- test/input/test_file.rb
+- test/input/test_io.rb
+- test/jsonl/test_format.rb
+- test/jsonl/test_reader.rb
+- test/jsonl/test_record.rb
+- test/jsonl/test_writer.rb
+- test/output/test_file.rb
+- test/output/test_io.rb
+- test/test_conversions.rb
+- test/test_format.rb
+- test/test_helper.rb
+- test/test_merge.rb
+- test/test_merge_tree.rb
+- test/test_version.rb
+- test/xsv/test_format.rb
+- test/xsv/test_reader.rb
+- test/xsv/test_record.rb
+- test/xsv/test_writer.rb
+homepage: http://github.com/copiousfreetime/flat_kit
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options:
+- "--main"
+- README.md
+- "--markup"
+- tomdoc
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: A library and commandline program for reading, writing, indexing, sorting,
+  and merging CSV, TSV, JSON and other flat-file formats.
+test_files:
+- test/device_dataset.rb
+- test/input/test_file.rb
+- test/input/test_io.rb
+- test/jsonl/test_format.rb
+- test/jsonl/test_reader.rb
+- test/jsonl/test_record.rb
+- test/jsonl/test_writer.rb
+- test/output/test_file.rb
+- test/output/test_io.rb
+- test/test_conversions.rb
+- test/test_format.rb
+- test/test_helper.rb
+- test/test_merge.rb
+- test/test_merge_tree.rb
+- test/test_version.rb
+- test/xsv/test_format.rb
+- test/xsv/test_reader.rb
+- test/xsv/test_record.rb
+- test/xsv/test_writer.rb