flat_kit 0.3.0 → 1.0.0

data/test/run

@@ -1,23 +0,0 @@
-#!/usr/bin/env ruby
-
-
-test_dir = __dir__
-
-$: << File.join(File.dirname(test_dir), "lib")
-$: << test_dir
-
-require_relative './test_helper'
-
-require 'find'
-
-if ARGV.empty? then
-  Find.find(test_dir) do |path|
-    next unless File.file?(path) && File.basename(path) =~ /\Atest_.*\.rb\Z/
-    require path
-  end
-else
-  ARGV.each do |f|
-    require File.expand_path(f)
-  end
-end
-

data/test/stat_type/test_nominal_stats.rb

@@ -1,69 +0,0 @@
-require_relative '../test_helper'
-
-module TestStatType
-  class TestNominalStats < ::Minitest::Test
-    def setup
-      @unique_values = ('a'..'f').to_a
-      @values = Array.new.tap do |a|
-        @unique_values.each do |letter|
-          (Random.rand(42) + 1).times { a << letter }
-        end
-      end
-
-      @frequencies = @values.tally
-
-      @stats = ::FlatKit::StatType::NominalStats.new
-      @all_stats = ::FlatKit::StatType::NominalStats.new(collecting_frequencies: true)
-
-      @values.each do |v|
-        @stats.update(v)
-        @all_stats.update(v)
-      end
-    end
-
-    def test_count
-      assert_equal(@values.size, @stats.count)
-      assert_equal(@values.size, @all_stats.count)
-    end
-
-    def test_does_not_collect_unique_count_by_default
-      assert_nil(@stats.unique_count)
-    end
-
-    def test_does_not_collect_unique_values_by_default
-      assert_nil(@stats.unique_values)
-    end
-
-    def test_does_not_collect_frequencies_by_default
-      assert_nil(@stats.frequencies)
-    end
-
-    def test_unique_count
-      assert_equal(@unique_values.size, @all_stats.unique_count)
-    end
-
-    def test_unique_values
-      assert_equal(@unique_values.sort, @all_stats.unique_values.sort)
-    end
-
-    def test_frequencies
-      assert_equal(@frequencies, @all_stats.frequencies)
-    end
-
-    def test_default_to_hash
-      expecting = { "count" => @values.size }
-      assert_equal(expecting, @stats.to_hash)
-    end
-
-    def test_all_stats_hash
-      expecting = {
-        "count" => @values.size,
-        "unique_count" => @unique_values.size,
-        "unique_values" => @unique_values.sort,
-        "mode" => @frequencies.max_by { |k,v| v }.first
-      }
-      assert_equal(expecting, @all_stats.to_hash)
-    end
-  end
-end
-

data/test/stat_type/test_numerical_stats.rb

@@ -1,118 +0,0 @@
-require_relative '../test_helper'
-
-
-module TestStatType
-  class TestNumericalStats < ::Minitest::Test
-    def setup
-      @stats = FlatKit::StatType::NumericalStats.new
-      @full_stats = FlatKit::StatType::NumericalStats.new
-      @all_stats = FlatKit::StatType::NumericalStats.new(collecting_frequencies: true)
-      [ 1, 2, 3].each { |i| @full_stats.update( i ) }
-    end
-
-    def test_intialized_with_usable_values
-      assert_equal(0, @stats.count)
-      assert_equal(Float::INFINITY, @stats.min)
-      assert_equal(-Float::INFINITY, @stats.max)
-      assert_equal(0.0, @stats.sum)
-      assert_equal(0.0, @stats.rate)
-    end
-
-    def test_calculates_mean
-      assert_equal(2.0, @full_stats.mean)
-    end
-
-    def test_calculates_rate
-      assert_equal(0.5, @full_stats.rate)
-    end
-
-    def test_tracks_the_maximum_value
-      assert_equal(3.0, @full_stats.max)
-    end
-
-    def test_tracks_the_minimum_value
-      assert_equal(1.0, @full_stats.min)
-    end
-
-    def test_tracks_the_count
-      assert_equal(3,@full_stats.count)
-    end
-
-    def test_tracks_the_sum
-      assert_equal(6.0, @full_stats.sum)
-    end
-
-    def test_calculates_the_standard_deviation
-      assert_equal(1.0, @full_stats.stddev)
-    end 
-
-    def test_calculates_the_sum_of_squares
-      assert_equal(14, @full_stats.sumsq)
-    end 
-
-    def test_converts_to_a_hash
-      h = @full_stats.to_hash
-      assert_equal(::FlatKit::StatType::NumericalStats.default_stats.size, h.size)
-      assert_equal(::FlatKit::StatType::NumericalStats.default_stats, h.keys.sort)
-    end
-
-    def test_converts_to_a_limited_hash_if_given_arguments
-      h = @full_stats.to_hash( "min", "max", "mean" )
-      assert_equal(3, h.size)
-      assert_equal(%w[ max mean min], h.keys.sort)
-
-      h = @full_stats.to_hash( %w[ count rate ] )
-      assert_equal(2, h.size)
-      assert_equal(%w[ count rate ], h.keys.sort)
-    end
-
-    def test_raises_nomethoderror_if_an_invalid_stat_is_used
-      assert_raises(NoMethodError) { @full_stats.to_hash( "wibble" ) }
-    end
-
-    def test_converts_to_a_json_string
-      j = @full_stats.to_json
-      h = JSON.parse( j )
-      assert_equal(::FlatKit::StatType::NumericalStats.default_stats.size, h.size)
-      assert_equal(::FlatKit::StatType::NumericalStats.default_stats, h.keys.sort)
-    end
-
-    def test_converts_to_a_limited_json_hash_if_given_arguments
-      j = @full_stats.to_json( "min", "max", "mean" )
-      h = JSON.parse( j )
-      assert_equal(3, h.size)
-      assert_equal(%w[ max mean min], h.keys.sort)
-
-      j = @full_stats.to_json( %w[ count rate ] )
-      h = JSON.parse( j )
-      assert_equal(2, h.size)
-      assert_equal(%w[ count rate ], h.keys.sort)
-    end
-
-    def test_raises_nomethoderror_if_an_invalid_json_stat_is_used
-      assert_raises(NoMethodError) { @full_stats.to_json( "wibble" ) }
-    end
-
-    def test_collects_mode
-      values = Array.new.tap do |a|
-        100.times {
-          n = Random.rand(10)
-          a << n
-          @all_stats.update(n)
-        }
-      end
-
-      tally = values.tally
-      mode_value = tally.max_by { |v, count| count }.first
-
-      assert_equal(mode_value, @all_stats.mode)
-    end
-
-    def test_collecting_frequences_reports_extra_stat_names
-      stat_names = @all_stats.collected_stats
-      assert_includes(stat_names, "mode")
-      assert_includes(stat_names, "unique_count")
-      assert_includes(stat_names, "unique_values")
-    end
-  end
-end

data/test/stat_type/test_ordinal_stats.rb

@@ -1,92 +0,0 @@
-require_relative '../test_helper'
-
-module TestStatType
-  class TestOrdinalStats < ::Minitest::Test
-    def setup
-      today = Date.today
-      next_month = today >> 1
-      last_day_of_month = (next_month - 1).mday
-
-      @start_date = Date.new(today.year, today.month, 1)
-      @end_date   = Date.new(today.year, today.month, last_day_of_month)
-
-      @unique_values = (@start_date..@end_date).to_a
-      @values = Array.new.tap do |a|
-        @unique_values.each do |date|
-          (Random.rand(42) + 1).times { a << date}
-        end
-      end
-
-      @frequencies = @values.tally
-
-      @stats = ::FlatKit::StatType::OrdinalStats.new
-      @all_stats = ::FlatKit::StatType::OrdinalStats.new(collecting_frequencies: true)
-
-      @values.each do |v|
-        @stats.update(v)
-        @all_stats.update(v)
-      end
-    end
-
-    def test_count
-      assert_equal(@values.size, @stats.count)
-      assert_equal(@values.size, @all_stats.count)
-    end
-
-    def test_min
-      assert_equal(@values.min, @stats.min)
-      assert_equal(@values.min, @all_stats.min)
-    end
-
-    def test_max
-      assert_equal(@values.max, @stats.max)
-      assert_equal(@values.max, @all_stats.max)
-    end
-
-    def test_does_not_collect_unique_count_by_default
-      assert_nil(@stats.unique_count)
-    end
-
-    def test_does_not_collect_unique_values_by_default
-      assert_nil(@stats.unique_values)
-    end
-
-    def test_does_not_collect_frequencies_by_default
-      assert_nil(@stats.frequencies)
-    end
-
-    def test_unique_count
-      assert_equal(@unique_values.size, @all_stats.unique_count)
-    end
-
-    def test_unique_values
-      assert_equal(@unique_values.sort, @all_stats.unique_values.sort)
-    end
-
-    def test_frequencies
-      assert_equal(@frequencies, @all_stats.frequencies)
-    end
-
-    def test_default_to_hash
-      expecting = {
-        "count" => @values.size,
-        "max"   => @values.max,
-        "min"   => @values.min,
-      }
-      assert_equal(expecting, @stats.to_hash)
-    end
-
-    def test_all_stats_hash
-      expecting = {
-        "count"         => @values.size,
-        "unique_count"  => @unique_values.size,
-        "unique_values" => @unique_values.sort,
-        "mode"          => @frequencies.max_by { |k,v| v }.first,
-        "max"           => @values.max,
-        "min"           => @values.min,
-      }
-      assert_equal(expecting, @all_stats.to_hash)
-    end
-  end
-end
-

data/test/test_conversions.rb

@@ -1,45 +0,0 @@
-require 'test_helper'
-
-class TestConversions < ::Minitest::Test
-  def setup
-    @one_row_dataset = DeviceDataset.new(count: 1)
-    @src_record = @one_row_dataset.records.first
-    @csv_row = @one_row_dataset.records_as_csv_rows.first
-    @compare_fields = @one_row_dataset.compare_fields
-  end
-
-  def test_from_csv_to_json
-    xsv_record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
-    json_record = FlatKit::Jsonl::Record.from_record(xsv_record)
-
-    assert_equal(@one_row_dataset.records.first, xsv_record.to_hash)
-    assert_equal(@one_row_dataset.records.first, json_record.to_hash)
-    assert_equal(xsv_record, json_record)
-  end
-
-  def test_from_json_to_csv
-    src_json = JSON.generate(@src_record)
-    json_record = FlatKit::Jsonl::Record.new(data: src_json, compare_fields: @compare_fields)
-    xsv_record = FlatKit::Xsv::Record.from_record(json_record)
-
-    assert_equal(@one_row_dataset.records.first, xsv_record.to_hash)
-    assert_equal(@one_row_dataset.records.first, json_record.to_hash)
-    assert_equal(xsv_record, json_record)
-  end
-
-  def test_roundtrip_csv_json_csv
-    xsv_record = FlatKit::Xsv::Record.new(data: @csv_row, compare_fields: @compare_fields)
-    json_record = FlatKit::Jsonl::Record.from_record(xsv_record)
-    xsv2 = FlatKit::Xsv::Record.from_record(json_record)
-
-    assert_equal(xsv_record.to_s, xsv2.to_s)
-  end
-
-  def test_roundtrip_json_csv_json
-    src_json = JSON.generate(@src_record)
-    json_record = FlatKit::Jsonl::Record.new(data: src_json, compare_fields: @compare_fields)
-    xsv_record = FlatKit::Xsv::Record.from_record(json_record)
-    json2 = FlatKit::Jsonl::Record.from_record(xsv_record)
-    assert_equal(src_json, json2.to_s)
-  end
-end

data/test/test_event_emitter.rb

@@ -1,89 +0,0 @@
-require 'test_helper'
-
-class TestEventEmitter < ::Minitest::Test
-  class Pub
-    include ::FlatKit::EventEmitter
-  end
-
-  class Sub
-    attr_reader :name
-    attr_reader :data
-    attr_reader :meta
-
-    def initialize
-      @name = nil
-      @data = nil
-      @meta = nil
-    end
-
-    def [](key)
-      @meta[key]
-    end
-
-    def on_event(name:, data:, meta:)
-      @name = name
-      @data = data
-      @meta = meta
-    end
-  end
-
-  class BadSub; end
-
-  def setup
-    @emitter = Pub.new
-    @receiver = Sub.new
-  end
-
-  def test_counts_no_listeners_before_adding_one
-    assert_equal(0, @emitter.count_listeners)
-  end
-
-  def test_adds_listener
-    @emitter.add_listener(@receiver)
-    assert_equal(1, @emitter.count_listeners)
-  end
-
-  def test_removes_listener
-    @emitter.add_listener(@receiver)
-    assert_equal(1, @emitter.count_listeners)
-
-    @emitter.remove_listener(@receiver)
-    assert_equal(0, @emitter.count_listeners)
-  end
-
-  def test_only_adds_an_listener_once
-    @emitter.add_listener(@receiver)
-    assert_equal(1, @emitter.count_listeners)
-
-    @emitter.add_listener(@receiver)
-    assert_equal(1, @emitter.count_listeners)
-  end
-
-  def test_verifies_reciever_responds_t_observed
-    assert_raises(::NoMethodError) { @emitter.add_listener(BadSub.new) }
-  end
-
-  def test_listeners_get_notified
-    @receiver_2 = Sub.new
-    @emitter.add_listener(@receiver)
-    @emitter.add_listener(@receiver_2)
-
-    meta = {
-      foo: "foo",
-      bar: 42,
-    }
-    @emitter.notify_listeners(name: :notification, data: "DATA!", meta: meta)
-
-    assert_equal(:notification, @receiver.name)
-    assert_equal(:notification, @receiver_2.name)
-
-    assert_equal("DATA!", @receiver.data)
-    assert_equal("DATA!", @receiver_2.data)
-
-    assert_equal("foo", @receiver[:foo])
-    assert_equal("foo", @receiver_2[:foo])
-
-    assert_equal(42, @receiver[:bar])
-    assert_equal(42, @receiver_2[:bar])
-  end
-end

data/test/test_field_stats.rb

@@ -1,134 +0,0 @@
-require_relative './test_helper'
-
-require 'faker'
-
-class TestFieldStats < ::Minitest::Test
-  # returns [FieldStats, Array] where the array is the original data
-  def generate_data_with(count: 100, stats: ::FlatKit::FieldStats.new(name: "data"), &block)
-    list = Array.new.tap do |a|
-      count.times do
-        n = block.call
-        stats.update(n)
-        a << n
-      end
-    end
-    [stats, list]
-  end
-
-  def test_raises_error_on_invalid_stats
-    assert_raises(ArgumentError) { ::FlatKit::FieldStats.new(name: "test", stats_to_collect: :whatever) }
-  end
-
-  def test_collects_numeric_default_stats
-    field_stats, number_data = generate_data_with { Faker::Number.within(range: 1.0..100.0) }
-
-    null_count = 5
-    null_count.times {
-      field_stats.update(nil)
-    }
-
-    avg = number_data.sum / number_data.size
-    min = number_data.min
-    max = number_data.max
-    sum = number_data.sum
-
-    refute(field_stats.field_type_determined?)
-
-    assert_equal(null_count, field_stats.null_count)
-    assert_equal(number_data.size, field_stats.count)
-
-    assert(field_stats.field_type_determined?)
-
-    assert_in_epsilon(avg, field_stats.mean)
-    assert_equal(min, field_stats.min)
-    assert_equal(max, field_stats.max)
-    assert_in_epsilon(sum, field_stats.sum)
-    expected_percent = (null_count.to_f / (null_count + number_data.size)) * 100.0
-    assert_in_epsilon(expected_percent, field_stats.null_percent)
-  end
-
-  def test_collect_numeric_cardinality_stats
-    field_stats = ::FlatKit::FieldStats.new(name: "number-cardinality",
-                                            stats_to_collect: ::FlatKit::FieldStats::ALL_STATS)
-    field_stats, number_data = generate_data_with(stats: field_stats) {
-        Faker::Number.within(range:1..25)
-    }
-
-    avg = number_data.sum.to_f / number_data.size
-    min = number_data.min
-    max = number_data.max
-
-    assert(field_stats.collecting_frequencies?)
-    refute(field_stats.field_type_determined?)
-
-    assert_equal(number_data.size, field_stats.count)
-
-    assert(field_stats.field_type_determined?)
-
-    assert_in_epsilon(avg, field_stats.mean)
-    assert_equal(min, field_stats.min)
-    assert_equal(max, field_stats.max)
-
-    assert_equal(number_data.tally.keys.size, field_stats.unique_count)
-    assert_equal(number_data.tally.keys.sort, field_stats.unique_values.sort)
-    assert_equal(number_data.tally, field_stats.frequencies)
-
-    mode = number_data.tally.max_by{ |k,v| v }.first
-    assert_equal(mode, field_stats.mode)
-  end
-
-  def test_unknown_type_stats
-    field_stats = ::FlatKit::FieldStats.new(name: "numeric-with-unknown")
-    field_stats, number_data = generate_data_with { Faker::Number.within(range: 1.0..100.0) }
-
-    unknown_count = 20
-    unknown_count.times {
-      field_stats.update("unknown")
-    }
-
-    refute(field_stats.field_type_determined?)
-
-    assert_equal(unknown_count, field_stats.unknown_count)
-    assert_equal(unknown_count + number_data.size, field_stats.total_count)
-
-    expected_percent = (unknown_count.to_f / (unknown_count + number_data.size)) * 100.0
-
-    assert_in_epsilon(expected_percent, field_stats.unknown_percent)
-  end
-
-  def test_resolves_type_automatically
-    field_stats = ::FlatKit::FieldStats.new(name: "numeric-autoresolve",guess_threshold: 101)
-    field_stats, _ = generate_data_with(stats: field_stats) { Faker::Number.within(range: 1.0..100.0) }
-
-    refute(field_stats.field_type_determined?)
-    field_stats, _ = generate_data_with(stats: field_stats) { Faker::Number.within(range: 200.0..300.0) }
-    assert(field_stats.field_type_determined?)
-  end
-
-  def test_resolves_integer_appropriately_with_mixed_data
-    field_stats = ::FlatKit::FieldStats.new(name: "numeric-integer",guess_threshold: 100)
-    field_stats, _ = generate_data_with(count: 40, stats: field_stats) { Faker::Number.within(range: 0..1).to_s }
-    field_stats, _ = generate_data_with(count: 70, stats: field_stats) { Faker::Number.within(range: 0..200).to_s }
-
-    assert_equal(::FlatKit::FieldType::IntegerType, field_stats.field_type)
-
-  end
-
-  def test_resolves_boolean_appropriately_with_mixed_data
-    field_stats = ::FlatKit::FieldStats.new(name: "numeric-integer",guess_threshold: 100)
-    field_stats, _ = generate_data_with(count: 70, stats: field_stats) { Faker::Boolean.boolean.to_s }
-    field_stats, _ = generate_data_with(count: 40, stats: field_stats) { Faker::Number.within(range: 0..200).to_s }
-    assert_equal(::FlatKit::FieldType::BooleanType, field_stats.field_type)
-  end
-
-  def test_resolves_string_appropriately_with_mixed_data
-    field_stats = ::FlatKit::FieldStats.new(name: "string",guess_threshold: 100)
-    field_stats, _ = generate_data_with(count: 61, stats: field_stats) { Faker::Color.name.to_s }
-    field_stats, _ = generate_data_with(count: 59, stats: field_stats) { Faker::Number.within(range: 0..200).to_s }
-    assert_equal(::FlatKit::FieldType::StringType, field_stats.field_type)
-
-    assert_equal(120, field_stats.count)
-    assert_equal(0, field_stats.unknown_count)
-    assert_equal(0, field_stats.null_count)
-  end
-end

data/test/test_field_type.rb

@@ -1,34 +0,0 @@
-require_relative './test_helper'
-
-module TestFieldType
-  class TestFieldType < ::Minitest::Test
-
-    def test_weight_raises_exception
-      assert_raises(NotImplementedError) { ::FlatKit::FieldType.weight }
-    end
-
-    def test_best_guesses
-      guesses = {
-        "t"          => ::FlatKit::FieldType::BooleanType,
-        "1"          => ::FlatKit::FieldType::BooleanType,
-        "0"          => ::FlatKit::FieldType::BooleanType,
-        "n"          => ::FlatKit::FieldType::BooleanType,
-        "42"         => ::FlatKit::FieldType::IntegerType,
-        "nil"        => ::FlatKit::FieldType::NullType,
-        "n/a"        => ::FlatKit::FieldType::UnknownType,
-        "foo"        => ::FlatKit::FieldType::StringType,
-        "12.3"       => ::FlatKit::FieldType::FloatType,
-        "2021-02-26" => ::FlatKit::FieldType::DateType,
-        "2020-03-03T12:34:56Z" => ::FlatKit::FieldType::TimestampType,
-      }
-
-      guesses.each do |test, expected|
-        assert_equal(expected, ::FlatKit::FieldType.best_guess(test), "Expected '#{test}' to be #{expected}")
-      end
-    end
-
-    def test_children_exist
-      assert_equal(9,::FlatKit::FieldType.children.size)
-    end
-  end
-end

data/test/test_format.rb

@@ -1,24 +0,0 @@
-require_relative 'test_helper'
-
-class TestFormat < ::Minitest::Test
-  def test_finds_jsonl_format
-    klass = ::FlatKit::Format.for("data.json.gz")
-    assert_equal(::FlatKit::Jsonl::Format, klass)
-  end
-
-  def test_finds_xsv_format
-    klass = ::FlatKit::Format.for("data.csv.gz")
-    assert_equal(::FlatKit::Xsv::Format, klass)
-  end
-
-  def test_finds_jsonl_format_for_full_path
-    klass = ::FlatKit::Format.for("tmp/sorted/foo.jsonl")
-    assert_equal(::FlatKit::Jsonl::Format, klass)
-  end
-
-  def test_finds_jsonl_format_with_fallback
-    path = "tmp/sorted/foo.json"
-    klass = ::FlatKit::Format.for_with_fallback!(path: path, fallback: "auto")
-    assert_equal(::FlatKit::Jsonl::Format, klass)
-  end
-end

data/test/test_helper.rb

@@ -1,26 +0,0 @@
-require 'simplecov'
-SimpleCov.start if ENV['COVERAGE']
-
-require 'byebug'
-
-require 'minitest/autorun'
-require 'minitest/focus'
-require 'minitest/pride'
-
-module TestHelper
-  def scratch_dir
-    p = Pathname.new(__FILE__).parent.parent.join('tmp/testing_scratch')
-    p.mkpath
-    p
-  end
-
-  def generate_slug(length: 10)
-    SecureRandom.alphanumeric(10)
-  end
-
-  def scratch_file(prefix: "test_", slug: generate_slug, extension: ".jsonl")
-    scratch_dir.join("#{prefix}#{slug}#{extension}")
-  end
-end
-require_relative '../lib/flat_kit'
-require_relative './device_dataset'

data/test/test_merge.rb

@@ -1,40 +0,0 @@
-require 'test_helper'
-
-class TestMerge < ::Minitest::Test
-
-  def test_can_use_use_dash_as_output
-    merge = ::FlatKit::Merge.new(inputs: [], input_fallback: "json",
-                                 output: "-", output_fallback: "json", compare_fields: [])
-    assert_match(/STDOUT/, merge.writer.output.name)
-    assert_instance_of(::FlatKit::Output::IO, merge.writer.output)
-  end
-
-  def test_can_use_a_text_path_as_output
-    test_path = "tmp/test_can_use_a_text_path_as_output.json"
-    begin
-      merge = ::FlatKit::Merge.new(output: test_path, inputs: [], input_fallback: "json",  compare_fields: [])
-      assert_equal(test_path, merge.writer.output.name)
-      assert_instance_of(::FlatKit::Output::File, merge.writer.output)
-      merge.writer.close
-    ensure
-      File.unlink(test_path) if File.exist?(test_path)
-    end
-  end
-
-  def test_can_use_a_pathname_as_output
-    test_path = Pathname.new("tmp/test_can_use_a_pathname_as_output.json")
-    begin
-      merge = ::FlatKit::Merge.new(output: test_path, inputs: [], input_fallback: "json", compare_fields: [])
-      assert_equal(test_path.to_s, merge.writer.output.name)
-      assert_instance_of(::FlatKit::Output::File, merge.writer.output)
-      merge.writer.close
-    ensure
-      test_path.unlink if test_path.exist?
-    end
-  end
-
-  def test_raises_error_if_unable_to_parse_output
-    test_path = Object.new
-    assert_raises(FlatKit::Error) { ::FlatKit::Merge.new(output: test_path, inputs: [], compare_fields: []) }
-  end
-end