flat_kit 0.2.0 → 0.3.0

data/test/stat_type/test_ordinal_stats.rb

@@ -0,0 +1,92 @@
+require_relative '../test_helper'
+
+module TestStatType
+  class TestOrdinalStats < ::Minitest::Test
+    def setup
+      today = Date.today
+      next_month = today >> 1
+      last_day_of_month = (next_month - 1).mday
+
+      @start_date = Date.new(today.year, today.month, 1)
+      @end_date   = Date.new(today.year, today.month, last_day_of_month)
+
+      @unique_values = (@start_date..@end_date).to_a
+      @values = Array.new.tap do |a|
+        @unique_values.each do |date|
+          (Random.rand(42) + 1).times { a << date}
+        end
+      end
+
+      @frequencies = @values.tally
+
+      @stats = ::FlatKit::StatType::OrdinalStats.new
+      @all_stats = ::FlatKit::StatType::OrdinalStats.new(collecting_frequencies: true)
+
+      @values.each do |v|
+        @stats.update(v)
+        @all_stats.update(v)
+      end
+    end
+
+    def test_count
+      assert_equal(@values.size, @stats.count)
+      assert_equal(@values.size, @all_stats.count)
+    end
+
+    def test_min
+      assert_equal(@values.min, @stats.min)
+      assert_equal(@values.min, @all_stats.min)
+    end
+
+    def test_max
+      assert_equal(@values.max, @stats.max)
+      assert_equal(@values.max, @all_stats.max)
+    end
+
+    def test_does_not_collect_unique_count_by_default
+      assert_nil(@stats.unique_count)
+    end
+
+    def test_does_not_collect_unique_values_by_default
+      assert_nil(@stats.unique_values)
+    end
+
+    def test_does_not_collect_frequencies_by_default
+      assert_nil(@stats.frequencies)
+    end
+
+    def test_unique_count
+      assert_equal(@unique_values.size, @all_stats.unique_count)
+    end
+
+    def test_unique_values
+      assert_equal(@unique_values.sort, @all_stats.unique_values.sort)
+    end
+
+    def test_frequencies
+      assert_equal(@frequencies, @all_stats.frequencies)
+    end
+
+    def test_default_to_hash
+      expecting = {
+        "count" => @values.size,
+        "max"   => @values.max,
+        "min"   => @values.min,
+      }
+      assert_equal(expecting, @stats.to_hash)
+    end
+
+    def test_all_stats_hash
+      expecting = {
+        "count"         => @values.size,
+        "unique_count"  => @unique_values.size,
+        "unique_values" => @unique_values.sort,
+        "mode"          => @frequencies.max_by { |k,v| v }.first,
+        "max"           => @values.max,
+        "min"           => @values.min,
+      }
+      assert_equal(expecting, @all_stats.to_hash)
+    end
+  end
+end
+

data/test/test_event_emitter.rb

@@ -8,15 +8,22 @@ class TestEventEmitter < ::Minitest::Test
   class Sub
     attr_reader :name
     attr_reader :data
+    attr_reader :meta
 
     def initialize
       @name = nil
       @data = nil
+      @meta = nil
     end
 
-    def on_event(name:, data:)
+    def [](key)
+      @meta[key]
+    end
+
+    def on_event(name:, data:, meta:)
       @name = name
       @data = data
+      @meta = meta
     end
   end
 
@@ -61,12 +68,22 @@ class TestEventEmitter < ::Minitest::Test
     @emitter.add_listener(@receiver)
     @emitter.add_listener(@receiver_2)
 
-    @emitter.notify_listeners(name: :notification, data: "DATA!")
+    meta = {
+      foo: "foo",
+      bar: 42,
+    }
+    @emitter.notify_listeners(name: :notification, data: "DATA!", meta: meta)
 
     assert_equal(:notification, @receiver.name)
     assert_equal(:notification, @receiver_2.name)
 
     assert_equal("DATA!", @receiver.data)
     assert_equal("DATA!", @receiver_2.data)
+
+    assert_equal("foo", @receiver[:foo])
+    assert_equal("foo", @receiver_2[:foo])
+
+    assert_equal(42, @receiver[:bar])
+    assert_equal(42, @receiver_2[:bar])
   end
 end

data/test/test_field_stats.rb

@@ -0,0 +1,134 @@
+require_relative './test_helper'
+
+require 'faker'
+
+class TestFieldStats < ::Minitest::Test
+  # returns [FieldStats, Array] where the array is the original data
+  def generate_data_with(count: 100, stats: ::FlatKit::FieldStats.new(name: "data"), &block)
+    list = Array.new.tap do |a|
+      count.times do
+        n = block.call
+        stats.update(n)
+        a << n
+      end
+    end
+    [stats, list]
+  end
+
+  def test_raises_error_on_invalid_stats
+    assert_raises(ArgumentError) { ::FlatKit::FieldStats.new(name: "test", stats_to_collect: :whatever) }
+  end
+
+  def test_collects_numeric_default_stats
+    field_stats, number_data = generate_data_with { Faker::Number.within(range: 1.0..100.0) }
+
+    null_count = 5
+    null_count.times {
+      field_stats.update(nil)
+    }
+
+    avg = number_data.sum / number_data.size
+    min = number_data.min
+    max = number_data.max
+    sum = number_data.sum
+
+    refute(field_stats.field_type_determined?)
+
+    assert_equal(null_count, field_stats.null_count)
+    assert_equal(number_data.size, field_stats.count)
+
+    assert(field_stats.field_type_determined?)
+
+    assert_in_epsilon(avg, field_stats.mean)
+    assert_equal(min, field_stats.min)
+    assert_equal(max, field_stats.max)
+    assert_in_epsilon(sum, field_stats.sum)
+    expected_percent = (null_count.to_f / (null_count + number_data.size)) * 100.0
+    assert_in_epsilon(expected_percent, field_stats.null_percent)
+  end
+
+  def test_collect_numeric_cardinality_stats
+    field_stats = ::FlatKit::FieldStats.new(name: "number-cardinality",
+                                            stats_to_collect: ::FlatKit::FieldStats::ALL_STATS)
+    field_stats, number_data = generate_data_with(stats: field_stats) {
+        Faker::Number.within(range:1..25)
+    }
+
+    avg = number_data.sum.to_f / number_data.size
+    min = number_data.min
+    max = number_data.max
+
+    assert(field_stats.collecting_frequencies?)
+    refute(field_stats.field_type_determined?)
+
+    assert_equal(number_data.size, field_stats.count)
+
+    assert(field_stats.field_type_determined?)
+
+    assert_in_epsilon(avg, field_stats.mean)
+    assert_equal(min, field_stats.min)
+    assert_equal(max, field_stats.max)
+
+    assert_equal(number_data.tally.keys.size, field_stats.unique_count)
+    assert_equal(number_data.tally.keys.sort, field_stats.unique_values.sort)
+    assert_equal(number_data.tally, field_stats.frequencies)
+
+    mode = number_data.tally.max_by{ |k,v| v }.first
+    assert_equal(mode, field_stats.mode)
+  end
+
+  def test_unknown_type_stats
+    field_stats = ::FlatKit::FieldStats.new(name: "numeric-with-unknown")
+    field_stats, number_data = generate_data_with { Faker::Number.within(range: 1.0..100.0) }
+
+    unknown_count = 20
+    unknown_count.times {
+      field_stats.update("unknown")
+    }
+
+    refute(field_stats.field_type_determined?)
+
+    assert_equal(unknown_count, field_stats.unknown_count)
+    assert_equal(unknown_count + number_data.size, field_stats.total_count)
+
+    expected_percent = (unknown_count.to_f / (unknown_count + number_data.size)) * 100.0
+
+    assert_in_epsilon(expected_percent, field_stats.unknown_percent)
+  end
+
+  def test_resolves_type_automatically
+    field_stats = ::FlatKit::FieldStats.new(name: "numeric-autoresolve",guess_threshold: 101)
+    field_stats, _ = generate_data_with(stats: field_stats) { Faker::Number.within(range: 1.0..100.0) }
+
+    refute(field_stats.field_type_determined?)
+    field_stats, _ = generate_data_with(stats: field_stats) { Faker::Number.within(range: 200.0..300.0) }
+    assert(field_stats.field_type_determined?)
+  end
+
+  def test_resolves_integer_appropriately_with_mixed_data
+    field_stats = ::FlatKit::FieldStats.new(name: "numeric-integer",guess_threshold: 100)
+    field_stats, _ = generate_data_with(count: 40, stats: field_stats) { Faker::Number.within(range: 0..1).to_s }
+    field_stats, _ = generate_data_with(count: 70, stats: field_stats) { Faker::Number.within(range: 0..200).to_s }
+
+    assert_equal(::FlatKit::FieldType::IntegerType, field_stats.field_type)
+
+  end
+
+  def test_resolves_boolean_appropriately_with_mixed_data
+    field_stats = ::FlatKit::FieldStats.new(name: "numeric-integer",guess_threshold: 100)
+    field_stats, _ = generate_data_with(count: 70, stats: field_stats) { Faker::Boolean.boolean.to_s }
+    field_stats, _ = generate_data_with(count: 40, stats: field_stats) { Faker::Number.within(range: 0..200).to_s }
+    assert_equal(::FlatKit::FieldType::BooleanType, field_stats.field_type)
+  end
+
+  def test_resolves_string_appropriately_with_mixed_data
+    field_stats = ::FlatKit::FieldStats.new(name: "string",guess_threshold: 100)
+    field_stats, _ = generate_data_with(count: 61, stats: field_stats) { Faker::Color.name.to_s }
+    field_stats, _ = generate_data_with(count: 59, stats: field_stats) { Faker::Number.within(range: 0..200).to_s }
+    assert_equal(::FlatKit::FieldType::StringType, field_stats.field_type)
+
+    assert_equal(120, field_stats.count)
+    assert_equal(0, field_stats.unknown_count)
+    assert_equal(0, field_stats.null_count)
+  end
+end

data/test/test_field_type.rb

@@ -0,0 +1,34 @@
+require_relative './test_helper'
+
+module TestFieldType
+  class TestFieldType < ::Minitest::Test
+
+    def test_weight_raises_exception
+      assert_raises(NotImplementedError) { ::FlatKit::FieldType.weight }
+    end
+
+    def test_best_guesses
+      guesses = {
+        "t"          => ::FlatKit::FieldType::BooleanType,
+        "1"          => ::FlatKit::FieldType::BooleanType,
+        "0"          => ::FlatKit::FieldType::BooleanType,
+        "n"          => ::FlatKit::FieldType::BooleanType,
+        "42"         => ::FlatKit::FieldType::IntegerType,
+        "nil"        => ::FlatKit::FieldType::NullType,
+        "n/a"        => ::FlatKit::FieldType::UnknownType,
+        "foo"        => ::FlatKit::FieldType::StringType,
+        "12.3"       => ::FlatKit::FieldType::FloatType,
+        "2021-02-26" => ::FlatKit::FieldType::DateType,
+        "2020-03-03T12:34:56Z" => ::FlatKit::FieldType::TimestampType,
+      }
+
+      guesses.each do |test, expected|
+        assert_equal(expected, ::FlatKit::FieldType.best_guess(test), "Expected '#{test}' to be #{expected}")
+      end
+    end
+
+    def test_children_exist
+      assert_equal(9,::FlatKit::FieldType.children.size)
+    end
+  end
+end

data/test/xsv/test_writer.rb

@@ -41,12 +41,33 @@ module TestXsv
       assert_equal(expected, actual)
     end
 
-    def test_writes_to_io
+    def test_position
       File.open(@write_path, "w+") do |f|
-        writer = ::FlatKit::Xsv::Writer.new(destination: f,fields: @reader.fields)
+        writer = ::FlatKit::Xsv::Writer.new(destination: f,fields: :auto)
+        records_bytes = 0
+        header_bytes = nil
+
+        @records.each_with_index do |r, idx|
+          record_length = r.to_s.bytesize
+
+          position = writer.write(r)
+          # make sure write stores the last_position api and returns that value
+          assert_equal(position, writer.last_position)
+
+          header_bytes = writer.header_bytes if header_bytes == nil
+          assert(header_bytes > 0)
+
+          assert_equal(idx, position.index)
+          assert_equal(header_bytes + records_bytes, position.offset)
+          assert_equal(record_length, position.bytesize)
+
+          records_bytes += record_length
+
+          current_position = writer.current_position
+          assert_equal(idx+1, current_position.index)
+          assert_equal(header_bytes + records_bytes, current_position.offset)
+          assert_equal(0, current_position.bytesize)
 
-        @records.each do |r|
-          writer.write(r)
         end
         writer.close
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: flat_kit
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Jeremy Hinegardner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-03 00:00:00.000000000 Z
+date: 2021-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: oj
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: faker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.16'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -128,15 +142,28 @@ files:
 - README.md
 - Rakefile
 - bin/fk
+- examples/stream-active-record-to-csv.rb
 - lib/flat_kit.rb
 - lib/flat_kit/cli.rb
 - lib/flat_kit/command.rb
 - lib/flat_kit/command/cat.rb
 - lib/flat_kit/command/merge.rb
 - lib/flat_kit/command/sort.rb
+- lib/flat_kit/command/stats.rb
 - lib/flat_kit/descendant_tracker.rb
 - lib/flat_kit/error.rb
 - lib/flat_kit/event_emitter.rb
+- lib/flat_kit/field_stats.rb
+- lib/flat_kit/field_type.rb
+- lib/flat_kit/field_type/boolean_type.rb
+- lib/flat_kit/field_type/date_type.rb
+- lib/flat_kit/field_type/float_type.rb
+- lib/flat_kit/field_type/guess_type.rb
+- lib/flat_kit/field_type/integer_type.rb
+- lib/flat_kit/field_type/null_type.rb
+- lib/flat_kit/field_type/string_type.rb
+- lib/flat_kit/field_type/timestamp_type.rb
+- lib/flat_kit/field_type/unknown_type.rb
 - lib/flat_kit/format.rb
 - lib/flat_kit/input.rb
 - lib/flat_kit/input/file.rb
@@ -154,11 +181,17 @@ files:
 - lib/flat_kit/output.rb
 - lib/flat_kit/output/file.rb
 - lib/flat_kit/output/io.rb
+- lib/flat_kit/position.rb
 - lib/flat_kit/reader.rb
 - lib/flat_kit/record.rb
 - lib/flat_kit/sentinel_internal_node.rb
 - lib/flat_kit/sentinel_leaf_node.rb
 - lib/flat_kit/sort.rb
+- lib/flat_kit/stat_type.rb
+- lib/flat_kit/stat_type/nominal_stats.rb
+- lib/flat_kit/stat_type/numerical_stats.rb
+- lib/flat_kit/stat_type/ordinal_stats.rb
+- lib/flat_kit/stats.rb
 - lib/flat_kit/writer.rb
 - lib/flat_kit/xsv.rb
 - lib/flat_kit/xsv/format.rb
@@ -170,6 +203,15 @@ files:
 - tasks/man.rake
 - tasks/this.rb
 - test/device_dataset.rb
+- test/field_type/test_boolean_type.rb
+- test/field_type/test_date_type.rb
+- test/field_type/test_float_type.rb
+- test/field_type/test_guess_type.rb
+- test/field_type/test_integer_type.rb
+- test/field_type/test_null_type.rb
+- test/field_type/test_string_type.rb
+- test/field_type/test_timestamp_type.rb
+- test/field_type/test_unknown_type.rb
 - test/input/test_file.rb
 - test/input/test_io.rb
 - test/jsonl/test_format.rb
@@ -178,8 +220,14 @@ files:
 - test/jsonl/test_writer.rb
 - test/output/test_file.rb
 - test/output/test_io.rb
+- test/run
+- test/stat_type/test_nominal_stats.rb
+- test/stat_type/test_numerical_stats.rb
+- test/stat_type/test_ordinal_stats.rb
 - test/test_conversions.rb
 - test/test_event_emitter.rb
+- test/test_field_stats.rb
+- test/test_field_type.rb
 - test/test_format.rb
 - test/test_helper.rb
 - test/test_merge.rb
@@ -219,6 +267,15 @@ summary: A library and commandline program for reading, writing, indexing, sorti
   and merging CSV, TSV, JSON and other flat-file formats.
 test_files:
 - test/device_dataset.rb
+- test/field_type/test_boolean_type.rb
+- test/field_type/test_date_type.rb
+- test/field_type/test_float_type.rb
+- test/field_type/test_guess_type.rb
+- test/field_type/test_integer_type.rb
+- test/field_type/test_null_type.rb
+- test/field_type/test_string_type.rb
+- test/field_type/test_timestamp_type.rb
+- test/field_type/test_unknown_type.rb
 - test/input/test_file.rb
 - test/input/test_io.rb
 - test/jsonl/test_format.rb
@@ -227,8 +284,14 @@ test_files:
 - test/jsonl/test_writer.rb
 - test/output/test_file.rb
 - test/output/test_io.rb
+- test/run
+- test/stat_type/test_nominal_stats.rb
+- test/stat_type/test_numerical_stats.rb
+- test/stat_type/test_ordinal_stats.rb
 - test/test_conversions.rb
 - test/test_event_emitter.rb
+- test/test_field_stats.rb
+- test/test_field_type.rb
 - test/test_format.rb
 - test/test_helper.rb
 - test/test_merge.rb