flat_kit 0.2.0 → 0.3.0

data/test/field_type/test_null_type.rb

@@ -0,0 +1,41 @@
+require_relative '../test_helper'
+
+module TestFieldType
+  class TestNullType < ::Minitest::Test
+    def nully_items
+      [ "null", "NULL", "nil", "\\N" ]
+    end
+
+    def test_null
+      assert(FlatKit::FieldType::NullType.matches?(nil))
+    end
+
+    def test_nully_items
+      nully_items.each do |s|
+        assert(FlatKit::FieldType::NullType.matches?(s), "#{s} should be null")
+      end
+    end
+
+    def test_other_class_does_not_match
+      [ 42, Object.new, true, false ].each do |x|
+        refute(FlatKit::FieldType::NullType.matches?(x), "#{x} should not be == null")
+      end
+    end
+
+    def test_coerce_coerces_nil
+      assert_nil(FlatKit::FieldType::NullType.coerce(nil))
+    end
+
+    def test_coerces_nully_items
+      nully_items.each do |s|
+        assert_nil(FlatKit::FieldType::NullType.coerce(s))
+      end
+    end
+
+    def test_coerce_failure_non_non_nully_items
+      [ "whatever", 42, Object.new, true, false, Class].each do |x|
+        assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::NullType.coerce(x))
+      end
+    end
+  end
+end

data/test/field_type/test_string_type.rb

@@ -0,0 +1,18 @@
+require_relative '../test_helper'
+
+module TestFieldType
+  class TestStringType < ::Minitest::Test
+
+    def test_string_will_not_match_non_string_data
+      [42, false, true, 12.5, Object.new].each do |o|
+        refute(FlatKit::FieldType::StringType.matches?(o))
+      end
+    end
+
+    def test_string_type_returns_coerce_failures
+      [BasicObject.new].each do |o|
+        assert_equal(::FlatKit::FieldType::CoerceFailure, ::FlatKit::FieldType::StringType.coerce(o))
+      end
+    end
+  end
+end

data/test/field_type/test_timestamp_type.rb

@@ -0,0 +1,108 @@
+require_relative '../test_helper'
+require 'set'
+
+module TestFieldType
+  class TestTimestampType < ::Minitest::Test
+    def test_time
+      assert(FlatKit::FieldType::TimestampType.matches?(Time.now))
+    end
+
+    def test_date
+      refute(FlatKit::FieldType::TimestampType.matches?(Date.today))
+    end
+
+    def test_date_time
+      refute(FlatKit::FieldType::TimestampType.matches?(DateTime.now))
+    end
+
+    def test_builtin_formats
+      stock_formats = [
+        Time.now.httpdate,
+        Time.now.utc.httpdate,
+        Time.now.iso8601,
+        Time.now.utc.iso8601,
+        Time.now.rfc2822,
+        Time.now.utc.rfc2822,
+      ]
+
+      stock_formats.each do |t|
+        assert(FlatKit::FieldType::TimestampType.matches?(t), "#{t} should match timestamp")
+        assert_instance_of(Time, FlatKit::FieldType::TimestampType.coerce(t), "#{t} should convert to timestamp")
+      end
+    end
+
+    def test_no_duplicate_formats
+      parse_formats = ::FlatKit::FieldType::TimestampType.parse_formats
+
+      assert_equal(parse_formats.size, parse_formats.sort.uniq.size)
+    end
+
+    def test_parse_formats
+      parse_formats = ::FlatKit::FieldType::TimestampType.parse_formats.dup
+      extra_formats = [
+          "%Y-%m-%dT%H:%M:%S.%N%z",
+          "%Y-%m-%d %H:%M:%S.%NZ",
+          "%Y-%m-%d %H:%M:%S.%N",
+          "%Y-%m-%dT%H:%M:%S.%3N%z",
+          "%Y-%m-%d %H:%M:%S.%3NZ",
+          "%Y-%m-%d %H:%M:%S.%3N",
+          "%Y-%m-%dT%H:%M:%S.%6N%z",
+          "%Y-%m-%d %H:%M:%S.%6NZ",
+          "%Y-%m-%d %H:%M:%S.%6N",
+          "%Y-%m-%dT%H:%M:%S.%9N%z",
+          "%Y-%m-%d %H:%M:%S.%9NZ",
+          "%Y-%m-%d %H:%M:%S.%9N",
+      ]
+      parse_formats.concat(extra_formats)
+
+      parse_formats.each do |format|
+        now = Time.now
+        str = now.strftime(format)
+
+        assert(FlatKit::FieldType::TimestampType.matches?(str), "#{str} should match timestamp")
+        coerced = FlatKit::FieldType::TimestampType.coerce(str)
+
+        assert_instance_of(Time, coerced)
+      end
+    end
+
+    def test_other_class_does_not_match
+      [ 42, Object.new, true, false ].each do |x|
+        refute(FlatKit::FieldType::TimestampType.matches?(x), "#{x} should not be date")
+      end
+    end
+
+    def test_N_number_does_not_match
+      x = "N89362"
+      refute(FlatKit::FieldType::TimestampType.matches?(x), "#{x} should not be date")
+    end
+
+    def test_coerce_passthrough_time
+      t = Time.now
+      assert_equal(t, FlatKit::FieldType::TimestampType.coerce(t))
+    end
+
+    def test_coerce_do_not_passthrough_date
+      t = Date.today
+      assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::TimestampType.coerce(t))
+    end
+
+    def test_date_coerce_passthrough_datetime
+      t = Time.now
+      assert_equal(t, FlatKit::FieldType::TimestampType.coerce(t))
+    end
+
+    def test_date_only_does_not_parse
+      t = Time.now.strftime("%Y-%m-%d")
+      assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::TimestampType.coerce(t))
+    end
+
+    def test_number_coerce_failure
+      assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::TimestampType.coerce(42))
+    end
+
+    def test_number_coerce_failure_bad_parse
+      assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::TimestampType.coerce("1234 56 78 90"))
+    end
+  end
+end

data/test/field_type/test_unknown_type.rb

@@ -0,0 +1,35 @@
+require_relative '../test_helper'
+
+module TestFieldType
+  class TestUnknownType < ::Minitest::Test
+
+    def unknown_items
+      [ 'na', 'n/a', 'unk', 'unknown']
+    end
+
+    def test_unknown_items
+      unknown_items.each do |u|
+        assert(FlatKit::FieldType::UnknownType.matches?(u), "#{u} should be unknown")
+      end
+    end
+
+    def test_other_class_does_not_match
+      [ 42, Object.new, true, false, "whatever" ].each do |x|
+        refute(FlatKit::FieldType::UnknownType.matches?(x), "#{x} should not unknown ")
+      end
+    end
+
+    def test_coerce_unknown
+      unknown_items.each do |u|
+        assert_equal(u, FlatKit::FieldType::UnknownType.coerce(u), "#{u} should be unknown")
+      end
+    end
+
+    def test_other_class_does_not_coerce
+      [ 42, Object.new, true, false, "whatever" ].each do |x|
+        assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::UnknownType.coerce(x))
+      end
+    end
+
+  end
+end

data/test/jsonl/test_writer.rb

@@ -41,12 +41,30 @@ module TestJsonl
       assert_equal(expected, actual)
     end
 
-    def test_writes_to_io
+    def test_postion
       File.open(@write_path, "w+") do |f|
         writer = ::FlatKit::Jsonl::Writer.new(destination: f)
 
-        @records.each do |r|
-          writer.write(r)
+        byte_offset = 0
+        @records.each_with_index do |r, idx|
+          record_length = r.data.bytesize
+
+          position = writer.write(r)
+
+          # make sure write stores the last_position api and returns that value
+          assert_equal(position, writer.last_position)
+
+          assert_equal(idx, position.index)
+          assert_equal(byte_offset, position.offset)
+          assert_equal(record_length, position.bytesize)
+
+          byte_offset += record_length
+
+          current_position = writer.current_position
+          assert_equal(idx+1, current_position.index)
+          assert_equal(byte_offset, current_position.offset)
+          assert_equal(0, current_position.bytesize)
+
         end
         writer.close
 

data/test/run

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+
+test_dir = __dir__
+
+$: << File.join(File.dirname(test_dir), "lib")
+$: << test_dir
+
+require_relative './test_helper'
+
+require 'find'
+
+if ARGV.empty? then
+  Find.find(test_dir) do |path|
+    next unless File.file?(path) && File.basename(path) =~ /\Atest_.*\.rb\Z/
+    require path
+  end
+else
+  ARGV.each do |f|
+    require File.expand_path(f)
+  end
+end
+

data/test/stat_type/test_nominal_stats.rb

@@ -0,0 +1,69 @@
+require_relative '../test_helper'
+
+module TestStatType
+  class TestNominalStats < ::Minitest::Test
+    def setup
+      @unique_values = ('a'..'f').to_a
+      @values = Array.new.tap do |a|
+        @unique_values.each do |letter|
+          (Random.rand(42) + 1).times { a << letter }
+        end
+      end
+
+      @frequencies = @values.tally
+
+      @stats = ::FlatKit::StatType::NominalStats.new
+      @all_stats = ::FlatKit::StatType::NominalStats.new(collecting_frequencies: true)
+
+      @values.each do |v|
+        @stats.update(v)
+        @all_stats.update(v)
+      end
+    end
+
+    def test_count
+      assert_equal(@values.size, @stats.count)
+      assert_equal(@values.size, @all_stats.count)
+    end
+
+    def test_does_not_collect_unique_count_by_default
+      assert_nil(@stats.unique_count)
+    end
+
+    def test_does_not_collect_unique_values_by_default
+      assert_nil(@stats.unique_values)
+    end
+
+    def test_does_not_collect_frequencies_by_default
+      assert_nil(@stats.frequencies)
+    end
+
+    def test_unique_count
+      assert_equal(@unique_values.size, @all_stats.unique_count)
+    end
+
+    def test_unique_values
+      assert_equal(@unique_values.sort, @all_stats.unique_values.sort)
+    end
+
+    def test_frequencies
+      assert_equal(@frequencies, @all_stats.frequencies)
+    end
+
+    def test_default_to_hash
+      expecting = { "count" => @values.size }
+      assert_equal(expecting, @stats.to_hash)
+    end
+
+    def test_all_stats_hash
+      expecting = {
+        "count" => @values.size,
+        "unique_count" => @unique_values.size,
+        "unique_values" => @unique_values.sort,
+        "mode" => @frequencies.max_by { |k,v| v }.first
+      }
+      assert_equal(expecting, @all_stats.to_hash)
+    end
+  end
+end
+

data/test/stat_type/test_numerical_stats.rb

@@ -0,0 +1,118 @@
+require_relative '../test_helper'
+
+
+module TestStatType
+  class TestNumericalStats < ::Minitest::Test
+    def setup
+      @stats = FlatKit::StatType::NumericalStats.new
+      @full_stats = FlatKit::StatType::NumericalStats.new
+      @all_stats = FlatKit::StatType::NumericalStats.new(collecting_frequencies: true)
+      [ 1, 2, 3].each { |i| @full_stats.update( i ) }
+    end
+
+    def test_intialized_with_usable_values
+      assert_equal(0, @stats.count)
+      assert_equal(Float::INFINITY, @stats.min)
+      assert_equal(-Float::INFINITY, @stats.max)
+      assert_equal(0.0, @stats.sum)
+      assert_equal(0.0, @stats.rate)
+    end
+
+    def test_calculates_mean
+      assert_equal(2.0, @full_stats.mean)
+    end
+
+    def test_calculates_rate
+      assert_equal(0.5, @full_stats.rate)
+    end
+
+    def test_tracks_the_maximum_value
+      assert_equal(3.0, @full_stats.max)
+    end
+
+    def test_tracks_the_minimum_value
+      assert_equal(1.0, @full_stats.min)
+    end
+
+    def test_tracks_the_count
+      assert_equal(3,@full_stats.count)
+    end
+
+    def test_tracks_the_sum
+      assert_equal(6.0, @full_stats.sum)
+    end
+
+    def test_calculates_the_standard_deviation
+      assert_equal(1.0, @full_stats.stddev)
+    end 
+
+    def test_calculates_the_sum_of_squares
+      assert_equal(14, @full_stats.sumsq)
+    end 
+
+    def test_converts_to_a_hash
+      h = @full_stats.to_hash
+      assert_equal(::FlatKit::StatType::NumericalStats.default_stats.size, h.size)
+      assert_equal(::FlatKit::StatType::NumericalStats.default_stats, h.keys.sort)
+    end
+
+    def test_converts_to_a_limited_hash_if_given_arguments
+      h = @full_stats.to_hash( "min", "max", "mean" )
+      assert_equal(3, h.size)
+      assert_equal(%w[ max mean min], h.keys.sort)
+
+      h = @full_stats.to_hash( %w[ count rate ] )
+      assert_equal(2, h.size)
+      assert_equal(%w[ count rate ], h.keys.sort)
+    end
+
+    def test_raises_nomethoderror_if_an_invalid_stat_is_used
+      assert_raises(NoMethodError) { @full_stats.to_hash( "wibble" ) }
+    end
+
+    def test_converts_to_a_json_string
+      j = @full_stats.to_json
+      h = JSON.parse( j )
+      assert_equal(::FlatKit::StatType::NumericalStats.default_stats.size, h.size)
+      assert_equal(::FlatKit::StatType::NumericalStats.default_stats, h.keys.sort)
+    end
+
+    def test_converts_to_a_limited_json_hash_if_given_arguments
+      j = @full_stats.to_json( "min", "max", "mean" )
+      h = JSON.parse( j )
+      assert_equal(3, h.size)
+      assert_equal(%w[ max mean min], h.keys.sort)
+
+      j = @full_stats.to_json( %w[ count rate ] )
+      h = JSON.parse( j )
+      assert_equal(2, h.size)
+      assert_equal(%w[ count rate ], h.keys.sort)
+    end
+
+    def test_raises_nomethoderror_if_an_invalid_json_stat_is_used
+      assert_raises(NoMethodError) { @full_stats.to_json( "wibble" ) }
+    end
+
+    def test_collects_mode
+      values = Array.new.tap do |a|
+        100.times {
+          n = Random.rand(10)
+          a << n
+          @all_stats.update(n)
+        }
+      end
+
+      tally = values.tally
+      mode_value = tally.max_by { |v, count| count }.first
+
+      assert_equal(mode_value, @all_stats.mode)
+    end
+
+    def test_collecting_frequences_reports_extra_stat_names
+      stat_names = @all_stats.collected_stats
+      assert_includes(stat_names, "mode")
+      assert_includes(stat_names, "unique_count")
+      assert_includes(stat_names, "unique_values")
+    end
+  end
+end