flat_kit 0.2.0 → 0.3.0

data/lib/flat_kit/stats.rb

@@ -0,0 +1,66 @@
+module FlatKit
+  class Stats
+    include ::FlatKit::EventEmitter
+
+    AllFields = Class.new.freeze
+
+    attr_reader :reader
+    attr_reader :writer
+    attr_reader :fields_to_stat
+    attr_reader :stats_to_collect
+    attr_reader :stats_by_field
+
+    def initialize(input:, input_fallback: "auto",
+                   output:, output_fallback: "auto",
+                   fields_to_stat: AllFields, stats_to_collect: FieldStats::CORE_STATS)
+
+      @fields_to_stat   = fields_to_stat
+      @stats_to_collect = stats_to_collect
+      @stats_by_field   = Hash.new
+      @record_count     = 0
+
+      @reader = ::FlatKit::Reader.create_reader_from_path(path: input, fallback: input_fallback)
+      @writer = ::FlatKit::Writer.create_writer_from_path(path: output, fallback: output_fallback,
+                                                          reader_format: @reader.format_name)
+    end
+
+    def call
+      calculate_stats
+      write_stat_records
+      @writer.close
+    end
+
+    def collecting_stats_on_field?(name)
+      return true if @fields_to_stat == AllFields
+      return @fields_to_stat.include?(name)
+    end
+
+    private
+
+    def calculate_stats
+      ::FlatKit.logger.debug "Calculating statistics on #{reader.source}"
+      reader.each do |record|
+        record.to_hash.each do |field_name, field_value|
+          if collecting_stats_on_field?(field_name) then
+            update_stats_for_field(name: field_name, value: field_value)
+          end
+        end
+        @record_count += 1
+      end
+    end
+
+    def update_stats_for_field(name:, value:)
+      field_stats = @stats_by_field[name] ||= FieldStats.new(name: name, stats_to_collect: @stats_to_collect)
+      field_stats.update(value)
+    end
+
+    def write_stat_records
+      @stats_by_field.each do |name, stats|
+        h = stats.to_hash.merge({"total_record_count" => @record_count })
+        record = ::FlatKit::Jsonl::Record.new(data: nil, complete_structured_data: h)
+
+        @writer.write(record)
+      end
+    end
+  end
+end

data/lib/flat_kit/writer.rb

@@ -12,6 +12,9 @@ module FlatKit
   #
   class Writer
     attr_reader :destination
+    attr_reader :output
+    attr_reader :count
+    attr_reader :last_position
 
     def self.create_writer_from_path(path:, fallback:, reader_format:)
       fallback = reader_format if fallback == "auto"
@@ -21,18 +24,30 @@ module FlatKit
 
     def initialize(destination:)
       @destination = destination
+      @output = ::FlatKit::Output.from(@destination)
+      @count = 0
+      @last_position = nil
     end
 
     def format_name
       self.class.format_name
     end
 
+    def current_position
+      ::FlatKit::Position.new(index: @count, # since this hasn't been written yet its the right index
+                              offset: output.tell,
+                              bytesize: 0)       # nothing has been written yet
+    end
+
+    # The write method MUST return a Position object detailing the location the
+    # record was written in the output stream.
+    #
     def write(record)
-      raise NotImplementedError, "#{self.class} needs to implement #write"
+      raise NotImplementedError, "#{self.class} needs to implement #write that returns Position"
     end
 
     def close
-      raise NotImplementedError, "#{self.class} needs to implement #close"
+      output.close
     end
   end
 end

data/lib/flat_kit/xsv/writer.rb

@@ -1,9 +1,8 @@
 module FlatKit
   module Xsv
     class Writer < ::FlatKit::Writer
-      attr_reader :output
-      attr_reader :count
       attr_reader :fields
+      attr_reader :header_bytes
 
       def self.format_name
         ::FlatKit::Xsv::Format.format_name
@@ -19,8 +18,6 @@ module FlatKit
       def initialize(destination:, fields: :auto, **csv_options)
         super(destination: destination)
         @fields = fields
-        @output = ::FlatKit::Output.from(@destination)
-        @count = 0
         @we_write_the_header = nil
         @csv_options = Writer.default_csv_options.dup
 
@@ -31,10 +28,16 @@ module FlatKit
           @we_write_the_header = false
         end
 
+        @header_bytes = 0
         @csv_options.merge!(csv_options)
         @csv = CSV.new(output.io, **@csv_options)
       end
 
+      # write the record and return the Position the record was written
+      #
+      # In the case of the header being written automatcially, the Postion returned is the
+      # position of the reocrd, not the header
+      #
       def write(record)
         case record
         when FlatKit::Xsv::Record
@@ -52,18 +55,30 @@ module FlatKit
         raise ::FlatKit::Error, e
       end
 
-      def close
-        @output.close
-      end
-
       private
 
       def write_record(record)
         if @we_write_the_header && @count == 0 then
           @csv << record.ordered_fields
+          @header_bytes = output.tell
         end
-        @count += 1
+
+        # the index of the record being written is the same as the count of records written so far
+        record_index = @count
+
+        # get the current output stream position to calculate bytes written
+        start_offset = output.tell
+
         @csv << record.to_a
+
+        ending_offset = output.io.tell
+        bytes_written = (ending_offset - start_offset)
+
+        @count += 1
+
+        @last_position = ::FlatKit::Position.new(index: record_index,
+                                                 offset: start_offset,
+                                                 bytesize: bytes_written)
       end
     end
   end

data/test/field_type/test_boolean_type.rb

@@ -0,0 +1,65 @@
+require_relative '../test_helper'
+
+module TestFieldType
+  class TestBooleanType < ::Minitest::Test
+    def truthy_items
+      t = %w[ yes Y true t 1 y ]
+      t << 1
+    end
+
+    def falsey_items
+      f = %w[ no n false f 0 N ]
+      f << 0
+    end
+
+    def test_true
+      assert(FlatKit::FieldType::BooleanType.matches?(true))
+    end
+
+    def test_false
+      assert(FlatKit::FieldType::BooleanType.matches?(false))
+    end
+
+    def test_truthy_items
+      truthy_items.each do |s|
+        assert(FlatKit::FieldType::BooleanType.matches?(s), "#{s} should be boolean")
+      end
+    end
+
+    def test_falsey_items
+      falsey_items.each do |s|
+        assert(FlatKit::FieldType::BooleanType.matches?(s), "#{s} should be boolean")
+      end
+    end
+
+    def test_other_class_does_not_match
+      refute(FlatKit::FieldType::BooleanType.matches?(Object.new))
+    end
+
+    def test_coerces_falsey_to_boolean
+      falsey_items.each do |t|
+        refute(FlatKit::FieldType::BooleanType.coerce(t))
+      end
+    end
+
+    def test_true_is_truthy
+      assert(FlatKit::FieldType::BooleanType.coerce(true))
+    end
+
+    def test_false_is_falsey
+      refute(FlatKit::FieldType::BooleanType.coerce(false))
+    end
+
+    def test_0_is_false
+      refute(FlatKit::FieldType::BooleanType.coerce(0))
+    end
+
+    def test_1_is_false
+      assert(FlatKit::FieldType::BooleanType.coerce(1))
+    end
+
+    def test_42_is_false
+      assert(FlatKit::FieldType::BooleanType.coerce(42.0))
+    end
+  end
+end

data/test/field_type/test_date_type.rb

@@ -0,0 +1,71 @@
+require_relative '../test_helper'
+
+module TestFieldType
+  class TestDateType < ::Minitest::Test
+    def test_time_does_not_match
+      refute(FlatKit::FieldType::DateType.matches?(Time.now))
+    end
+
+    def test_date
+      assert(FlatKit::FieldType::DateType.matches?(Date.today))
+    end
+
+    def test_datetime_does_not_match
+      refute(FlatKit::FieldType::DateType.matches?(DateTime.now))
+    end
+
+    def test_formats
+      formats = ::FlatKit::FieldType::DateType.parse_formats
+
+      assert_equal(formats.size, formats.sort.uniq.size)
+
+      formats.each do |df|
+        s = Time.now.strftime("#{df}")
+        assert(FlatKit::FieldType::DateType.matches?(s), "#{s} should match date")
+      end
+    end
+
+    def test_other_class_does_not_match
+      [ 42, Object.new, true, false ].each do |x|
+        refute(FlatKit::FieldType::DateType.matches?(x), "#{x} should not be date")
+      end
+    end
+
+    def test_N_number_does_not_match
+      x = "N89362"
+      refute(FlatKit::FieldType::DateType.matches?(x), "#{x} should not be date")
+    end
+
+    def test_coerce
+      formats = ::FlatKit::FieldType::DateType.parse_formats
+
+      formats.each do |df|
+        s = Time.now.strftime("#{df}")
+        assert_instance_of(Date, FlatKit::FieldType::DateType.coerce(s), "#{s} should convert to date")
+      end
+    end
+
+    def test_date_coerce_does_not_passthrough_time
+      t = Time.now
+      assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::DateType.coerce(t))
+    end
+
+    def test_date_coerce_passthrough_date
+      t = Date.today
+      assert_equal(t, FlatKit::FieldType::DateType.coerce(t))
+    end
+
+    def test_date_coerce_does_not_passthrough_datetime
+      t = DateTime.now
+      assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::DateType.coerce(t))
+    end
+
+    def test_number_coerce_failure
+      assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::DateType.coerce(42))
+    end
+
+    def test_number_coerce_failure_bad_parse
+      assert_equal(::FlatKit::FieldType::CoerceFailure, FlatKit::FieldType::DateType.coerce("1234 56 78 90"))
+    end
+  end
+end

data/test/field_type/test_float_type.rb

@@ -0,0 +1,56 @@
+require_relative '../test_helper'
+
+module TestFieldType
+  class TestFloatType < ::Minitest::Test
+    def test_float_matches
+      assert(FlatKit::FieldType::FloatType.matches?(42.0))
+      assert(FlatKit::FieldType::FloatType.matches?(42.1))
+    end
+
+    def test_integer_does_not_match
+      refute(FlatKit::FieldType::FloatType.matches?(42))
+    end
+
+    def test_string_of_digits_does_not_match
+      refute(FlatKit::FieldType::FloatType.matches?("42"))
+    end
+
+    def test_string_of_digits_with_dot_matches
+      assert(FlatKit::FieldType::FloatType.matches?("42.0"))
+    end
+
+    def test_string_of_leters_does_not_match
+      refute(FlatKit::FieldType::FloatType.matches?("abc"))
+    end
+
+    def test_scientific_notation_matches
+      assert(FlatKit::FieldType::FloatType.matches?("1e-10"))
+    end
+
+    def test_other_class_does_not_match
+      refute(FlatKit::FieldType::FloatType.matches?(Object.new))
+    end
+
+    def test_integer_coerces
+      assert_equal(42.0, ::FlatKit::FieldType::FloatType.coerce(42))
+    end
+
+    def test_integer_strings_coerce
+      assert_equal(42.0, ::FlatKit::FieldType::FloatType.coerce("42"))
+    end
+
+    def test_float_strings_coerce
+      assert_equal(42.6, ::FlatKit::FieldType::FloatType.coerce("42.6"))
+    end
+
+    def test_float_coerces
+      assert_equal(42.6, ::FlatKit::FieldType::FloatType.coerce(42.6))
+    end
+
+    def test_non_numercic_do_not_coerce
+      [ "eleven", nil, false, Object.new ].each do |nope|
+        assert_equal(::FlatKit::FieldType::CoerceFailure, ::FlatKit::FieldType::FloatType.coerce(nope))
+      end
+    end
+  end
+end

data/test/field_type/test_guess_type.rb

@@ -0,0 +1,14 @@
+require_relative '../test_helper'
+
+module TestFieldType
+  class TestGuessType < ::Minitest::Test
+
+    def test_guess_type_should_not_match_anything
+      refute(FlatKit::FieldType::GuessType.matches?(nil))
+    end
+
+    def test_guess_type_returns_coerce_failure
+      assert_equal(::FlatKit::FieldType::CoerceFailure, ::FlatKit::FieldType::GuessType.coerce(nil))
+    end
+  end
+end

data/test/field_type/test_integer_type.rb

@@ -0,0 +1,52 @@
+require_relative '../test_helper'
+
+module TestFieldType
+  class TestIntegerType < ::Minitest::Test
+    def test_matches_integer
+      assert(FlatKit::FieldType::IntegerType.matches?(42))
+    end
+
+    def test_matches_negative_integer
+      assert(FlatKit::FieldType::IntegerType.matches?("-42"))
+    end
+
+    def test_float_does_not_match
+      refute(FlatKit::FieldType::IntegerType.matches?(42.0))
+    end
+
+    def test_string_of_digits_matches
+      assert(FlatKit::FieldType::IntegerType.matches?("42"))
+    end
+
+    def test_string_with_some_digiets_does_not_match
+      refute(FlatKit::FieldType::IntegerType.matches?("42.0"))
+      refute(FlatKit::FieldType::IntegerType.matches?("abc"))
+    end
+
+    def test_other_class_does_not_match
+      refute(FlatKit::FieldType::IntegerType.matches?(Object.new))
+    end
+
+    def test_integer_coerces
+      assert_equal(42, ::FlatKit::FieldType::IntegerType.coerce(42))
+    end
+
+    def test_integer_strings_coerce
+      assert_equal(42, ::FlatKit::FieldType::IntegerType.coerce("42"))
+    end
+
+    def test_float_coerces
+      assert_equal(42, ::FlatKit::FieldType::IntegerType.coerce(42.6))
+    end
+
+    def test_float_strings_do_not_coerce
+      assert_equal(::FlatKit::FieldType::CoerceFailure, ::FlatKit::FieldType::IntegerType.coerce("42.6"))
+    end
+
+    def test_non_numercic_do_not_coerce
+      [ "eleven", nil, false, Object.new ].each do |nope|
+        assert_equal(::FlatKit::FieldType::CoerceFailure, ::FlatKit::FieldType::IntegerType.coerce(nope))
+      end
+    end
+  end
+end