flat_kit 0.2.0 → 0.3.0

data/lib/flat_kit/field_type/timestamp_type.rb

@@ -0,0 +1,47 @@
+module FlatKit
+  class FieldType
+    class TimestampType < FieldType
+
+      def self.parse_formats
+        @timestamp_formats ||= [
+          "%Y-%m-%d %H:%M:%S.%NZ",
+          "%Y-%m-%d %H:%M:%S.%N",
+          "%Y-%m-%dT%H:%M:%S.%N%z", # w3cdtf
+          "%Y-%m-%d %H:%M:%S",
+          "%Y-%m-%dT%H:%M:%S%z",
+          "%Y-%m-%dT%H:%M:%SZ", 
+          "%Y%m%dT%H%M%S",
+          "%a, %d %b %Y %H:%M:%S %z", # rfc2822, httpdate
+        ].freeze
+      end
+
+      def self.type_name
+        "timestamp"
+      end
+
+      def self.matches?(data)
+        coerced = coerce(data)
+        return coerced.kind_of?(Time)
+      end
+
+      def self.coerce(data)
+        case data
+        when Time
+          data
+        when String
+          parse_formats.each do |format|
+            begin
+              coerced_data = Time.strptime(data, format).utc
+              return coerced_data
+            rescue => _
+              # do nothing
+            end
+          end
+          CoerceFailure
+        else
+          CoerceFailure
+        end
+      end
+    end
+  end
+end

data/lib/flat_kit/field_type/unknown_type.rb

@@ -0,0 +1,26 @@
+module FlatKit
+  class FieldType
+    class UnknownType < FieldType
+
+      REGEX = %r{\A(na|n/a|unk|unknown)\Z}i
+
+      def self.type_name
+        "unknown"
+      end
+
+      def self.matches?(data)
+        return false unless data.kind_of?(String)
+        return true if data.length == 0
+        return REGEX.match?(data)
+      end
+
+      def self.coerce(data)
+        return data if REGEX.match?(data)
+        return CoerceFailure
+      rescue
+        return CoerceFailure
+      end
+
+    end
+  end
+end

data/lib/flat_kit/jsonl/record.rb

@@ -48,7 +48,7 @@ module FlatKit
       end
 
       def complete_structured_data
-        @complete_structured_data ||= Oj.load(data)
+        @complete_structured_data ||= Oj.load(data, mode: :strict)
       end
       alias to_hash complete_structured_data
 
@@ -60,7 +60,7 @@ module FlatKit
       # to parse
       def data
         if @data.nil? && complete_structured_data? then
-          @data = Oj.dump(complete_structured_data)
+          @data = Oj.dump(complete_structured_data, mode: :json)
         end
         @data
       end

data/lib/flat_kit/jsonl/writer.rb

@@ -1,8 +1,6 @@
 module FlatKit
   module Jsonl
     class Writer < ::FlatKit::Writer
-      attr_reader :output
-      attr_reader :count
 
       def self.format_name
         ::FlatKit::Jsonl::Format.format_name
@@ -10,10 +8,10 @@ module FlatKit
 
       def initialize(destination:)
         super
-        @output = ::FlatKit::Output.from(@destination)
-        @count = 0
       end
 
+      # write the record and return the Position the record was written
+      #
       def write(record)
         case record
         when FlatKit::Jsonl::Record
@@ -31,14 +29,25 @@ module FlatKit
         raise ::FlatKit::Error, e
       end
 
-      def close
-        @output.close
-      end
-
       def write_record(record)
-        # enforces ending in newlin if it doesn't already have one
+        # the index of the record being written is the same as the count of records written so far
+        record_index = @count
+
+        # get the current output stream position to calculate bytes written
+        start_offset = output.io.tell
+
+        # enforces ending in newline if it doesn't already have one
         output.io.puts record.to_s
+
+        ending_offset = output.io.tell
+        bytes_written = (ending_offset - start_offset)
+
         @count += 1
+
+        @last_position = ::FlatKit::Position.new(index: record_index,
+                                                 offset: start_offset,
+                                                 bytesize: bytes_written)
+
       end
     end
   end

data/lib/flat_kit/merge.rb

@@ -28,8 +28,10 @@ module FlatKit
 
       notify_listeners(name: :start, data: :start)
       merge_tree.each do |record|
-        writer.write(record)
-        notify_listeners(name: :record, data: record)
+
+        position = writer.write(record)
+        meta = { position: position }
+        notify_listeners(name: :record, data: record, meta: meta)
       end
       notify_listeners(name: :stop, data: :stop)
 

data/lib/flat_kit/output.rb

@@ -22,6 +22,10 @@ module FlatKit
       raise NotImplementedError, "#{self.class} must implement #io"
     end
 
+    def tell
+      io.tell
+    end
+
     def close
       raise NotImplementedError, "#{self.class} must implement #close"
     end

data/lib/flat_kit/position.rb

@@ -0,0 +1,19 @@
+module FlatKit
+  # The information about the position of a record in an IO stream
+  #
+  # Generally this is going to be returned by a write_record method to return
+  # information about the record that was just written
+  #
+  class Position
+
+    attr_reader :index    # zero based
+    attr_reader :offset   # byte offset in the IO stream
+    attr_reader :bytesize # byte length of the record
+
+    def initialize(index: nil, offset: nil, bytesize: nil)
+      @index    = index
+      @offset   = offset
+      @bytesize = bytesize
+    end
+  end
+end

data/lib/flat_kit/stat_type.rb

@@ -0,0 +1,65 @@
+module FlatKit
+  class StatType
+    def self.nominal_types
+      [FieldType::BooleanType, FieldType::StringType, FieldType::NullType ]
+    end
+
+    def self.ordinal_types
+      [FieldType::DateType, FieldType::TimestampType]
+    end
+
+    def self.numerical_types
+      [FieldType::FloatType, FieldType::IntegerType]
+    end
+
+    def self.for(type)
+      return OrdinalStats   if ordinal_types.include?(type)
+      return NominalStats   if nominal_types.include?(type)
+      return NumericalStats if numerical_types.include?(type)
+      raise ArgumentError, "Unknown stat type for #{type}"
+    end
+
+    def collected_stats
+      raise NotImplementedError, "#{self.class.name} must implement #collected_stats"
+    end
+
+    # 
+    # call-seq:
+    #   stat.to_hash   -> Hash
+    #   stat.to_hash( %w[ count max mean ]) -> Hash
+    #
+    # return a hash of the stats.  By default this returns a hash of all stats
+    # but passing in an array of items will limit the stats returned to only
+    # those in the Array. 
+    #
+    # If passed in an empty array or nil to to_hash then STATS is assumed to be
+    # the list of stats to return in the hash.
+    #
+    def to_hash( *args )
+      h = {}
+      args = [ args ].flatten
+      args = self.collected_stats if args.empty?
+      args.each do |meth|
+        h[meth] = self.send( meth )
+      end
+      return h
+    end
+
+    #
+    # call-seq:
+    #   stat.to_json  -> String
+    #   stat.to_json( *args ) -> String
+    #
+    # return a json string of the stats.  By default this returns a json string
+    # of all the stats.  If an array of items is passed in, those that match the
+    # known stats will be all that is included in the json output.
+    #
+    def to_json( *args )
+      h = to_hash( *args )
+      Oj.dump(h)
+    end
+  end
+end
+require 'flat_kit/stat_type/nominal_stats'
+require 'flat_kit/stat_type/ordinal_stats'
+require 'flat_kit/stat_type/numerical_stats'

data/lib/flat_kit/stat_type/nominal_stats.rb

@@ -0,0 +1,58 @@
+module FlatKit
+  class StatType
+
+    # Status object to keep track of the count and frequency of values
+    #
+    class NominalStats < StatType
+
+      attr_reader :count
+
+      def self.default_stats
+        @default_stats ||= %w[ count ]
+      end
+
+      def self.all_stats
+        @all_stats ||= %w[ count unique_count unique_values mode ]
+      end
+
+      def initialize(collecting_frequencies: false)
+        @mutex = Mutex.new
+        @count = 0
+        @collecting_frequencies = collecting_frequencies
+        @frequencies = Hash.new(0)
+      end
+
+      def collected_stats
+        return self.class.default_stats unless @collecting_frequencies
+        return self.class.all_stats
+      end
+
+      def mode
+        return nil unless @collecting_frequencies
+        @frequencies.max_by{ |item, item_count| item_count }.first
+      end
+
+      def unique_count
+        return nil unless @collecting_frequencies
+        @frequencies.size
+      end
+
+      def unique_values
+        return nil unless @collecting_frequencies
+        @frequencies.keys
+      end
+
+      def frequencies
+        return nil unless @collecting_frequencies
+        @frequencies
+      end
+
+      def update(value)
+        @mutex.synchronize do
+          @count += 1
+          @frequencies[value] += 1 if @collecting_frequencies
+        end
+      end
+    end
+  end
+end

data/lib/flat_kit/stat_type/numerical_stats.rb

@@ -0,0 +1,120 @@
+#--
+# Copyright (c) 2008, 2009 Jeremy Hinegardner
+# All rights reserved.  See LICENSE and/or COPYING for details.
+#
+# Pulled from Hitimes, which I also wrote
+#++
+
+require 'thread'
+require 'oj'
+
+module FlatKit
+  class StatType
+    #
+    # Stats object will keep track of the _min_, _max_, _count_, _sum_ and _sumsq_ 
+    # and when you want you may also retrieve the _mean_, _stddev_ and _rate_.
+    #
+    # this contrived example shows getting a list of all the files in a directory
+    # and running stats on file sizes.
+    #
+    #     s = FlatKit::Stats.new
+    #     dir = ARGV.shift || Dir.pwd
+    #     Dir.entries( dir ).each do |entry|
+    #       fs = File.stat( entry )
+    #       if fs.file? then
+    #         s.update( fs.size )
+    #        end
+    #     end
+    #
+    #     %w[ count min max mean sum stddev rate ].each do |m|
+    #       puts "#{m.rjust(6)} : #{s.send( m ) }"
+    #     end
+    #
+    class NumericalStats < NominalStats
+      # A list of the available stats
+
+      attr_reader :min
+      attr_reader :max
+      attr_reader :sum
+      attr_reader :sumsq
+
+      def self.default_stats
+        @default_stats ||= %w[ count max mean min rate stddev sum sumsq ]
+      end
+
+      def self.all_stats
+        @all_stats ||= %w[ count max mean min mode rate stddev sum sumsq unique_count unique_values ]
+      end
+
+      def initialize(collecting_frequencies: false)
+        super
+        @min = Float::INFINITY
+        @max = -Float::INFINITY
+        @sum = 0.0
+        @sumsq = 0.0
+      end
+
+      # call-seq:
+      #    stat.update( val ) -> val
+      #
+      # Update the running stats with the new value.
+      # Return the input value.
+      def update(value)
+        @mutex.synchronize do
+          @min = (value < @min) ? value : @min
+          @max = (value > @max) ? value : @max
+
+          @count += 1
+          @sum   += value
+          @sumsq += (value * value)
+
+          # from Nomnial update
+          @frequencies[value] += 1 if @collecting_frequencies
+        end
+
+        return value
+      end
+
+      # call-seq:
+      #    stat.mean -> Float
+      # 
+      # Return the arithmetic mean of the values put into the Stats object.  If no
+      # values have passed through the stats object then 0.0 is returned;
+      def mean
+        return 0.0 if @count.zero?
+        return @sum / @count
+      end
+
+      # call-seq:
+      #    stat.rate -> Float
+      #
+      # Return the +count+ divided by +sum+.
+      #
+      # In many cases when Stats#update( _value_ ) is called, the _value_ is a unit
+      # of time, typically seconds or microseconds.  #rate is a convenience for those
+      # times.  In this case, where _value_ is a unit if time, then count divided by
+      # sum is a useful value, i.e. +something per unit of time+.
+      #
+      # In the case where _value_ is a non-time related value, then the value
+      # returned by _rate_ is not really useful.
+      #
+      def rate
+        return 0.0 if @sum.zero?
+        return @count / @sum
+      end
+
+      #
+      # call-seq:
+      #    stat.stddev -> Float
+      #
+      # Return the standard deviation of all the values that have passed through the
+      # Stats object.  The standard deviation has no meaning unless the count is > 1,
+      # therefore if the current _stat.count_ is < 1 then 0.0 will be returned;
+      #
+      def stddev
+        return 0.0 unless @count > 1
+        Math.sqrt((@sumsq - ((@sum * @sum)/@count)) / (@count - 1))
+      end
+    end
+  end
+end

data/lib/flat_kit/stat_type/ordinal_stats.rb

@@ -0,0 +1,42 @@
+module FlatKit
+  class StatType
+    # Same as NominalStats and also collects min and max
+    #
+    class OrdinalStats < NominalStats
+
+      attr_reader :min
+      attr_reader :max
+
+      def self.default_stats
+        @default_stats ||= %w[ count max min ]
+      end
+
+      def self.all_stats
+        @all_stats ||= %w[ count max min unique_count unique_values mode ]
+      end
+
+      def initialize(collecting_frequencies: false)
+        super
+        @min = nil
+        @max = nil
+      end
+
+      def update(value)
+        @mutex.synchronize do
+
+          if @min.nil? || (value < @min) then
+            @min = value
+          end
+
+          if @max.nil? || (value > @max) then
+            @max = value
+          end
+
+          @count += 1
+
+          @frequencies[value] += 1 if @collecting_frequencies
+        end
+      end
+    end
+  end
+end