flat_kit 0.2.0 → 1.0.0

data/lib/flat_kit/sentinel_internal_node.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Private: The Sentinel Internal Node is a private class used by the MergeTree
   # class.
@@ -8,12 +10,10 @@ module FlatKit
   class SentinelInternalNode
     include Comparable
 
-    attr_reader :left
-    attr_reader :right
-    attr_reader :winner
+    attr_reader :left, :right, :winner
     attr_accessor :next_level
 
-    def initialize(left: nil, right: nil)
+    def initialize(*)
       @left = nil
       @right = nil
       @winner = nil
@@ -31,7 +31,8 @@ module FlatKit
     # A sentinal node is always greater than any other node
     def <=>(other)
       return 0 if other.sentinel?
-      return 1
+
+      1
     end
   end
 end

data/lib/flat_kit/sentinel_leaf_node.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Private: The Sentinel Leaf Node is used internally by the MergeTree
   #
@@ -31,7 +33,8 @@ module FlatKit
     # A sentinal node is always greater than any other node
     def <=>(other)
       return 0 if other.sentinel?
-      return 1
+
+      1
     end
   end
 end

data/lib/flat_kit/sort.rb

@@ -1,13 +1,12 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Sorts an Input and sends the sorted records to an Output
+  #
   class Sort
-    attr_reader :reader
-    attr_reader :writer
-    attr_reader :compare_fields
-
-    def initialize(input:, input_fallback: "auto",
-                   output:, output_fallback: "auto",
-                   compare_fields:)
+    attr_reader :reader, :writer, :compare_fields
 
+    def initialize(input:, output:, compare_fields:, input_fallback: "auto", output_fallback: "auto")
       @compare_fields = compare_fields
       @reader = ::FlatKit::Reader.create_reader_from_path(path: input, compare_fields: @compare_fields,
                                                           fallback: input_fallback)
@@ -16,8 +15,8 @@ module FlatKit
     end
 
     def call
-      ::FlatKit.logger.info "Sorting #{reader.source} into #{writer.destination} using key #{compare_fields.join(", ")}"
-      records = Array.new.tap do |a|
+      ::FlatKit.logger.info "Sorting #{reader.source} into #{writer.destination} using key #{compare_fields.join(', ')}"
+      records = [].tap do |a|
         reader.each do |r|
           a << r
         end

data/lib/flat_kit/stat_type/nominal_stats.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module FlatKit
+  class StatType
+    # Internal: Status object to keep track of the count and frequency of values.
+    #
+    class NominalStats < StatType
+      attr_reader :count
+
+      def self.default_stats
+        @default_stats ||= %w[count]
+      end
+
+      def self.all_stats
+        @all_stats ||= %w[count unique_count unique_values mode]
+      end
+
+      def initialize(collecting_frequencies: false)
+        super()
+        @mutex = Mutex.new
+        @count = 0
+        @collecting_frequencies = collecting_frequencies
+        @frequencies = Hash.new(0)
+      end
+
+      def collected_stats
+        return self.class.default_stats unless @collecting_frequencies
+
+        self.class.all_stats
+      end
+
+      def mode
+        return nil unless @collecting_frequencies
+
+        @frequencies.max_by { |_item, item_count| item_count }.first
+      end
+
+      def unique_count
+        return nil unless @collecting_frequencies
+
+        @frequencies.size
+      end
+
+      def unique_values
+        return nil unless @collecting_frequencies
+
+        @frequencies.keys
+      end
+
+      def frequencies
+        return nil unless @collecting_frequencies
+
+        @frequencies
+      end
+
+      def update(value)
+        @mutex.synchronize do
+          @count += 1
+          @frequencies[value] += 1 if @collecting_frequencies
+        end
+      end
+    end
+  end
+end

data/lib/flat_kit/stat_type/numerical_stats.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+#--
+# Copyright (c) 2008, 2009 Jeremy Hinegardner
+# All rights reserved.  See LICENSE and/or COPYING for details.
+#
+# Pulled from Hitimes, which I also wrote
+#++
+
+require "oj"
+
+module FlatKit
+  class StatType
+    # Internal: Stats object to keep track of the _min_, _max_, _count_, _sum_ and _sumsq_
+    # and when you want you may also retrieve the _mean_, _stddev_ and _rate_.
+    #
+    # This contrived example shows getting a list of all the files in a directory
+    # and running stats on file sizes.
+    #
+    #     s = FlatKit::Stats.new
+    #     dir = ARGV.shift || Dir.pwd
+    #     Dir.entries( dir ).each do |entry|
+    #       fs = File.stat( entry )
+    #       if fs.file? then
+    #         s.update( fs.size )
+    #        end
+    #     end
+    #
+    #     %w[ count min max mean sum stddev rate ].each do |m|
+    #       puts "#{m.rjust(6)} : #{s.send( m ) }"
+    #     end
+    #
+    class NumericalStats < NominalStats
+      # A list of the available stats
+
+      attr_reader :min, :max, :sum, :sumsq
+
+      def self.default_stats
+        @default_stats ||= %w[count max mean min rate stddev sum sumsq]
+      end
+
+      def self.all_stats
+        @all_stats ||= %w[count max mean min mode rate stddev sum sumsq unique_count unique_values]
+      end
+
+      def initialize(collecting_frequencies: false)
+        super
+        @min = Float::INFINITY
+        @max = -Float::INFINITY
+        @sum = 0.0
+        @sumsq = 0.0
+      end
+
+      # call-seq:
+      #    stat.update( val ) -> val
+      #
+      # Update the running stats with the new value.
+      # Return the input value.
+      def update(value)
+        @mutex.synchronize do
+          @min = [value, @min].min
+          @max = [value, @max].max
+
+          @count += 1
+          @sum   += value
+          @sumsq += (value * value)
+
+          # from Nomnial update
+          @frequencies[value] += 1 if @collecting_frequencies
+        end
+
+        value
+      end
+
+      # call-seq:
+      #    stat.mean -> Float
+      #
+      # Return the arithmetic mean of the values put into the Stats object.  If no
+      # values have passed through the stats object then 0.0 is returned;
+      def mean
+        return 0.0 if @count.zero?
+
+        @sum / @count
+      end
+
+      # call-seq:
+      #    stat.rate -> Float
+      #
+      # Return the +count+ divided by +sum+.
+      #
+      # In many cases when Stats#update( _value_ ) is called, the _value_ is a unit
+      # of time, typically seconds or microseconds.  #rate is a convenience for those
+      # times.  In this case, where _value_ is a unit if time, then count divided by
+      # sum is a useful value, i.e. +something per unit of time+.
+      #
+      # In the case where _value_ is a non-time related value, then the value
+      # returned by _rate_ is not really useful.
+      #
+      def rate
+        return 0.0 if @sum.zero?
+
+        @count / @sum
+      end
+
+      #
+      # call-seq:
+      #    stat.stddev -> Float
+      #
+      # Return the standard deviation of all the values that have passed through the
+      # Stats object.  The standard deviation has no meaning unless the count is > 1,
+      # therefore if the current _stat.count_ is < 1 then 0.0 will be returned;
+      #
+      def stddev
+        return 0.0 unless @count > 1
+
+        Math.sqrt((@sumsq - ((@sum * @sum) / @count)) / (@count - 1))
+      end
+    end
+  end
+end

data/lib/flat_kit/stat_type/ordinal_stats.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module FlatKit
+  class StatType
+    # Internal: Same as NominalStats and also collects min and max
+    #
+    class OrdinalStats < NominalStats
+      attr_reader :min, :max
+
+      def self.default_stats
+        @default_stats ||= %w[count max min]
+      end
+
+      def self.all_stats
+        @all_stats ||= %w[count max min unique_count unique_values mode]
+      end
+
+      def initialize(collecting_frequencies: false)
+        super
+        @min = nil
+        @max = nil
+      end
+
+      def update(value)
+        @mutex.synchronize do
+          @min = value if @min.nil? || (value < @min)
+
+          @max = value if @max.nil? || (value > @max)
+
+          @count += 1
+
+          @frequencies[value] += 1 if @collecting_frequencies
+        end
+      end
+    end
+  end
+end

data/lib/flat_kit/stat_type.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module FlatKit
+  # Internal: Base class of all the statistic types
+  #
+  class StatType
+    def self.nominal_types
+      [FieldType::BooleanType, FieldType::StringType, FieldType::NullType]
+    end
+
+    def self.ordinal_types
+      [FieldType::DateType, FieldType::TimestampType]
+    end
+
+    def self.numerical_types
+      [FieldType::FloatType, FieldType::IntegerType]
+    end
+
+    def self.for(type)
+      return OrdinalStats   if ordinal_types.include?(type)
+      return NominalStats   if nominal_types.include?(type)
+      return NumericalStats if numerical_types.include?(type)
+
+      raise ArgumentError, "Unknown stat type for #{type}"
+    end
+
+    def collected_stats
+      raise NotImplementedError, "#{self.class.name} must implement #collected_stats"
+    end
+
+    #
+    # call-seq:
+    #   stat.to_hash   -> Hash
+    #   stat.to_hash( %w[ count max mean ]) -> Hash
+    #
+    # return a hash of the stats.  By default this returns a hash of all stats
+    # but passing in an array of items will limit the stats returned to only
+    # those in the Array.
+    #
+    # If passed in an empty array or nil to to_hash then STATS is assumed to be
+    # the list of stats to return in the hash.
+    #
+    def to_hash(*args)
+      h = {}
+      args = [args].flatten
+      args = collected_stats if args.empty?
+      args.each do |meth|
+        h[meth] = send(meth)
+      end
+      h
+    end
+
+    #
+    # call-seq:
+    #   stat.to_json  -> String
+    #   stat.to_json( *args ) -> String
+    #
+    # return a json string of the stats.  By default this returns a json string
+    # of all the stats.  If an array of items is passed in, those that match the
+    # known stats will be all that is included in the json output.
+    #
+    def to_json(*args)
+      h = to_hash(*args)
+      Oj.dump(h)
+    end
+  end
+end
+require "flat_kit/stat_type/nominal_stats"
+require "flat_kit/stat_type/ordinal_stats"
+require "flat_kit/stat_type/numerical_stats"

data/lib/flat_kit/stats.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module FlatKit
+  # Internal: Collects stats from an Input and sends thos stats to an Output
+  #
+  class Stats
+    include ::FlatKit::EventEmitter
+
+    AllFields = Class.new.freeze
+
+    attr_reader :reader, :writer, :fields_to_stat, :stats_to_collect, :stats_by_field
+
+    def initialize(input:, output:, input_fallback: "auto", output_fallback: "auto",
+                   fields_to_stat: AllFields, stats_to_collect: FieldStats::CORE_STATS)
+
+      @fields_to_stat   = fields_to_stat
+      @stats_to_collect = stats_to_collect
+      @stats_by_field   = {}
+      @record_count     = 0
+
+      @reader = ::FlatKit::Reader.create_reader_from_path(path: input, fallback: input_fallback)
+      @writer = ::FlatKit::Writer.create_writer_from_path(path: output, fallback: output_fallback,
+                                                          reader_format: @reader.format_name)
+    end
+
+    def call
+      calculate_stats
+      write_stat_records
+      @writer.close
+    end
+
+    def collecting_stats_on_field?(name)
+      return true if @fields_to_stat == AllFields
+
+      @fields_to_stat.include?(name)
+    end
+
+    private
+
+    def calculate_stats
+      ::FlatKit.logger.debug "Calculating statistics on #{reader.source}"
+      reader.each do |record|
+        record.to_hash.each do |field_name, field_value|
+          update_stats_for_field(name: field_name, value: field_value) if collecting_stats_on_field?(field_name)
+        end
+        @record_count += 1
+      end
+    end
+
+    def update_stats_for_field(name:, value:)
+      field_stats = @stats_by_field[name] ||= FieldStats.new(name: name, stats_to_collect: @stats_to_collect)
+      field_stats.update(value)
+    end
+
+    def write_stat_records
+      @stats_by_field.each_value do |stats|
+        h = stats.to_hash.merge({ "total_record_count" => @record_count })
+        record = ::FlatKit::Jsonl::Record.new(data: nil, complete_structured_data: h)
+
+        @writer.write(record)
+      end
+    end
+  end
+end

data/lib/flat_kit/writer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Public: The base class for all format writers.
   #
@@ -11,7 +13,7 @@ module FlatKit
   # See the Xsv::Writer and Jsonl::Writer for examples.
   #
   class Writer
-    attr_reader :destination
+    attr_reader :destination, :output, :count, :last_position
 
     def self.create_writer_from_path(path:, fallback:, reader_format:)
       fallback = reader_format if fallback == "auto"
@@ -21,18 +23,30 @@ module FlatKit
 
     def initialize(destination:)
       @destination = destination
+      @output = ::FlatKit::Output.from(@destination)
+      @count = 0
+      @last_position = nil
     end
 
     def format_name
       self.class.format_name
     end
 
+    def current_position
+      ::FlatKit::Position.new(index: @count,       # since this hasn't been written yet its the right index
+                              offset: output.tell,
+                              bytesize: 0)         # nothing has been written yet
+    end
+
+    # The write method MUST return a Position object detailing the location the
+    # record was written in the output stream.
+    #
     def write(record)
-      raise NotImplementedError, "#{self.class} needs to implement #write"
+      raise NotImplementedError, "#{self.class} needs to implement #write that returns Position"
     end
 
     def close
-      raise NotImplementedError, "#{self.class} needs to implement #close"
+      output.close
     end
   end
 end

data/lib/flat_kit/xsv/format.rb

@@ -1,5 +1,9 @@
+# frozen_string_literal: true
+
 module FlatKit
   module Xsv
+    # Internal: xsv format class holding the metadata about the xsv format utilities
+    #
     class Format < ::FlatKit::Format
       def self.format_name
         "xsv"
@@ -7,10 +11,10 @@ module FlatKit
 
       def self.handles?(filename)
         parts = filename.split(".")
-        %w[ csv tsv txt ].each do |ext|
+        %w[csv tsv txt].each do |ext|
           return true if parts.include?(ext)
         end
-        return false
+        false
       end
 
       def self.reader

data/lib/flat_kit/xsv/reader.rb

@@ -1,11 +1,13 @@
-require 'csv'
+# frozen_string_literal: true
+
+require "csv"
 
 module FlatKit
   module Xsv
+    # Internal: Reader class that parses and yields records from xsv files
+    #
     class Reader < ::FlatKit::Reader
-      attr_reader :input
-      attr_reader :count
-      attr_reader :fields
+      attr_reader :input, :count, :fields
 
       def self.format_name
         ::FlatKit::Xsv::Format.format_name
@@ -15,7 +17,7 @@ module FlatKit
         {
           headers: :first_row,
           converters: :numeric,
-          return_headers: false
+          return_headers: false,
         }
       end
 
@@ -36,7 +38,7 @@ module FlatKit
           yield record
         end
         input.close
-      rescue => e
+      rescue StandardError => e
         ::FlatKit.logger.error "Error reading xsv records from #{input.name}: #{e}"
         raise ::FlatKit::Error, e
       end

data/lib/flat_kit/xsv/record.rb

@@ -1,8 +1,12 @@
-require 'csv'
-require 'flat_kit/record'
+# frozen_string_literal: true
+
+require "csv"
+require "flat_kit/record"
 
 module FlatKit
   module Xsv
+    # Internal: Class that exposes the data from an XSV format record to the flatkit API
+    #
     class Record < ::FlatKit::Record
       attr_reader :ordered_fields
 
@@ -10,8 +14,8 @@ module FlatKit
         ::FlatKit::Xsv::Format.format_name
       end
 
-      def self.from_record(record, ordered_fields: nil)
-        if record.instance_of?(FlatKit::Xsv::Record) then
+      def self.from_record(record)
+        if record.instance_of?(FlatKit::Xsv::Record)
           new(data: record.data, compare_fields: record.compare_fields)
         else
           new(data: nil, compare_fields: record.compare_fields,
@@ -28,9 +32,9 @@ module FlatKit
         @complete_structured_data = complete_structured_data
         @ordered_fields = ordered_fields
 
-        if data.nil? && (complete_structured_data.nil? || complete_structured_data.empty?) then
+        if data.nil? && (complete_structured_data.nil? || complete_structured_data.empty?)
           raise FlatKit::Error,
-            "#{self.class} requires initialization from data: or complete_structured_data:"
+                "#{self.class} requires initialization from data: or complete_structured_data:"
         end
 
         resolve_ordered_fields
@@ -38,7 +42,8 @@ module FlatKit
 
       def [](key)
         return nil unless @compare_fields.include?(key)
-        if data.nil? && !@complete_structured_data.nil? then
+
+        if data.nil? && !@complete_structured_data.nil?
           @complete_structured_data[key]
         else
           data[key]
@@ -53,7 +58,7 @@ module FlatKit
       def to_a
         return data.fields unless data.nil?
 
-        Array.new.tap do |a|
+        [].tap do |a|
           @ordered_fields.each do |field|
             a << @complete_structured_data[field]
           end
@@ -71,19 +76,20 @@ module FlatKit
       # values in that order.
       def to_s
         return data.to_csv unless data.nil?
+
         CSV.generate_line(to_a)
       end
 
       private
 
       def resolve_ordered_fields
-        if (@ordered_fields == :auto) || (@ordered_fields.nil? || @ordered_fields.empty?) then
-          if @data.nil? || @data.empty? then
-            @ordered_fields = complete_structured_data.keys
-          else
-            @ordered_fields = @data.headers
-          end
-        end
+        return unless (@ordered_fields == :auto) || (@ordered_fields.nil? || @ordered_fields.empty?)
+
+        @ordered_fields = if @data.nil? || @data.empty?
+                            complete_structured_data.keys
+                          else
+                            @data.headers
+                          end
       end
     end
   end