flat_kit 0.3.0 → 1.1.0

data/lib/flat_kit/stat_type/numerical_stats.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #--
 # Copyright (c) 2008, 2009 Jeremy Hinegardner
 # All rights reserved.  See LICENSE and/or COPYING for details.
@@ -5,16 +7,14 @@
 # Pulled from Hitimes, which I also wrote
 #++
 
-require 'thread'
-require 'oj'
+require "oj"
 
 module FlatKit
   class StatType
-    #
-    # Stats object will keep track of the _min_, _max_, _count_, _sum_ and _sumsq_ 
+    # Internal: Stats object to keep track of the _min_, _max_, _count_, _sum_ and _sumsq_
     # and when you want you may also retrieve the _mean_, _stddev_ and _rate_.
     #
-    # this contrived example shows getting a list of all the files in a directory
+    # This contrived example shows getting a list of all the files in a directory
     # and running stats on file sizes.
     #
     #     s = FlatKit::Stats.new
@@ -33,17 +33,14 @@ module FlatKit
     class NumericalStats < NominalStats
       # A list of the available stats
 
-      attr_reader :min
-      attr_reader :max
-      attr_reader :sum
-      attr_reader :sumsq
+      attr_reader :min, :max, :sum, :sumsq
 
       def self.default_stats
-        @default_stats ||= %w[ count max mean min rate stddev sum sumsq ]
+        @default_stats ||= %w[count max mean min rate stddev sum sumsq]
       end
 
       def self.all_stats
-        @all_stats ||= %w[ count max mean min mode rate stddev sum sumsq unique_count unique_values ]
+        @all_stats ||= %w[count max mean min mode rate stddev sum sumsq unique_count unique_values]
       end
 
       def initialize(collecting_frequencies: false)
@@ -61,8 +58,8 @@ module FlatKit
       # Return the input value.
       def update(value)
         @mutex.synchronize do
-          @min = (value < @min) ? value : @min
-          @max = (value > @max) ? value : @max
+          @min = [value, @min].min
+          @max = [value, @max].max
 
           @count += 1
           @sum   += value
@@ -72,17 +69,18 @@ module FlatKit
           @frequencies[value] += 1 if @collecting_frequencies
         end
 
-        return value
+        value
       end
 
       # call-seq:
       #    stat.mean -> Float
-      # 
+      #
       # Return the arithmetic mean of the values put into the Stats object.  If no
       # values have passed through the stats object then 0.0 is returned;
       def mean
         return 0.0 if @count.zero?
-        return @sum / @count
+
+        @sum / @count
       end
 
       # call-seq:
@@ -100,7 +98,8 @@ module FlatKit
       #
       def rate
         return 0.0 if @sum.zero?
-        return @count / @sum
+
+        @count / @sum
       end
 
       #
@@ -113,7 +112,8 @@ module FlatKit
       #
       def stddev
         return 0.0 unless @count > 1
-        Math.sqrt((@sumsq - ((@sum * @sum)/@count)) / (@count - 1))
+
+        Math.sqrt((@sumsq - ((@sum * @sum) / @count)) / (@count - 1))
       end
     end
   end

data/lib/flat_kit/stat_type/ordinal_stats.rb

@@ -1,18 +1,18 @@
+# frozen_string_literal: true
+
 module FlatKit
   class StatType
-    # Same as NominalStats and also collects min and max
+    # Internal: Same as NominalStats and also collects min and max
     #
     class OrdinalStats < NominalStats
-
-      attr_reader :min
-      attr_reader :max
+      attr_reader :min, :max
 
       def self.default_stats
-        @default_stats ||= %w[ count max min ]
+        @default_stats ||= %w[count max min]
       end
 
       def self.all_stats
-        @all_stats ||= %w[ count max min unique_count unique_values mode ]
+        @all_stats ||= %w[count max min unique_count unique_values mode]
       end
 
       def initialize(collecting_frequencies: false)
@@ -23,14 +23,9 @@ module FlatKit
 
       def update(value)
         @mutex.synchronize do
+          @min = value if @min.nil? || (value < @min)
 
-          if @min.nil? || (value < @min) then
-            @min = value
-          end
-
-          if @max.nil? || (value > @max) then
-            @max = value
-          end
+          @max = value if @max.nil? || (value > @max)
 
           @count += 1
 

data/lib/flat_kit/stat_type.rb

@@ -1,7 +1,11 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Base class of all the statistic types
+  #
   class StatType
     def self.nominal_types
-      [FieldType::BooleanType, FieldType::StringType, FieldType::NullType ]
+      [FieldType::BooleanType, FieldType::StringType, FieldType::NullType]
     end
 
     def self.ordinal_types
@@ -16,6 +20,7 @@ module FlatKit
       return OrdinalStats   if ordinal_types.include?(type)
       return NominalStats   if nominal_types.include?(type)
       return NumericalStats if numerical_types.include?(type)
+
       raise ArgumentError, "Unknown stat type for #{type}"
     end
 
@@ -23,26 +28,26 @@ module FlatKit
       raise NotImplementedError, "#{self.class.name} must implement #collected_stats"
     end
 
-    # 
+    #
     # call-seq:
     #   stat.to_hash   -> Hash
     #   stat.to_hash( %w[ count max mean ]) -> Hash
     #
     # return a hash of the stats.  By default this returns a hash of all stats
     # but passing in an array of items will limit the stats returned to only
-    # those in the Array. 
+    # those in the Array.
     #
     # If passed in an empty array or nil to to_hash then STATS is assumed to be
     # the list of stats to return in the hash.
     #
-    def to_hash( *args )
+    def to_hash(*args)
       h = {}
-      args = [ args ].flatten
-      args = self.collected_stats if args.empty?
+      args = [args].flatten
+      args = collected_stats if args.empty?
       args.each do |meth|
-        h[meth] = self.send( meth )
+        h[meth] = send(meth)
       end
-      return h
+      h
     end
 
     #
@@ -54,12 +59,12 @@ module FlatKit
     # of all the stats.  If an array of items is passed in, those that match the
     # known stats will be all that is included in the json output.
     #
-    def to_json( *args )
-      h = to_hash( *args )
+    def to_json(*args)
+      h = to_hash(*args)
       Oj.dump(h)
     end
   end
 end
-require 'flat_kit/stat_type/nominal_stats'
-require 'flat_kit/stat_type/ordinal_stats'
-require 'flat_kit/stat_type/numerical_stats'
+require "flat_kit/stat_type/nominal_stats"
+require "flat_kit/stat_type/ordinal_stats"
+require "flat_kit/stat_type/numerical_stats"

data/lib/flat_kit/stats.rb

@@ -1,22 +1,20 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Collects stats from an Input and sends thos stats to an Output
+  #
   class Stats
     include ::FlatKit::EventEmitter
 
     AllFields = Class.new.freeze
 
-    attr_reader :reader
-    attr_reader :writer
-    attr_reader :fields_to_stat
-    attr_reader :stats_to_collect
-    attr_reader :stats_by_field
+    attr_reader :reader, :writer, :fields_to_stat, :stats_to_collect, :stats_by_field
 
-    def initialize(input:, input_fallback: "auto",
-                   output:, output_fallback: "auto",
+    def initialize(input:, output:, input_fallback: "auto", output_fallback: "auto",
                    fields_to_stat: AllFields, stats_to_collect: FieldStats::CORE_STATS)
-
       @fields_to_stat   = fields_to_stat
       @stats_to_collect = stats_to_collect
-      @stats_by_field   = Hash.new
+      @stats_by_field   = {}
       @record_count     = 0
 
       @reader = ::FlatKit::Reader.create_reader_from_path(path: input, fallback: input_fallback)
@@ -32,7 +30,8 @@ module FlatKit
 
     def collecting_stats_on_field?(name)
       return true if @fields_to_stat == AllFields
-      return @fields_to_stat.include?(name)
+
+      @fields_to_stat.include?(name)
     end
 
     private
@@ -41,9 +40,7 @@ module FlatKit
       ::FlatKit.logger.debug "Calculating statistics on #{reader.source}"
       reader.each do |record|
         record.to_hash.each do |field_name, field_value|
-          if collecting_stats_on_field?(field_name) then
-            update_stats_for_field(name: field_name, value: field_value)
-          end
+          update_stats_for_field(name: field_name, value: field_value) if collecting_stats_on_field?(field_name)
         end
         @record_count += 1
       end
@@ -55,8 +52,8 @@ module FlatKit
     end
 
     def write_stat_records
-      @stats_by_field.each do |name, stats|
-        h = stats.to_hash.merge({"total_record_count" => @record_count })
+      @stats_by_field.each_value do |stats|
+        h = stats.to_hash.merge({ "total_record_count" => @record_count })
         record = ::FlatKit::Jsonl::Record.new(data: nil, complete_structured_data: h)
 
         @writer.write(record)

data/lib/flat_kit/writer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Public: The base class for all format writers.
   #
@@ -11,10 +13,7 @@ module FlatKit
   # See the Xsv::Writer and Jsonl::Writer for examples.
   #
   class Writer
-    attr_reader :destination
-    attr_reader :output
-    attr_reader :count
-    attr_reader :last_position
+    attr_reader :destination, :output, :count, :last_position
 
     def self.create_writer_from_path(path:, fallback:, reader_format:)
       fallback = reader_format if fallback == "auto"
@@ -34,9 +33,9 @@ module FlatKit
     end
 
     def current_position
-      ::FlatKit::Position.new(index: @count, # since this hasn't been written yet its the right index
+      ::FlatKit::Position.new(index: @count,       # since this hasn't been written yet its the right index
                               offset: output.tell,
-                              bytesize: 0)       # nothing has been written yet
+                              bytesize: 0)         # nothing has been written yet
     end
 
     # The write method MUST return a Position object detailing the location the

data/lib/flat_kit/xsv/format.rb

@@ -1,5 +1,9 @@
+# frozen_string_literal: true
+
 module FlatKit
   module Xsv
+    # Internal: xsv format class holding the metadata about the xsv format utilities
+    #
     class Format < ::FlatKit::Format
       def self.format_name
         "xsv"
@@ -7,10 +11,10 @@ module FlatKit
 
       def self.handles?(filename)
         parts = filename.split(".")
-        %w[ csv tsv txt ].each do |ext|
+        %w[csv tsv txt].each do |ext|
           return true if parts.include?(ext)
         end
-        return false
+        false
       end
 
       def self.reader

data/lib/flat_kit/xsv/reader.rb

@@ -1,11 +1,13 @@
-require 'csv'
+# frozen_string_literal: true
+
+require "csv"
 
 module FlatKit
   module Xsv
+    # Internal: Reader class that parses and yields records from xsv files
+    #
     class Reader < ::FlatKit::Reader
-      attr_reader :input
-      attr_reader :count
-      attr_reader :fields
+      attr_reader :input, :count, :fields
 
       def self.format_name
         ::FlatKit::Xsv::Format.format_name
@@ -15,7 +17,7 @@ module FlatKit
         {
           headers: :first_row,
           converters: :numeric,
-          return_headers: false
+          return_headers: false,
         }
       end
 
@@ -36,7 +38,7 @@ module FlatKit
           yield record
         end
         input.close
-      rescue => e
+      rescue StandardError => e
         ::FlatKit.logger.error "Error reading xsv records from #{input.name}: #{e}"
         raise ::FlatKit::Error, e
       end

data/lib/flat_kit/xsv/record.rb

@@ -1,8 +1,12 @@
-require 'csv'
-require 'flat_kit/record'
+# frozen_string_literal: true
+
+require "csv"
+require "flat_kit/record"
 
 module FlatKit
   module Xsv
+    # Internal: Class that exposes the data from an XSV format record to the flatkit API
+    #
     class Record < ::FlatKit::Record
       attr_reader :ordered_fields
 
@@ -10,8 +14,8 @@ module FlatKit
         ::FlatKit::Xsv::Format.format_name
       end
 
-      def self.from_record(record, ordered_fields: nil)
-        if record.instance_of?(FlatKit::Xsv::Record) then
+      def self.from_record(record)
+        if record.instance_of?(FlatKit::Xsv::Record)
           new(data: record.data, compare_fields: record.compare_fields)
         else
           new(data: nil, compare_fields: record.compare_fields,
@@ -28,9 +32,9 @@ module FlatKit
         @complete_structured_data = complete_structured_data
         @ordered_fields = ordered_fields
 
-        if data.nil? && (complete_structured_data.nil? || complete_structured_data.empty?) then
+        if data.nil? && (complete_structured_data.nil? || complete_structured_data.empty?)
           raise FlatKit::Error,
-            "#{self.class} requires initialization from data: or complete_structured_data:"
+                "#{self.class} requires initialization from data: or complete_structured_data:"
         end
 
         resolve_ordered_fields
@@ -38,7 +42,8 @@ module FlatKit
 
       def [](key)
         return nil unless @compare_fields.include?(key)
-        if data.nil? && !@complete_structured_data.nil? then
+
+        if data.nil? && !@complete_structured_data.nil?
           @complete_structured_data[key]
         else
           data[key]
@@ -53,10 +58,8 @@ module FlatKit
       def to_a
         return data.fields unless data.nil?
 
-        Array.new.tap do |a|
-          @ordered_fields.each do |field|
-            a << @complete_structured_data[field]
-          end
+        @ordered_fields.map do |field|
+          @complete_structured_data[field]
         end
       end
 
@@ -71,19 +74,20 @@ module FlatKit
       # values in that order.
       def to_s
         return data.to_csv unless data.nil?
+
         CSV.generate_line(to_a)
       end
 
       private
 
       def resolve_ordered_fields
-        if (@ordered_fields == :auto) || (@ordered_fields.nil? || @ordered_fields.empty?) then
-          if @data.nil? || @data.empty? then
-            @ordered_fields = complete_structured_data.keys
-          else
-            @ordered_fields = @data.headers
-          end
-        end
+        return unless (@ordered_fields == :auto) || (@ordered_fields.nil? || @ordered_fields.empty?)
+
+        @ordered_fields = if @data.nil? || @data.empty?
+                            complete_structured_data.keys
+                          else
+                            @data.headers
+                          end
       end
     end
   end

data/lib/flat_kit/xsv/writer.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 module FlatKit
   module Xsv
+    # Internal: Write that takes flatkit records and writes them to XSV Output
+    #
     class Writer < ::FlatKit::Writer
-      attr_reader :fields
-      attr_reader :header_bytes
+      attr_reader :fields, :header_bytes
 
       def self.format_name
         ::FlatKit::Xsv::Format.format_name
@@ -11,7 +14,7 @@ module FlatKit
       def self.default_csv_options
         {
           headers: nil,
-          write_headers: true
+          write_headers: true,
         }
       end
 
@@ -21,10 +24,10 @@ module FlatKit
         @we_write_the_header = nil
         @csv_options = Writer.default_csv_options.dup
 
-        if @fields == :auto then
+        if @fields == :auto
           @we_write_the_header = true
         else
-          @csv_options.merge!(headers: fields)
+          @csv_options[:headers] = fields
           @we_write_the_header = false
         end
 
@@ -48,17 +51,17 @@ module FlatKit
         else
           raise FlatKit::Error, "Unable to write records of type #{record.class}"
         end
-      rescue FlatKit::Error => fe
-        raise fe
-      rescue => e
-        ::FlatKit.logger.error "Error reading jsonl records from #{output.name}: #{e}"
+      rescue FlatKit::Error => e
+        raise e
+      rescue StandardError => e
+        ::FlatKit.logger.error "Error writing xsv records to #{output.name}: #{e}"
         raise ::FlatKit::Error, e
       end
 
       private
 
       def write_record(record)
-        if @we_write_the_header && @count == 0 then
+        if @we_write_the_header && @count.zero?
           @csv << record.ordered_fields
           @header_bytes = output.tell
         end

data/lib/flat_kit/xsv.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Namespace for the xsv file fromat handling
   module Xsv
   end
 end
-require 'flat_kit/xsv/record'
-require 'flat_kit/xsv/reader'
-require 'flat_kit/xsv/writer'
-require 'flat_kit/xsv/format'
+require "flat_kit/xsv/record"
+require "flat_kit/xsv/reader"
+require "flat_kit/xsv/writer"
+require "flat_kit/xsv/format"

data/lib/flat_kit.rb

@@ -1,31 +1,36 @@
+# frozen_string_literal: true
+
+# Public: Top level namespace for this gem
+#
 module FlatKit
-  VERSION = "0.3.0"
+  VERSION = "1.1.0"
 end
-require 'flat_kit/error'
-require 'flat_kit/descendant_tracker'
-require 'flat_kit/logger'
-require 'flat_kit/event_emitter'
+require "flat_kit/error"
+require "flat_kit/descendant_tracker"
+require "flat_kit/log_formatter"
+require "flat_kit/logger"
+require "flat_kit/event_emitter"
 
-require 'flat_kit/field_type'
-require 'flat_kit/format'
-require 'flat_kit/position'
-require 'flat_kit/record'
-require 'flat_kit/reader'
-require 'flat_kit/writer'
-require 'flat_kit/input'
-require 'flat_kit/output'
-require 'flat_kit/cli'
-require 'flat_kit/xsv'
-require 'flat_kit/jsonl'
-require 'flat_kit/merge'
-require 'flat_kit/sort'
-require 'flat_kit/stats'
+require "flat_kit/field_type"
+require "flat_kit/format"
+require "flat_kit/position"
+require "flat_kit/record"
+require "flat_kit/reader"
+require "flat_kit/writer"
+require "flat_kit/input"
+require "flat_kit/output"
+require "flat_kit/cli"
+require "flat_kit/xsv"
+require "flat_kit/jsonl"
+require "flat_kit/merge"
+require "flat_kit/sort"
+require "flat_kit/stats"
 
-require 'flat_kit/stat_type'
-require 'flat_kit/field_stats'
+require "flat_kit/stat_type"
+require "flat_kit/field_stats"
 
-require 'flat_kit/merge_tree'
-require 'flat_kit/internal_node'
-require 'flat_kit/sentinel_internal_node'
-require 'flat_kit/sentinel_leaf_node'
-require 'flat_kit/leaf_node'
+require "flat_kit/merge_tree"
+require "flat_kit/internal_node"
+require "flat_kit/sentinel_internal_node"
+require "flat_kit/sentinel_leaf_node"
+require "flat_kit/leaf_node"