flat_kit 0.3.0 → 1.0.0

data/lib/flat_kit/logger.rb

@@ -1,19 +1,12 @@
-require 'logger'
+# frozen_string_literal: true
 
-module FlatKit
-  class LogFormatter < ::Logger::Formatter
-    FORMAT          = "%s %5d %05s : %s\n".freeze
-    DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ".freeze
-    def initialize
-      super
-      self.datetime_format = DATETIME_FORMAT
-    end
-
-    def call(severity, time, progname, msg)
-      FORMAT % [format_datetime(time.utc), Process.pid, severity, msg2str(msg)]
-    end
-  end
+require "logger"
 
+# Public: Top level namespace for the gem
+#
+module FlatKit
+  # Internal: Logger class
+  #
   class Logger
     def self.for_io(io)
       ::Logger.new(io, formatter: LogFormatter.new)
@@ -26,11 +19,11 @@ module FlatKit
   end
 
   def self.log_to(destination = $stderr)
-    if destination.kind_of?(::IO) then
-      @logger = ::FlatKit::Logger.for_io(destination)
-    else
-      @logger = ::FlatKit::Logger.for_path(destination)
-    end
+    @logger = if destination.is_a?(::IO)
+                ::FlatKit::Logger.for_io(destination)
+              else
+                ::FlatKit::Logger.for_path(destination)
+              end
   end
 
   def self.logger

data/lib/flat_kit/merge.rb

@@ -1,15 +1,14 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Class implementing merging from N inputs and output to 1 output.
+  #
   class Merge
-
     include ::FlatKit::EventEmitter
 
-    attr_reader :readers
-    attr_reader :writer
-    attr_reader :compare_fields
+    attr_reader :readers, :writer, :compare_fields
 
-    def initialize(inputs:, input_fallback: "auto",
-                   output:, output_fallback: "auto",
-                   compare_fields:)
+    def initialize(inputs:, output:, compare_fields:, input_fallback: "auto", output_fallback: "auto")
       @compare_fields = compare_fields
       @readers = ::FlatKit::Reader.create_readers_from_paths(paths: inputs, compare_fields: @compare_fields,
                                                              fallback: input_fallback)
@@ -19,21 +18,12 @@ module FlatKit
 
     def call
       ::FlatKit.logger.debug "Merging the following files into #{writer.destination}"
-      ::FlatKit.logger.debug "Using this key for sorting: #{compare_fields.join(", ")}"
+      ::FlatKit.logger.debug "Using this key for sorting: #{compare_fields.join(', ')}"
       readers.each do |r|
         ::FlatKit.logger.debug "  #{r.source}"
       end
 
-      merge_tree = ::FlatKit::MergeTree.new(readers)
-
-      notify_listeners(name: :start, data: :start)
-      merge_tree.each do |record|
-
-        position = writer.write(record)
-        meta = { position: position }
-        notify_listeners(name: :record, data: record, meta: meta)
-      end
-      notify_listeners(name: :stop, data: :stop)
+      run_merge(readers)
 
       readers.each do |r|
         ::FlatKit.logger.debug "  #{r.source} produced #{r.count} records"
@@ -42,5 +32,18 @@ module FlatKit
       writer.close
       ::FlatKit.logger.debug "Wrote #{writer.count} records to #{writer.destination}"
     end
+
+    private
+
+    def run_merge(readers)
+      tree = ::FlatKit::MergeTree.new(readers)
+      notify_listeners(name: :start, data: :start)
+      tree.each do |record|
+        position = writer.write(record)
+        meta = { position: position }
+        notify_listeners(name: :record, data: record, meta: meta)
+      end
+      notify_listeners(name: :stop, data: :stop)
+    end
   end
 end

data/lib/flat_kit/merge_tree.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Public: Merge a list of sorted records from Readers into a single output Writer
   #
@@ -29,9 +31,7 @@ module FlatKit
   class MergeTree
     include Enumerable
 
-    attr_reader :leaves
-    attr_reader :levels
-    attr_reader :readers
+    attr_reader :leaves, :levels, :readers
 
     def initialize(readers)
       @readers = readers
@@ -44,9 +44,7 @@ module FlatKit
 
       # Need to pad the leaves to an even number so that the slicing by 2 for
       # the tournament will work
-      if @leaves.size.odd? then
-        @leaves << SentinelLeafNode.new
-      end
+      @leaves << SentinelLeafNode.new if @leaves.size.odd?
 
       init_tree
     end
@@ -94,6 +92,7 @@ module FlatKit
     def each
       loop do
         break if root.leaf.finished?
+
         yield root.value
         # consume the yielded value and have the tournament tree replay those
         # brackets affected

data/lib/flat_kit/output/file.rb

@@ -1,22 +1,31 @@
-require 'zlib'
+# frozen_string_literal: true
+
+require "zlib"
+require "pathname"
 
 module FlatKit
   class Output
+    # Internal: File output implementation
+    #
     class File < Output
       attr_reader :path
 
+      # internal api method for testing purposes
+      attr_reader :io
+
       def self.handles?(obj)
         return true if obj.instance_of?(Pathname)
         return false unless obj.instance_of?(String)
 
         # incase these get loaded in different orders
-        return false if ::FlatKit::Output::IO.is_stdout?(obj)
-        return false if ::FlatKit::Output::IO.is_stderr?(obj)
+        return false if ::FlatKit::Output::IO.stdout?(obj)
+        return false if ::FlatKit::Output::IO.stderr?(obj)
 
-        return true
+        true
       end
 
       def initialize(obj)
+        super()
         @path = Pathname.new(obj)
         path.dirname.mkpath
         @io = open_output(path)
@@ -30,11 +39,6 @@ module FlatKit
         @io.close
       end
 
-      # internal api method for testing purposes
-      def io
-        @io
-      end
-
       private
 
       # open the opropriate otuput type depending on the destination file name

data/lib/flat_kit/output/io.rb

@@ -1,73 +1,78 @@
+# frozen_string_literal: true
+
 module FlatKit
   class Output
+    # Internal: Non-file Output impelementation - this is genrally to stdout or stderr
+    #
     class IO < Output
-      attr_reader :count
+      attr_reader :count, :name
+
+      # internal api method for testing
+      attr_reader :io
 
-      STDOUTS = %w[ stdout STDOUT - <stdout> ]
-      STDERRS = %w[ stderr STDERR <stderr> ]
+      STDOUTS = %w[stdout STDOUT - <stdout>].freeze
+      STDERRS = %w[stderr STDERR <stderr>].freeze
 
       def self.handles?(obj)
-        return true if is_stderr?(obj)
-        return true if is_stdout?(obj)
-        return true if [ ::File, ::StringIO, ::IO ].any? { |klass| obj.kind_of?(klass) }
-        return false
+        return true if stderr?(obj)
+        return true if stdout?(obj)
+        return true if [::File, ::StringIO, ::IO].any? { |klass| obj.is_a?(klass) }
+
+        false
       end
 
-      def self.is_stderr?(obj)
+      def self.stderr?(obj)
         case obj
         when String
           return true if STDERRS.include?(obj)
         when ::IO
-          return true if obj == ::STDERR
+          return true if obj == $stderr
         end
-        return false
+        false
       end
 
-      def self.is_stdout?(obj)
+      def self.stdout?(obj)
         case obj
         when String
           return true if STDOUTS.include?(obj)
         when ::IO
-          return true if obj == ::STDOUT
+          return true if obj == $stdout
         end
-        return false
+        false
       end
 
       def initialize(obj)
+        super()
         @count = 0
-        if self.class.is_stdout?(obj) then
+        @name = nil
+        @io = nil
+        init_name_and_io(obj)
+      end
+
+      # this goes to an io stream and we are not in charge of opening it
+      def close
+        @io.close
+      end
+
+      private
+
+      def init_name_and_io(obj)
+        if self.class.stdout?(obj)
           @name = "<STDOUT>"
           @io = $stdout
-        elsif self.class.is_stderr?(obj) then
+        elsif self.class.stderr?(obj)
           @name = "<STDERR>"
           @io = $stderr
-        elsif obj.kind_of?(::File) then
-          @name = obj.path
+        elsif obj.is_a?(::IO)
+          @name = (obj.respond_to?(:path) && obj.path) || obj.inspect
           @io = obj
-        elsif obj.kind_of?(::StringIO) then
-          @name = obj.inspect
-          @io = obj
-        elsif obj.kind_of?(::IO) then
+        elsif obj.is_a?(::StringIO)
           @name = obj.inspect
           @io = obj
         else
           raise ::FlatKit::Error, "Unable to create #{self.class} from #{obj.class} : #{obj.inspect}"
         end
       end
-
-      def name
-        @name
-      end
-
-      # this goes to an io stream and we are not in charge of opening it
-      def close
-        @io.close
-      end
-
-      # internal api method for testing
-      def io
-        @io
-      end
     end
   end
 end

data/lib/flat_kit/output.rb

@@ -1,14 +1,16 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Base clases for all output handlers
+  #
   class Output
     extend DescendantTracker
 
     def self.from(out)
-      return out if out.kind_of?(::FlatKit::Output)
+      return out if out.is_a?(::FlatKit::Output)
 
       out_klass = find_child(:handles?, out)
-      if out_klass then
-        return out_klass.new(out)
-      end
+      return out_klass.new(out) if out_klass
 
       raise FlatKit::Error, "Unable to create output from #{out.class} : #{out.inspect}"
     end
@@ -17,7 +19,6 @@ module FlatKit
       raise NotImplementedError, "#{self.class} must implement #name"
     end
 
-    #
     def io
       raise NotImplementedError, "#{self.class} must implement #io"
     end
@@ -32,5 +33,5 @@ module FlatKit
   end
 end
 
-require 'flat_kit/output/io'
-require 'flat_kit/output/file'
+require "flat_kit/output/io"
+require "flat_kit/output/file"

data/lib/flat_kit/position.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # The information about the position of a record in an IO stream
   #
@@ -5,10 +7,7 @@ module FlatKit
   # information about the record that was just written
   #
   class Position
-
-    attr_reader :index    # zero based
-    attr_reader :offset   # byte offset in the IO stream
-    attr_reader :bytesize # byte length of the record
+    attr_reader :index, :offset, :bytesize # zero based   # byte offset in the IO stream # byte length of the record
 
     def initialize(index: nil, offset: nil, bytesize: nil)
       @index    = index

data/lib/flat_kit/reader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Public: the base class for all format readers.
   #
@@ -14,24 +16,21 @@ module FlatKit
   # API:
   #
   #   initialize(source:, compare_fields:)
-  #   each -> Yields / returns 
+  #   each -> Yields / returns
   #
   class Reader
     include Enumerable
 
-    attr_reader :source
-    attr_reader :compare_fields
+    attr_reader :source, :compare_fields
 
     def self.create_reader_from_path(path: "-", fallback: "auto", compare_fields: :none)
       format = ::FlatKit::Format.for_with_fallback!(path: path, fallback: fallback)
-      return format.reader.new(source: path, compare_fields: compare_fields)
+      format.reader.new(source: path, compare_fields: compare_fields)
     end
 
     def self.create_readers_from_paths(paths:, fallback: "auto", compare_fields: :none)
       # default to stdin if there are no paths
-      if paths.empty? then
-        paths << "-"
-      end
+      paths << "-" if paths.empty?
 
       paths.map do |path|
         create_reader_from_path(path: path, fallback: fallback, compare_fields: compare_fields)
@@ -55,7 +54,8 @@ module FlatKit
 
     def resolve_compare_fields(value)
       return [] if value == :none
-      return value
+
+      value
     end
   end
 end

data/lib/flat_kit/record.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Public: The base class that all record classes should inherit from.
   #
@@ -35,11 +37,9 @@ module FlatKit
   #   # the initialize method must call super(data:, compare_fields:) to
   #     initializa the root data structures
   class Record
-
     include Comparable
 
-    attr_reader :data
-    attr_reader :compare_fields
+    attr_reader :data, :compare_fields
 
     def initialize(data:, compare_fields:)
       @data = data
@@ -57,15 +57,15 @@ module FlatKit
         my_val         = self[field]
         other_val      = other[field]
 
-        if my_val.nil? && other_val.nil? then
-          compare_result = 0
-        elsif my_val.nil?
-          compare_result = -1
-        elsif other_val.nil?
-          compare_result = 1
-        else
-          compare_result = my_val.<=>(other_val)
-        end
+        compare_result = if my_val.nil? && other_val.nil?
+                           0
+                         elsif my_val.nil?
+                           -1
+                         elsif other_val.nil?
+                           1
+                         else
+                           my_val <=> (other_val)
+                         end
 
         return compare_result unless compare_result.zero?
       end

data/lib/flat_kit/sentinel_internal_node.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Private: The Sentinel Internal Node is a private class used by the MergeTree
   # class.
@@ -8,12 +10,10 @@ module FlatKit
   class SentinelInternalNode
     include Comparable
 
-    attr_reader :left
-    attr_reader :right
-    attr_reader :winner
+    attr_reader :left, :right, :winner
     attr_accessor :next_level
 
-    def initialize(left: nil, right: nil)
+    def initialize(*)
       @left = nil
       @right = nil
       @winner = nil
@@ -31,7 +31,8 @@ module FlatKit
     # A sentinal node is always greater than any other node
     def <=>(other)
       return 0 if other.sentinel?
-      return 1
+
+      1
     end
   end
 end

data/lib/flat_kit/sentinel_leaf_node.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Private: The Sentinel Leaf Node is used internally by the MergeTree
   #
@@ -31,7 +33,8 @@ module FlatKit
     # A sentinal node is always greater than any other node
     def <=>(other)
       return 0 if other.sentinel?
-      return 1
+
+      1
     end
   end
 end

data/lib/flat_kit/sort.rb

@@ -1,13 +1,12 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Sorts an Input and sends the sorted records to an Output
+  #
   class Sort
-    attr_reader :reader
-    attr_reader :writer
-    attr_reader :compare_fields
-
-    def initialize(input:, input_fallback: "auto",
-                   output:, output_fallback: "auto",
-                   compare_fields:)
+    attr_reader :reader, :writer, :compare_fields
 
+    def initialize(input:, output:, compare_fields:, input_fallback: "auto", output_fallback: "auto")
       @compare_fields = compare_fields
       @reader = ::FlatKit::Reader.create_reader_from_path(path: input, compare_fields: @compare_fields,
                                                           fallback: input_fallback)
@@ -16,8 +15,8 @@ module FlatKit
     end
 
     def call
-      ::FlatKit.logger.info "Sorting #{reader.source} into #{writer.destination} using key #{compare_fields.join(", ")}"
-      records = Array.new.tap do |a|
+      ::FlatKit.logger.info "Sorting #{reader.source} into #{writer.destination} using key #{compare_fields.join(', ')}"
+      records = [].tap do |a|
         reader.each do |r|
           a << r
         end

data/lib/flat_kit/stat_type/nominal_stats.rb

@@ -1,21 +1,22 @@
+# frozen_string_literal: true
+
 module FlatKit
   class StatType
-
-    # Status object to keep track of the count and frequency of values
+    # Internal: Status object to keep track of the count and frequency of values.
     #
     class NominalStats < StatType
-
       attr_reader :count
 
       def self.default_stats
-        @default_stats ||= %w[ count ]
+        @default_stats ||= %w[count]
       end
 
       def self.all_stats
-        @all_stats ||= %w[ count unique_count unique_values mode ]
+        @all_stats ||= %w[count unique_count unique_values mode]
       end
 
       def initialize(collecting_frequencies: false)
+        super()
         @mutex = Mutex.new
         @count = 0
         @collecting_frequencies = collecting_frequencies
@@ -24,26 +25,31 @@ module FlatKit
 
       def collected_stats
         return self.class.default_stats unless @collecting_frequencies
-        return self.class.all_stats
+
+        self.class.all_stats
       end
 
       def mode
         return nil unless @collecting_frequencies
-        @frequencies.max_by{ |item, item_count| item_count }.first
+
+        @frequencies.max_by { |_item, item_count| item_count }.first
       end
 
       def unique_count
         return nil unless @collecting_frequencies
+
         @frequencies.size
       end
 
       def unique_values
         return nil unless @collecting_frequencies
+
         @frequencies.keys
       end
 
       def frequencies
         return nil unless @collecting_frequencies
+
         @frequencies
       end
 

data/lib/flat_kit/stat_type/numerical_stats.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #--
 # Copyright (c) 2008, 2009 Jeremy Hinegardner
 # All rights reserved.  See LICENSE and/or COPYING for details.
@@ -5,16 +7,14 @@
 # Pulled from Hitimes, which I also wrote
 #++
 
-require 'thread'
-require 'oj'
+require "oj"
 
 module FlatKit
   class StatType
-    #
-    # Stats object will keep track of the _min_, _max_, _count_, _sum_ and _sumsq_ 
+    # Internal: Stats object to keep track of the _min_, _max_, _count_, _sum_ and _sumsq_
     # and when you want you may also retrieve the _mean_, _stddev_ and _rate_.
     #
-    # this contrived example shows getting a list of all the files in a directory
+    # This contrived example shows getting a list of all the files in a directory
     # and running stats on file sizes.
     #
     #     s = FlatKit::Stats.new
@@ -33,17 +33,14 @@ module FlatKit
     class NumericalStats < NominalStats
       # A list of the available stats
 
-      attr_reader :min
-      attr_reader :max
-      attr_reader :sum
-      attr_reader :sumsq
+      attr_reader :min, :max, :sum, :sumsq
 
       def self.default_stats
-        @default_stats ||= %w[ count max mean min rate stddev sum sumsq ]
+        @default_stats ||= %w[count max mean min rate stddev sum sumsq]
       end
 
       def self.all_stats
-        @all_stats ||= %w[ count max mean min mode rate stddev sum sumsq unique_count unique_values ]
+        @all_stats ||= %w[count max mean min mode rate stddev sum sumsq unique_count unique_values]
       end
 
       def initialize(collecting_frequencies: false)
@@ -61,8 +58,8 @@ module FlatKit
       # Return the input value.
       def update(value)
         @mutex.synchronize do
-          @min = (value < @min) ? value : @min
-          @max = (value > @max) ? value : @max
+          @min = [value, @min].min
+          @max = [value, @max].max
 
           @count += 1
           @sum   += value
@@ -72,17 +69,18 @@ module FlatKit
           @frequencies[value] += 1 if @collecting_frequencies
         end
 
-        return value
+        value
       end
 
       # call-seq:
       #    stat.mean -> Float
-      # 
+      #
       # Return the arithmetic mean of the values put into the Stats object.  If no
       # values have passed through the stats object then 0.0 is returned;
       def mean
         return 0.0 if @count.zero?
-        return @sum / @count
+
+        @sum / @count
       end
 
       # call-seq:
@@ -100,7 +98,8 @@ module FlatKit
       #
       def rate
         return 0.0 if @sum.zero?
-        return @count / @sum
+
+        @count / @sum
       end
 
       #
@@ -113,7 +112,8 @@ module FlatKit
       #
       def stddev
         return 0.0 unless @count > 1
-        Math.sqrt((@sumsq - ((@sum * @sum)/@count)) / (@count - 1))
+
+        Math.sqrt((@sumsq - ((@sum * @sum) / @count)) / (@count - 1))
       end
     end
   end