flat_kit 0.3.0 → 1.0.0

data/lib/flat_kit/field_type/unknown_type.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module FlatKit
   class FieldType
+    # Internal: Unknown type, this is what we use for unknown values in the data
+    #
     class UnknownType < FieldType
-
       REGEX = %r{\A(na|n/a|unk|unknown)\Z}i
 
       def self.type_name
@@ -9,18 +12,19 @@ module FlatKit
       end
 
       def self.matches?(data)
-        return false unless data.kind_of?(String)
-        return true if data.length == 0
-        return REGEX.match?(data)
+        return false unless data.is_a?(String)
+        return true if data.empty?
+
+        REGEX.match?(data)
       end
 
       def self.coerce(data)
         return data if REGEX.match?(data)
-        return CoerceFailure
-      rescue
-        return CoerceFailure
-      end
 
+        CoerceFailure
+      rescue StandardError
+        CoerceFailure
+      end
     end
   end
 end

data/lib/flat_kit/field_type.rb

@@ -1,75 +1,83 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: The base class for all field types
+  #
   class FieldType
-
     extend FlatKit::DescendantTracker
 
     CoerceFailure = Class.new(::Object).freeze
 
+    def self.weights
+      @weights ||= {
+        # Boolean has crossover with Integer so going to let it overrule Integer
+        BooleanType => 5,
+
+        # Integer could potentially overlap with Float, but it is more restrictive
+        # so let it override Flaot
+        IntegerType => 4,
+        FloatType => 3,
+
+        # Date and Timestamps string representation shouldn't intersect with anything so
+        # leaving it at the same level as Null and Unkonwn
+        DateType => 2,
+        TimestampType => 2,
+
+        # Null and Unknown shoulnd't conflict since their string representations
+        # do not intersect
+        NullType => 2,
+        UnknownType => 2,
+
+        # Stringtype is the fallback for anything that has a string
+        # representation, so it should lose out on integers, floats, nulls,
+        # unknowns as strings
+        StringType => 1,
+
+        # at the bottom - since it should never match anywhere
+        GuessType => 0,
+      }
+    end
+
     def self.candidate_types(data)
       find_children(:matches?, data)
     end
 
+    # rubocop:disable Style/RedundantSort
+    # We need the stable sort, max_by(&:weight) returns the wrong one
     def self.best_guess(data)
-      candidate_types(data).sort_by { |t| t.weight }.last
+      candidate_types(data).sort_by(&:weight).last
     end
+    # rubocop:enable Style/RedundantSort
 
     def self.type_name
-      raise NotImplementedError, "must impleent #{self.type_name}"
+      raise NotImplementedError, "must impleent #{type_name}"
     end
 
     def self.matches?(data)
-      raise NotImplementedError, "must implement #{self.name}.matches?(data)"
+      raise NotImplementedError, "must implement #{name}.matches?(data)"
     end
 
     def self.coerce(data)
-      raise NotImplementedError, "must implement #{self.name}.coerce(data)"
+      raise NotImplementedError, "must implement #{name}.coerce(data)"
     end
 
     # Each type has a weight so if a value matches multiple types, then the list
     # can be compared to see where the tie breakers are
     #
-    # All the weights are here so that 
-    #
+    # All the weights are here so that we can see the order of precedence
     #
     def self.weight
-      # Boolean has crossover with Integer so going to let it overrule Integer
-      return 5 if self == BooleanType
-
-
-      # Integer could potentially overlap with Float, but it is more restrictive
-      # so let it override Flaot
-      return 4 if self == IntegerType
-      return 3 if self == FloatType
-
-      # Date and Timestamps string representation shouldn't intersect with anything so
-      # leaving it at the same level as Null and Unkonwn
-      return 2 if self == DateType
-      return 2 if self == TimestampType
-
-      # Null and Unknown shoulnd't conflict since their string representations
-      # do not intersect
-      return 2 if self == NullType
-      return 2 if self == UnknownType
-
-      # Stringtype is the fallback for anything that has a string
-      # representation, so it should lose out on integers, floats, nulls,
-      # unknowns as strings
-      return 1 if self == StringType
-
-      # at the bottom - since it should never match anywhere
-      return 0 if self == GuessType
-
-      raise NotImplementedError, "No weight assigned to type #{self} - fix immediately"
+      weights.fetch(self) { raise NotImplementedError, "No weight assigned to type #{self} - fix immediately" }
     end
   end
 end
 
-require 'flat_kit/field_type/guess_type'
-require 'flat_kit/field_type/boolean_type'
-require 'flat_kit/field_type/date_type'
-require 'flat_kit/field_type/timestamp_type'
-require 'flat_kit/field_type/integer_type'
-require 'flat_kit/field_type/float_type'
-require 'flat_kit/field_type/null_type'
-require 'flat_kit/field_type/string_type'
-require 'flat_kit/field_type/unknown_type'
+require "flat_kit/field_type/guess_type"
+require "flat_kit/field_type/boolean_type"
+require "flat_kit/field_type/date_type"
+require "flat_kit/field_type/timestamp_type"
+require "flat_kit/field_type/integer_type"
+require "flat_kit/field_type/float_type"
+require "flat_kit/field_type/null_type"
+require "flat_kit/field_type/string_type"
+require "flat_kit/field_type/unknown_type"

data/lib/flat_kit/format.rb

@@ -1,9 +1,13 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: The base class of all data file format classes
+  #
   class Format
     extend DescendantTracker
 
     def self.format_name
-      raise NotImplementedError, "#{self.class} must implemente #{self.class}.format_name"
+      raise NotImplementedError, "#{self.class} must implement #{self.class}.format_name"
     end
 
     def format_name
@@ -20,15 +24,17 @@ module FlatKit
       return format unless format.nil?
 
       # now try the fallback
-      format = ::FlatKit::Format.for(fallback)
-      return format
+      ::FlatKit::Format.for(fallback)
     end
 
     def self.for_with_fallback!(path:, fallback: "auto")
       format = for_with_fallback(path: path, fallback: fallback)
-      raise ::FlatKit::Error::UnknownFormat, "Unable to figure out format for '#{path}' with fallback '#{fallback}'" if format.nil?
+      if format.nil?
+        raise ::FlatKit::Error::UnknownFormat,
+              "Unable to figure out format for '#{path}' with fallback '#{fallback}'"
+      end
 
-      return format
+      format
     end
   end
 end

data/lib/flat_kit/input/file.rb

@@ -1,25 +1,31 @@
-require 'zlib'
+# frozen_string_literal: true
+
+require "zlib"
+require "pathname"
 
 module FlatKit
   class Input
+    # Internal: Handler for file based input
+    #
     class File < Input
-      attr_reader :path
-      attr_reader :count
+      attr_reader :path, :count, :io
 
       def self.handles?(obj)
         return true if obj.instance_of?(Pathname)
         return false unless obj.instance_of?(String)
 
         # incase these get loaded in different orders
-        return false if ::FlatKit::Input::IO.is_stdin?(obj)
+        return false if ::FlatKit::Input::IO.stdin?(obj)
 
-        return true
+        true
       end
 
       def initialize(obj)
+        super()
         @count = 0
         @path = Pathname.new(obj)
         raise FlatKit::Error, "Input #{obj} is not readable" unless @path.readable?
+
         @io = open_input(path)
       end
 
@@ -31,10 +37,6 @@ module FlatKit
         @io.close
       end
 
-      def io
-        @io
-      end
-
       private
 
       # open the opropriate input type depending on the source file name

data/lib/flat_kit/input/io.rb

@@ -1,35 +1,38 @@
+# frozen_string_literal: true
+
 module FlatKit
   class Input
+    # Internal: Handler for non-filebased input. Generally this is just stdin
+    #
     class IO < Input
-      STDINS = %w[ stdin STDIN - <stdin> ]
+      STDINS = %w[stdin STDIN - <stdin>].freeze
 
       def self.handles?(obj)
-        return true if is_stdin?(obj)
-        return true if [ ::File, ::StringIO, ::IO ].any? { |klass| obj.kind_of?(klass) }
-        return false
+        return true if stdin?(obj)
+        return true if [::File, ::StringIO, ::IO].any? { |klass| obj.is_a?(klass) }
+
+        false
       end
 
-      def self.is_stdin?(obj)
+      def self.stdin?(obj)
         case obj
         when String
           return true if STDINS.include?(obj)
         when ::IO
-          return true if obj == ::STDIN
+          return true if obj == $stdin
         end
-        return false
+        false
       end
 
       def initialize(obj)
-        if self.class.is_stdin?(obj) then
+        super()
+        if self.class.stdin?(obj)
           @name = "<STDIN>"
           @io = $stdin
-        elsif obj.kind_of?(::File) then
-          @name = obj.path
-          @io = obj
-        elsif obj.kind_of?(::StringIO) then
-          @name = obj.inspect
+        elsif obj.is_a?(::IO)
+          @name = (obj.respond_to?(:path) && obj.path) || obj.inspect
           @io = obj
-        elsif obj.kind_of?(::IO) then
+        elsif obj.is_a?(::StringIO)
           @name = obj.inspect
           @io = obj
         else
@@ -37,18 +40,12 @@ module FlatKit
         end
       end
 
-      def name
-        @name
-      end
+      attr_reader :name, :io
 
       # this goes to an io stream and we are not in charge of opening it
       def close
         @io.close
       end
-
-      def io
-        @io
-      end
     end
   end
 end

data/lib/flat_kit/input.rb

@@ -1,14 +1,16 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Base class of all input handlers
+  #
   class Input
     extend DescendantTracker
 
     def self.from(input)
-      return input if input.kind_of?(::FlatKit::Input)
+      return input if input.is_a?(::FlatKit::Input)
 
       in_klass = find_child(:handles?, input)
-      if in_klass then
-        return in_klass.new(input)
-      end
+      return in_klass.new(input) if in_klass
 
       raise FlatKit::Error, "Unable to create input from #{input.class} : #{input.inspect}"
     end
@@ -17,7 +19,6 @@ module FlatKit
       raise NotImplementedError, "#{self.class} must implement #name"
     end
 
-    #
     def io
       raise NotImplementedError, "#{self.class} must implement #io"
     end
@@ -28,5 +29,5 @@ module FlatKit
   end
 end
 
-require 'flat_kit/input/io'
-require 'flat_kit/input/file'
+require "flat_kit/input/io"
+require "flat_kit/input/file"

data/lib/flat_kit/internal_node.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Private: This is a class used internally by MergeTree and should not be used
   # outside of that context.
@@ -10,22 +12,24 @@ module FlatKit
   # here.
   #
   class InternalNode
-
     include Comparable
 
-    attr_accessor :left        # Internal Node
-    attr_accessor :right       # Internal Node
-    attr_accessor :winner      # Internal Node
-    attr_accessor :next_level  # Who to tell
-    attr_accessor :leaf        # winning leaf node
+    # Internal Nodes
+    attr_accessor :left, :right, :winner
+
+    # Who to tell
+    attr_accessor :next_level
+
+    # winning leaf node
+    attr_accessor :leaf
 
     def initialize(left:, right:)
-      @left       = left
+      @left = left
       @left.next_level = self
 
-      @right      = right
+      @right = right
       @right.next_level = self
-      @next_level  = nil
+      @next_level = nil
 
       play
     end
@@ -53,32 +57,31 @@ module FlatKit
     # from the tree.
     #
     def player_finished(node)
-      if left.object_id == node.object_id then
+      if left.equal?(node)
         @left = SentinelInternalNode.new
         @left.next_level = self
-      elsif right.object_id == node.object_id then
+      elsif right.equal?(node)
         @right = SentinelInternalNode.new
         @right.next_level = self
       else
         raise FlatKit::Error, "Unknown player #{node}"
       end
 
-      if @right.sentinel? && @left.sentinel? then
-        next_level.player_finished(self) if next_level
-      end
+      return unless @right.sentinel? && @left.sentinel?
+
+      next_level.player_finished(self) if next_level
     end
 
     def play
-      @winner = left <= right ? left : right
-      if !@winner.sentinel? then
-        @leaf = winner.leaf
-      end
+      @winner = (left <= right) ? left : right
+      @leaf = winner.leaf unless @winner.sentinel?
       next_level.play if next_level
     end
 
     def <=>(other)
       return -1 if other.sentinel?
-      value.<=>(other.value)
+
+      value <=> (other.value)
     end
   end
 end

data/lib/flat_kit/jsonl/format.rb

@@ -1,5 +1,9 @@
+# frozen_string_literal: true
+
 module FlatKit
   module Jsonl
+    # Internal: JSONL format class holding the metadata about the JSONL format
+    #
     class Format < ::FlatKit::Format
       def self.format_name
         "jsonl"
@@ -7,10 +11,10 @@ module FlatKit
 
       def self.handles?(filename)
         parts = filename.split(".")
-        %w[ json jsonl ndjson ].each do |ext|
+        %w[json jsonl ndjson].each do |ext|
           return true if parts.include?(ext)
         end
-        return false
+        false
       end
 
       def self.reader

data/lib/flat_kit/jsonl/reader.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 module FlatKit
   module Jsonl
+    # Internal: Reader class that parses and yields records from JSONL files
+    #
     class Reader < ::FlatKit::Reader
-      attr_reader :input
-      attr_reader :count
+      attr_reader :input, :count
 
       def self.format_name
         ::FlatKit::Jsonl::Format.format_name
@@ -15,13 +18,13 @@ module FlatKit
       end
 
       def each
-        while line = input.io.gets do
+        while (line = input.io.gets)
           record = ::FlatKit::Jsonl::Record.new(data: line, compare_fields: compare_fields)
           @count += 1
           yield record
         end
         input.close
-      rescue => e
+      rescue StandardError => e
         ::FlatKit.logger.error "Error reading jsonl records from #{input.name}: #{e}"
         raise ::FlatKit::Error, e
       end

data/lib/flat_kit/jsonl/record.rb

@@ -1,8 +1,12 @@
-require 'oj'
-require 'flat_kit/record'
+# frozen_string_literal: true
+
+require "oj"
+require "flat_kit/record"
 
 module FlatKit
   module Jsonl
+    # Internal: Class that exposes data from a JSONL format record to the flatkit api
+    #
     class Record < ::FlatKit::Record
       attr_reader :compare_data
 
@@ -11,7 +15,7 @@ module FlatKit
       end
 
       def self.from_record(record)
-        if record.instance_of?(FlatKit::Jsonl::Record) then
+        if record.instance_of?(FlatKit::Jsonl::Record)
 
           structured = record.complete_structured_data? ? record.complete_structured_data : nil
 
@@ -25,22 +29,20 @@ module FlatKit
       end
 
       def initialize(data:, compare_fields: :none,
-                     compare_data: Hash.new,
+                     compare_data: {},
                      complete_structured_data: nil)
         super(data: data, compare_fields: compare_fields)
 
         @complete_structured_data = complete_structured_data
 
-        if complete_structured_data? && (compare_data.nil? || compare_data.empty?) then
-          @compare_data = complete_structured_data
-        else
-          @compare_data = compare_data
-        end
+        @compare_data = if complete_structured_data? && (compare_data.nil? || compare_data.empty?)
+                          complete_structured_data
+                        else
+                          compare_data
+                        end
 
         # only load compare data if it dosn't exist
-        if data && compare_data.empty? then
-          quick_parse
-        end
+        quick_parse if data && compare_data.empty?
       end
 
       def [](key)
@@ -59,9 +61,7 @@ module FlatKit
       # overriding parent accessor since we may be initialized without raw bytes
       # to parse
       def data
-        if @data.nil? && complete_structured_data? then
-          @data = Oj.dump(complete_structured_data, mode: :json)
-        end
+        @data = Oj.dump(complete_structured_data, mode: :json) if @data.nil? && complete_structured_data?
         @data
       end
       alias to_s data
@@ -79,6 +79,3 @@ module FlatKit
     end
   end
 end
-
-
-

data/lib/flat_kit/jsonl/writer.rb

@@ -1,15 +1,14 @@
+# frozen_string_literal: true
+
 module FlatKit
   module Jsonl
+    # Internal: Class that writes flatkit records to JSONL files
+    #
     class Writer < ::FlatKit::Writer
-
       def self.format_name
         ::FlatKit::Jsonl::Format.format_name
       end
 
-      def initialize(destination:)
-        super
-      end
-
       # write the record and return the Position the record was written
       #
       def write(record)
@@ -22,10 +21,10 @@ module FlatKit
         else
           raise FlatKit::Error, "Unable to write records of type #{record.class}"
         end
-      rescue FlatKit::Error => fe
-        raise fe
-      rescue => e
-        ::FlatKit.logger.error "Error reading jsonl records from #{output.name}: #{e}"
+      rescue FlatKit::Error => e
+        raise e
+      rescue StandardError => e
+        ::FlatKit.logger.error "Error writing jsonl records to #{output.name}: #{e}"
         raise ::FlatKit::Error, e
       end
 
@@ -47,7 +46,6 @@ module FlatKit
         @last_position = ::FlatKit::Position.new(index: record_index,
                                                  offset: start_offset,
                                                  bytesize: bytes_written)
-
       end
     end
   end

data/lib/flat_kit/jsonl.rb

@@ -1,8 +1,12 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Top level namespace for the newline-oriented JSON format
+  #
   module Jsonl
   end
 end
-require 'flat_kit/jsonl/record'
-require 'flat_kit/jsonl/reader'
-require 'flat_kit/jsonl/writer'
-require 'flat_kit/jsonl/format'
+require "flat_kit/jsonl/record"
+require "flat_kit/jsonl/reader"
+require "flat_kit/jsonl/writer"
+require "flat_kit/jsonl/format"

data/lib/flat_kit/leaf_node.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Private: The LeafNode is a wrapper around a Reader object to enable
   # a consistent api for use in the MergeTree
@@ -9,11 +11,9 @@ module FlatKit
   # If all the data is used up from the reader, it also notifies the next level
   # of that so the next level can remove it from the tree.
   class LeafNode
-
     include Comparable
 
-    attr_reader :reader
-    attr_reader :value
+    attr_reader :reader, :value
 
     attr_accessor :next_level
 
@@ -43,7 +43,7 @@ module FlatKit
 
     def update_and_replay
       self.next
-      if finished? then
+      if finished?
         ::FlatKit.logger.debug "#{reader.source} has finished reading #{reader.count} records"
         next_level.player_finished(self)
       end
@@ -65,7 +65,8 @@ module FlatKit
 
     def <=>(other)
       return -1 if other.sentinel?
-      self.value.<=>(other.value)
+
+      value <=> (other.value)
     end
   end
 end

data/lib/flat_kit/log_formatter.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "logger"
+
+module FlatKit
+  # Internal: Log formatting class for FlatKit
+  #
+  class LogFormatter < ::Logger::Formatter
+    FORMAT          = "%s %5d %05s : %s\n"
+    DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+    def initialize
+      super
+      self.datetime_format = DATETIME_FORMAT
+    end
+
+    def call(severity, time, _progname, msg)
+      format(FORMAT, format_datetime(time.utc), Process.pid, severity, msg2str(msg))
+    end
+  end
+end