flat_kit 0.2.0 → 1.0.0

data/lib/flat_kit/field_type/float_type.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module FlatKit
+  class FieldType
+    # Internal: Represeting floating point data and conversion to it
+    #
+    class FloatType < FieldType
+      def self.type_name
+        "float"
+      end
+
+      def self.matches?(data)
+        case data
+        when Float
+          true
+        when Integer
+          false
+        when String
+          return false if IntegerType.matches?(data)
+
+          maybe_float?(data)
+        else
+          false
+        end
+      end
+
+      def self.coerce(data)
+        Float(data)
+      rescue TypeError => _e
+        CoerceFailure
+      rescue ArgumentError => _e
+        CoerceFailure
+      end
+
+      def self.maybe_float?(data)
+        Float(data)
+        true
+      rescue ArgumentError => _e
+        false
+      end
+    end
+  end
+end

data/lib/flat_kit/field_type/guess_type.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module FlatKit
+  class FieldType
+    # Internal: GuessType is a field type where we don't know what type the
+    # field is, and it needs to be guessed. This is a sentinel type that doesn't
+    # match any data.
+    #
+    class GuessType < FieldType
+      def self.type_name
+        name
+      end
+
+      def self.matches?(*)
+        false
+      end
+
+      def self.coerce(*)
+        CoerceFailure
+      end
+    end
+  end
+end

data/lib/flat_kit/field_type/integer_type.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module FlatKit
+  class FieldType
+    # Internal: Class reprepseting the Integer type and coercian to it.
+    #
+    class IntegerType < FieldType
+      REGEX = /\A[-+]?\d+\Z/
+
+      def self.type_name
+        "integer"
+      end
+
+      def self.matches?(data)
+        case data
+        when Integer
+          true
+        when Float
+          false
+        when String
+          REGEX.match?(data)
+        else
+          false
+        end
+      end
+
+      def self.coerce(data)
+        Integer(data)
+      rescue TypeError => _e
+        CoerceFailure
+      rescue ArgumentError => _e
+        CoerceFailure
+      end
+    end
+  end
+end

data/lib/flat_kit/field_type/null_type.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module FlatKit
+  class FieldType
+    # Internal: Class reprepseting the null type and coercian to it.
+    #
+    class NullType < FieldType
+      REGEX = Regexp.union(/\A(null|nil)\Z/i, /\A\\N\Z/)
+
+      def self.type_name
+        "null"
+      end
+
+      def self.matches?(data)
+        case data
+        when nil
+          true
+        when String
+          REGEX.match?(data)
+        else
+          false
+        end
+      end
+
+      def self.coerce(data)
+        case data
+        when nil
+          data
+        when String
+          return nil if REGEX.match?(data)
+
+          CoerceFailure
+        else
+          CoerceFailure
+        end
+      end
+    end
+  end
+end

data/lib/flat_kit/field_type/string_type.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module FlatKit
+  class FieldType
+    # Internal: StringType is essentially a fallback - hence its lower weight
+    # than other types that might have string representations.
+    #
+    class StringType < FieldType
+      def self.type_name
+        "string"
+      end
+
+      def self.matches?(data)
+        data.is_a?(String)
+      end
+
+      def self.coerce(data)
+        data.to_s
+      rescue StandardError => _e
+        CoerceFailure
+      end
+    end
+  end
+end

data/lib/flat_kit/field_type/timestamp_type.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module FlatKit
+  class FieldType
+    # Internal: Type for all tiemstamps types more granular than Date.
+    #
+    class TimestampType < FieldType
+      def self.parse_formats
+        @parse_formats ||= [
+          "%Y-%m-%d %H:%M:%S.%NZ",
+          "%Y-%m-%d %H:%M:%S.%N",
+          "%Y-%m-%dT%H:%M:%S.%N%z", # w3cdtf
+          "%Y-%m-%d %H:%M:%S",
+          "%Y-%m-%dT%H:%M:%S%z",
+          "%Y-%m-%dT%H:%M:%SZ",
+          "%Y%m%dT%H%M%S",
+          "%a, %d %b %Y %H:%M:%S %z", # rfc2822, httpdate
+        ].freeze
+      end
+
+      def self.type_name
+        "timestamp"
+      end
+
+      def self.matches?(data)
+        coerced = coerce(data)
+        coerced.is_a?(Time)
+      end
+
+      def self.coerce(data)
+        case data
+        when Time
+          data
+        when String
+          parse_formats.each do |format|
+            coerced_data = Time.strptime(data, format).utc
+            return coerced_data
+          rescue StandardError => _e
+            # do nothing
+          end
+          CoerceFailure
+        else
+          CoerceFailure
+        end
+      end
+    end
+  end
+end

data/lib/flat_kit/field_type/unknown_type.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module FlatKit
+  class FieldType
+    # Internal: Unknown type, this is what we use for unknown values in the data
+    #
+    class UnknownType < FieldType
+      REGEX = %r{\A(na|n/a|unk|unknown)\Z}i
+
+      def self.type_name
+        "unknown"
+      end
+
+      def self.matches?(data)
+        return false unless data.is_a?(String)
+        return true if data.empty?
+
+        REGEX.match?(data)
+      end
+
+      def self.coerce(data)
+        return data if REGEX.match?(data)
+
+        CoerceFailure
+      rescue StandardError
+        CoerceFailure
+      end
+    end
+  end
+end

data/lib/flat_kit/field_type.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module FlatKit
+  # Internal: The base class for all field types
+  #
+  class FieldType
+    extend FlatKit::DescendantTracker
+
+    CoerceFailure = Class.new(::Object).freeze
+
+    def self.weights
+      @weights ||= {
+        # Boolean has crossover with Integer so going to let it overrule Integer
+        BooleanType => 5,
+
+        # Integer could potentially overlap with Float, but it is more restrictive
+        # so let it override Flaot
+        IntegerType => 4,
+        FloatType => 3,
+
+        # Date and Timestamps string representation shouldn't intersect with anything so
+        # leaving it at the same level as Null and Unkonwn
+        DateType => 2,
+        TimestampType => 2,
+
+        # Null and Unknown shoulnd't conflict since their string representations
+        # do not intersect
+        NullType => 2,
+        UnknownType => 2,
+
+        # Stringtype is the fallback for anything that has a string
+        # representation, so it should lose out on integers, floats, nulls,
+        # unknowns as strings
+        StringType => 1,
+
+        # at the bottom - since it should never match anywhere
+        GuessType => 0,
+      }
+    end
+
+    def self.candidate_types(data)
+      find_children(:matches?, data)
+    end
+
+    # rubocop:disable Style/RedundantSort
+    # We need the stable sort, max_by(&:weight) returns the wrong one
+    def self.best_guess(data)
+      candidate_types(data).sort_by(&:weight).last
+    end
+    # rubocop:enable Style/RedundantSort
+
+    def self.type_name
+      raise NotImplementedError, "must impleent #{type_name}"
+    end
+
+    def self.matches?(data)
+      raise NotImplementedError, "must implement #{name}.matches?(data)"
+    end
+
+    def self.coerce(data)
+      raise NotImplementedError, "must implement #{name}.coerce(data)"
+    end
+
+    # Each type has a weight so if a value matches multiple types, then the list
+    # can be compared to see where the tie breakers are
+    #
+    # All the weights are here so that we can see the order of precedence
+    #
+    def self.weight
+      weights.fetch(self) { raise NotImplementedError, "No weight assigned to type #{self} - fix immediately" }
+    end
+  end
+end
+
+require "flat_kit/field_type/guess_type"
+require "flat_kit/field_type/boolean_type"
+require "flat_kit/field_type/date_type"
+require "flat_kit/field_type/timestamp_type"
+require "flat_kit/field_type/integer_type"
+require "flat_kit/field_type/float_type"
+require "flat_kit/field_type/null_type"
+require "flat_kit/field_type/string_type"
+require "flat_kit/field_type/unknown_type"

data/lib/flat_kit/format.rb

@@ -1,9 +1,13 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: The base class of all data file format classes
+  #
   class Format
     extend DescendantTracker
 
     def self.format_name
-      raise NotImplementedError, "#{self.class} must implemente #{self.class}.format_name"
+      raise NotImplementedError, "#{self.class} must implement #{self.class}.format_name"
     end
 
     def format_name
@@ -20,15 +24,17 @@ module FlatKit
       return format unless format.nil?
 
       # now try the fallback
-      format = ::FlatKit::Format.for(fallback)
-      return format
+      ::FlatKit::Format.for(fallback)
     end
 
     def self.for_with_fallback!(path:, fallback: "auto")
       format = for_with_fallback(path: path, fallback: fallback)
-      raise ::FlatKit::Error::UnknownFormat, "Unable to figure out format for '#{path}' with fallback '#{fallback}'" if format.nil?
+      if format.nil?
+        raise ::FlatKit::Error::UnknownFormat,
+              "Unable to figure out format for '#{path}' with fallback '#{fallback}'"
+      end
 
-      return format
+      format
     end
   end
 end

data/lib/flat_kit/input/file.rb

@@ -1,25 +1,31 @@
-require 'zlib'
+# frozen_string_literal: true
+
+require "zlib"
+require "pathname"
 
 module FlatKit
   class Input
+    # Internal: Handler for file based input
+    #
     class File < Input
-      attr_reader :path
-      attr_reader :count
+      attr_reader :path, :count, :io
 
       def self.handles?(obj)
         return true if obj.instance_of?(Pathname)
         return false unless obj.instance_of?(String)
 
         # incase these get loaded in different orders
-        return false if ::FlatKit::Input::IO.is_stdin?(obj)
+        return false if ::FlatKit::Input::IO.stdin?(obj)
 
-        return true
+        true
       end
 
       def initialize(obj)
+        super()
         @count = 0
         @path = Pathname.new(obj)
         raise FlatKit::Error, "Input #{obj} is not readable" unless @path.readable?
+
         @io = open_input(path)
       end
 
@@ -31,10 +37,6 @@ module FlatKit
         @io.close
       end
 
-      def io
-        @io
-      end
-
       private
 
       # open the opropriate input type depending on the source file name

data/lib/flat_kit/input/io.rb

@@ -1,35 +1,38 @@
+# frozen_string_literal: true
+
 module FlatKit
   class Input
+    # Internal: Handler for non-filebased input. Generally this is just stdin
+    #
     class IO < Input
-      STDINS = %w[ stdin STDIN - <stdin> ]
+      STDINS = %w[stdin STDIN - <stdin>].freeze
 
       def self.handles?(obj)
-        return true if is_stdin?(obj)
-        return true if [ ::File, ::StringIO, ::IO ].any? { |klass| obj.kind_of?(klass) }
-        return false
+        return true if stdin?(obj)
+        return true if [::File, ::StringIO, ::IO].any? { |klass| obj.is_a?(klass) }
+
+        false
       end
 
-      def self.is_stdin?(obj)
+      def self.stdin?(obj)
         case obj
         when String
           return true if STDINS.include?(obj)
         when ::IO
-          return true if obj == ::STDIN
+          return true if obj == $stdin
         end
-        return false
+        false
       end
 
       def initialize(obj)
-        if self.class.is_stdin?(obj) then
+        super()
+        if self.class.stdin?(obj)
           @name = "<STDIN>"
           @io = $stdin
-        elsif obj.kind_of?(::File) then
-          @name = obj.path
-          @io = obj
-        elsif obj.kind_of?(::StringIO) then
-          @name = obj.inspect
+        elsif obj.is_a?(::IO)
+          @name = (obj.respond_to?(:path) && obj.path) || obj.inspect
           @io = obj
-        elsif obj.kind_of?(::IO) then
+        elsif obj.is_a?(::StringIO)
           @name = obj.inspect
           @io = obj
         else
@@ -37,18 +40,12 @@ module FlatKit
         end
       end
 
-      def name
-        @name
-      end
+      attr_reader :name, :io
 
       # this goes to an io stream and we are not in charge of opening it
       def close
         @io.close
       end
-
-      def io
-        @io
-      end
     end
   end
 end

data/lib/flat_kit/input.rb

@@ -1,14 +1,16 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: Base class of all input handlers
+  #
   class Input
     extend DescendantTracker
 
     def self.from(input)
-      return input if input.kind_of?(::FlatKit::Input)
+      return input if input.is_a?(::FlatKit::Input)
 
       in_klass = find_child(:handles?, input)
-      if in_klass then
-        return in_klass.new(input)
-      end
+      return in_klass.new(input) if in_klass
 
       raise FlatKit::Error, "Unable to create input from #{input.class} : #{input.inspect}"
     end
@@ -17,7 +19,6 @@ module FlatKit
       raise NotImplementedError, "#{self.class} must implement #name"
     end
 
-    #
     def io
       raise NotImplementedError, "#{self.class} must implement #io"
     end
@@ -28,5 +29,5 @@ module FlatKit
   end
 end
 
-require 'flat_kit/input/io'
-require 'flat_kit/input/file'
+require "flat_kit/input/io"
+require "flat_kit/input/file"

data/lib/flat_kit/internal_node.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module FlatKit
   # Private: This is a class used internally by MergeTree and should not be used
   # outside of that context.
@@ -10,22 +12,24 @@ module FlatKit
   # here.
   #
   class InternalNode
-
     include Comparable
 
-    attr_accessor :left        # Internal Node
-    attr_accessor :right       # Internal Node
-    attr_accessor :winner      # Internal Node
-    attr_accessor :next_level  # Who to tell
-    attr_accessor :leaf        # winning leaf node
+    # Internal Nodes
+    attr_accessor :left, :right, :winner
+
+    # Who to tell
+    attr_accessor :next_level
+
+    # winning leaf node
+    attr_accessor :leaf
 
     def initialize(left:, right:)
-      @left       = left
+      @left = left
       @left.next_level = self
 
-      @right      = right
+      @right = right
       @right.next_level = self
-      @next_level  = nil
+      @next_level = nil
 
       play
     end
@@ -53,32 +57,31 @@ module FlatKit
     # from the tree.
     #
     def player_finished(node)
-      if left.object_id == node.object_id then
+      if left.equal?(node)
         @left = SentinelInternalNode.new
         @left.next_level = self
-      elsif right.object_id == node.object_id then
+      elsif right.equal?(node)
         @right = SentinelInternalNode.new
         @right.next_level = self
       else
         raise FlatKit::Error, "Unknown player #{node}"
       end
 
-      if @right.sentinel? && @left.sentinel? then
-        next_level.player_finished(self) if next_level
-      end
+      return unless @right.sentinel? && @left.sentinel?
+
+      next_level.player_finished(self) if next_level
     end
 
     def play
-      @winner = left <= right ? left : right
-      if !@winner.sentinel? then
-        @leaf = winner.leaf
-      end
+      @winner = (left <= right) ? left : right
+      @leaf = winner.leaf unless @winner.sentinel?
       next_level.play if next_level
     end
 
     def <=>(other)
       return -1 if other.sentinel?
-      value.<=>(other.value)
+
+      value <=> (other.value)
     end
   end
 end

data/lib/flat_kit/jsonl/format.rb

@@ -1,5 +1,9 @@
+# frozen_string_literal: true
+
 module FlatKit
   module Jsonl
+    # Internal: JSONL format class holding the metadata about the JSONL format
+    #
     class Format < ::FlatKit::Format
       def self.format_name
         "jsonl"
@@ -7,10 +11,10 @@ module FlatKit
 
       def self.handles?(filename)
         parts = filename.split(".")
-        %w[ json jsonl ndjson ].each do |ext|
+        %w[json jsonl ndjson].each do |ext|
           return true if parts.include?(ext)
         end
-        return false
+        false
       end
 
       def self.reader

data/lib/flat_kit/jsonl/reader.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 module FlatKit
   module Jsonl
+    # Internal: Reader class that parses and yields records from JSONL files
+    #
     class Reader < ::FlatKit::Reader
-      attr_reader :input
-      attr_reader :count
+      attr_reader :input, :count
 
       def self.format_name
         ::FlatKit::Jsonl::Format.format_name
@@ -15,13 +18,13 @@ module FlatKit
       end
 
       def each
-        while line = input.io.gets do
+        while (line = input.io.gets)
           record = ::FlatKit::Jsonl::Record.new(data: line, compare_fields: compare_fields)
           @count += 1
           yield record
         end
         input.close
-      rescue => e
+      rescue StandardError => e
         ::FlatKit.logger.error "Error reading jsonl records from #{input.name}: #{e}"
         raise ::FlatKit::Error, e
       end