flat_kit 0.2.0 → 0.3.0

data/lib/flat_kit/field_type.rb

@@ -0,0 +1,75 @@
+module FlatKit
+  class FieldType
+
+    extend FlatKit::DescendantTracker
+
+    CoerceFailure = Class.new(::Object).freeze
+
+    def self.candidate_types(data)
+      find_children(:matches?, data)
+    end
+
+    def self.best_guess(data)
+      candidate_types(data).sort_by { |t| t.weight }.last
+    end
+
+    def self.type_name
+      raise NotImplementedError, "must impleent #{self.type_name}"
+    end
+
+    def self.matches?(data)
+      raise NotImplementedError, "must implement #{self.name}.matches?(data)"
+    end
+
+    def self.coerce(data)
+      raise NotImplementedError, "must implement #{self.name}.coerce(data)"
+    end
+
+    # Each type has a weight so if a value matches multiple types, then the list
+    # can be compared to see where the tie breakers are
+    #
+    # All the weights are here so that 
+    #
+    #
+    def self.weight
+      # Boolean has crossover with Integer so going to let it overrule Integer
+      return 5 if self == BooleanType
+
+
+      # Integer could potentially overlap with Float, but it is more restrictive
+      # so let it override Flaot
+      return 4 if self == IntegerType
+      return 3 if self == FloatType
+
+      # Date and Timestamps string representation shouldn't intersect with anything so
+      # leaving it at the same level as Null and Unkonwn
+      return 2 if self == DateType
+      return 2 if self == TimestampType
+
+      # Null and Unknown shoulnd't conflict since their string representations
+      # do not intersect
+      return 2 if self == NullType
+      return 2 if self == UnknownType
+
+      # Stringtype is the fallback for anything that has a string
+      # representation, so it should lose out on integers, floats, nulls,
+      # unknowns as strings
+      return 1 if self == StringType
+
+      # at the bottom - since it should never match anywhere
+      return 0 if self == GuessType
+
+      raise NotImplementedError, "No weight assigned to type #{self} - fix immediately"
+    end
+  end
+end
+
+require 'flat_kit/field_type/guess_type'
+require 'flat_kit/field_type/boolean_type'
+require 'flat_kit/field_type/date_type'
+require 'flat_kit/field_type/timestamp_type'
+require 'flat_kit/field_type/integer_type'
+require 'flat_kit/field_type/float_type'
+require 'flat_kit/field_type/null_type'
+require 'flat_kit/field_type/string_type'
+require 'flat_kit/field_type/unknown_type'

data/lib/flat_kit/field_type/boolean_type.rb

@@ -0,0 +1,48 @@
+module FlatKit
+  class FieldType
+    class BooleanType < FieldType
+
+      TRUTHY_REGEX = /\A(true|t|1|yes|y|on)\Z/i
+      FALSEY_REGEX = /\A(false|f|0|no|n|off)\Z/i
+      REGEX        = Regexp.union(TRUTHY_REGEX, FALSEY_REGEX)
+
+      def self.type_name
+        "boolean"
+      end
+
+      def self.matches?(data)
+        case data
+        when TrueClass
+          true
+        when FalseClass
+          true
+        when String
+          REGEX.match?(data)
+        when Integer
+          return true if data.zero?
+          return true if data == 1
+          return false
+        else
+          false
+        end
+      end
+
+      def self.coerce(data)
+        case data
+        when TrueClass
+          true
+        when FalseClass
+          false
+        when Numeric
+          return false if data.zero?
+          return true  if data == 1
+          CoerceFailure
+        when String
+          return true  if TRUTHY_REGEX.match?(data)
+          return false if FALSEY_REGEX.match?(data)
+          CoerceFailure
+        end
+      end
+    end
+  end
+end

data/lib/flat_kit/field_type/date_type.rb

@@ -0,0 +1,179 @@
+module FlatKit
+  class FieldType
+    # Representing the type of data which only includes data up to the day
+    # resolution
+    class DateType < FieldType
+
+      # %Y   4 digit year
+      # %y   2 didigt year (%Y mod 100) (00..99)
+      # %m   month of year zero padded
+      # %-m  month of year no-padding
+      # %B   Full month name
+      # %b   Abbreviated month name
+      # %^b  uppercased month name
+      # %d   day of month zero padded
+      # %-d  day of moneth not padded
+      # %e   day of month blank padded
+      # %j   day of year zero padded
+
+      # parse formats are not the same as print formats as parsing does not deal
+      # with flags and widths
+      def self.parse_formats
+        @parse_formats ||= [
+          # YMD formats
+          "%Y-%m-%d",
+          "%Y%m%d",
+          "%Y/%m/%d",
+          "%Y %m %d.",
+
+          # DMY formats
+          "%d %B %Y",
+          "%d %b %Y",
+          "%d-%b-%Y",
+          "%d/%b/%Y",
+          "%d-%m-%Y",
+          "%d-%m-%y",
+          "%d %b, %Y",
+          "%d %b,%Y",
+          "%d %B, %Y",
+          "%d %B,%Y",
+
+          # MDY formats
+          "%m/%d/%Y",
+          "%m-%d-%Y",
+          "%m/%d/%y",
+          "%m-%d-%y",
+
+          "%B %d, %Y",
+          "%b %d, %Y",
+
+          # other formats
+          "%Y-%j",
+          "%a %b %d %Y"
+       ]
+
+      end
+
+      # https://en.wikipedia.org/wiki/Date_format_by_country
+      # List of formats culled from the above - not using all as it is
+      # definitely a performance issue at the moment
+      # def self.known_formats
+      #   @known_formats ||= [
+      #     # YMD formats
+      #     "%Y-%m-%d",
+      #     "%Y%m%d",
+      #     "%Y/%m/%d",
+      #     "%Y.%m.%d",
+      #     "%Y.%m.%d.",
+      #     "%Y %m %d.",
+      #     "%Y %b %d",
+      #     "%Y %b %-d",
+      #     "%Y %B %-d",
+      #     "%Y %B %d",
+      #     "%Y-%m%d",
+      #     "%Y. %m. %-d.",
+      #     "%Y. %m. %d.",
+      #     "%Y.%-m.%-d.",
+      #     "%Y.%-m.%-d",
+      #     "%Y, %d %B",
+      #     "%Y, %d %b",
+      #
+      #     "%y.%-m.%-d",
+      #     "%y.%-m.%-d.",
+      #     "%y.%m.%d.",
+      #     "%y.%m.%d",
+      #     "%y/%m/%d",
+      #
+      #     # DMY formats
+      #     "%-d %b %Y",
+      #     "%-d %B %Y",
+      #     "%-d-%-m-%Y",
+      #     "%-d. %-m. %Y",
+      #     "%-d. %-m. %Y.",
+      #     "%-d. %B %Y",
+      #     "%-d. %B %Y.",
+      #     "%-d.%-m.%Y",
+      #     "%-d.%-m.%Y.",
+      #     "%-d.%m.%Y.",
+      #     "%-d.%m.%Y",
+      #     "%-d.%b.%Y",
+      #     "%-d.%B.%Y",
+      #     "%-d/%-m %Y",
+      #     "%-d/%-m/%Y",
+      #     "%d %B %Y",
+      #     "%d %b %Y",
+      #     "%d-%m-%Y",
+      #     "%d-%b-%Y",
+      #     "%d-%B-%Y",
+      #     "%d.%m.%Y",
+      #     "%d/%m %Y",
+      #     "%d/%m/%Y",
+      #
+      #     "%-d.%b.%y",
+      #     "%-d.%B.%y",
+      #     "%-d.%-m.%y",
+      #     "%-d/%-m-%y",
+      #     "%-d/%-m/%y",
+      #     "%d/%m/%y",
+      #     "%d-%m-%y",
+      #     "%d.%m.%y",
+      #     "%d%m%y",
+      #
+      #     # MDY formats
+      #     "%-m/%-d/%Y",
+      #     "%m/%d/%Y",
+      #     "%m-%d-%Y",
+      #     "%b-%d-%Y",
+      #     "%B %-d, %Y",
+      #     "%B %-d. %Y",
+      #     "%B %d, %Y",
+      #     "%B-%d-%Y",
+      #     "%B/%d/%Y",
+      #
+      #     "%-m/%-d/%y",
+      #
+      #     # other formats
+      #     "%Y-%j",
+      #     "%Y%m",
+      #     "%Y-%m",
+      #     "%Y %m",
+      #   ]
+      # end
+
+      def self.type_name
+        "date"
+      end
+
+      def self.matches?(data)
+        coerced = coerce(data)
+        return coerced.kind_of?(Date)
+      end
+
+      def self.coerce(data)
+        case data
+        when DateTime
+          CoerceFailure
+        when Date
+          data
+        when String
+          coerced_data = CoerceFailure
+          parse_formats.each do |format|
+            begin
+              coerced_data = Date.strptime(data, format)
+              break
+            rescue => _
+              false
+            end
+          end
+          coerced_data
+        else
+          CoerceFailure
+        end
+      end
+    end
+  end
+end
+
+
+__END__
+

data/lib/flat_kit/field_type/float_type.rb

@@ -0,0 +1,37 @@
+module FlatKit
+  class FieldType
+    class FloatType < FieldType
+
+      def self.type_name
+        "float"
+      end
+
+      def self.matches?(data)
+        case data
+        when Float
+          true
+        when Integer
+          false
+        when String
+          return false if IntegerType.matches?(data)
+          begin
+            Float(data)
+            true
+          rescue ArgumentError => _
+            false
+          end
+        else
+          false
+        end
+      end
+
+      def self.coerce(data)
+        Float(data)
+      rescue TypeError => _
+        CoerceFailure
+      rescue ArgumentError => _
+        CoerceFailure
+      end
+    end
+  end
+end

data/lib/flat_kit/field_type/guess_type.rb

@@ -0,0 +1,20 @@
+module FlatKit
+  class FieldType
+    # GuessType is a field type where we don't know what type the field is, and
+    # it needs to be guessed. This is a sentinel type that doesn't match any
+    # data.
+    class GuessType < FieldType
+      def self.type_name
+        self.name
+      end
+
+      def self.matches?(data)
+        false
+      end
+
+      def self.coerce(data)
+        CoerceFailure
+      end
+    end
+  end
+end

data/lib/flat_kit/field_type/integer_type.rb

@@ -0,0 +1,34 @@
+module FlatKit
+  class FieldType
+    class IntegerType < FieldType
+
+      REGEX = /\A[-+]?\d+\Z/
+
+      def self.type_name
+        "integer"
+      end
+
+      def self.matches?(data)
+        case data
+        when Integer
+          true
+        when Float
+          false
+        when String
+          REGEX.match?(data)
+        else
+          false
+        end
+      end
+
+      def self.coerce(data)
+        Integer(data)
+      rescue TypeError => _
+        CoerceFailure
+      rescue ArgumentError => _
+        CoerceFailure
+      end
+ 
+    end
+  end
+end

data/lib/flat_kit/field_type/null_type.rb

@@ -0,0 +1,35 @@
+module FlatKit
+  class FieldType
+    class NullType < FieldType
+
+      REGEX = Regexp.union(/\A(null|nil)\Z/i, /\A\\N\Z/)
+
+      def self.type_name
+        "null"
+      end
+
+      def self.matches?(data)
+        case data
+        when nil
+          true
+        when String
+          REGEX.match?(data)
+        else
+          false
+        end
+      end
+
+      def self.coerce(data)
+        case data
+        when nil
+          data
+        when String
+          return nil if REGEX.match?(data)
+          CoerceFailure
+        else
+          CoerceFailure
+        end
+      end
+    end
+  end
+end

data/lib/flat_kit/field_type/string_type.rb

@@ -0,0 +1,22 @@
+module FlatKit
+  class FieldType
+    # StringType is essentially a fallback - hence its lower weight than other
+    # types that might have string representations.
+    class StringType< FieldType
+
+      def self.type_name
+        "string"
+      end
+
+      def self.matches?(data)
+        data.kind_of?(String)
+      end
+
+      def self.coerce(data)
+        data.to_s
+      rescue => _
+        CoerceFailure
+      end
+    end
+  end
+end