flat_kit 0.3.0 → 1.0.0

data/lib/flat_kit/command/stats.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module FlatKit
   class Command
+    # Internal: The implementation of the stats command.
+    #
     class Stats < ::FlatKit::Command
-
       def self.name
         "stats"
       end
@@ -12,45 +15,45 @@ module FlatKit
 
       def self.parser
         ::Optimist::Parser.new do
-          banner "#{Sort.description}"
+          banner Sort.description.to_s
           banner ""
 
           banner <<~BANNER
-          Given an input file collect basic statistics.
+            Given an input file collect basic statistics.
 
-          The statistics can vary based upon the datatype of the field.
+            The statistics can vary based upon the datatype of the field.
 
-          Numeric fields will report the basic count, min, max, mean, standard deviation and sum.
-          Non-numeric fields that are comparable, like dates, will report count, min and max.
-          Other non-numeric fields will only report the count.
+            Numeric fields will report the basic count, min, max, mean, standard deviation and sum.
+            Non-numeric fields that are comparable, like dates, will report count, min and max.
+            Other non-numeric fields will only report the count.
 
-          Adding --cardinality will report the count, and frequency of distinct values in the result.
-          This will allow for reporting the median value.
+            Adding --cardinality will report the count, and frequency of distinct values in the result.
+            This will allow for reporting the median value.
 
-          The fields upon which stats are collected may be selected with the --fields parameter.
-          By default statistics are collected on all fields.
+            The fields upon which stats are collected may be selected with the --fields parameter.
+            By default statistics are collected on all fields.
 
-          The flatfile type(s) will be automatically determined by the file name.
+            The flatfile type(s) will be automatically determined by the file name.
 
-          The output can be dumped as a CSV, JSON or a a formated ascii table.
+            The output can be dumped as a CSV, JSON or a a formated ascii table.
 
           BANNER
 
           banner <<~USAGE
 
-          Usage:
-            fk stats --everything file.json
-            fk stats --select surname,given_name file.csv
-            fk stats --select surname,given_name --output-format json file.csv > stats.json
-            fk stats --select field1,field2 --output-format json input.csv
-            fk stats --select field1 file.json.gz -o stats.csv
-            gunzip -c file.json.gz | fk stats --input-format json --output-format text
+            Usage:
+              fk stats --everything file.json
+              fk stats --select surname,given_name file.csv
+              fk stats --select surname,given_name --output-format json file.csv > stats.json
+              fk stats --select field1,field2 --output-format json input.csv
+              fk stats --select field1 file.json.gz -o stats.csv
+              gunzip -c file.json.gz | fk stats --input-format json --output-format text
 
           USAGE
 
           banner <<~OPTIONS
 
-          Options:
+            Options:
 
           OPTIONS
 
@@ -65,24 +68,23 @@ module FlatKit
 
       def parse
         parser = self.class.parser
-        ::Optimist::with_standard_exception_handling(parser) do
-          begin
-            opts = parser.parse(argv)
-            fields = ::FlatKit::Stats::AllFields
-            fields = CSV.parse_line(opts[:select]) if opts[:select]
-
-            stats = [FieldStats::CORE_STATS]
-            stats << FieldStats::CARDINALITY_STATS if opts[:cardinality] || opts[:everything]
-
-            paths = parser.leftovers
-            raise ::Optimist::CommandlineError, "1 and only 1 input file is allowed" if paths.size > 1
-            path = paths.first || "-" # default to stdin
-            @stats = ::FlatKit::Stats.new(input: path, input_fallback: opts[:input_format],
-                                         output: opts[:output], output_fallback: opts[:output_format],
-                                         fields_to_stat: fields, stats_to_collect: stats)
-          rescue ::FlatKit::Error => e
-            raise ::Optimist::CommandlineError, e.message
-          end
+        ::Optimist.with_standard_exception_handling(parser) do
+          opts = parser.parse(argv)
+          fields = ::FlatKit::Stats::AllFields
+          fields = CSV.parse_line(opts[:select]) if opts[:select]
+
+          stats = [FieldStats::CORE_STATS]
+          stats << FieldStats::CARDINALITY_STATS if opts[:cardinality] || opts[:everything]
+
+          paths = parser.leftovers
+          raise ::Optimist::CommandlineError, "1 and only 1 input file is allowed" if paths.size > 1
+
+          path = paths.first || "-" # default to stdin
+          @stats = ::FlatKit::Stats.new(input: path, input_fallback: opts[:input_format],
+                                        output: opts[:output], output_fallback: opts[:output_format],
+                                        fields_to_stat: fields, stats_to_collect: stats)
+        rescue ::FlatKit::Error => e
+          raise ::Optimist::CommandlineError, e.message
         end
       end
 

data/lib/flat_kit/command.rb

@@ -1,13 +1,12 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: The base class for all commands in the CLI
+  #
   class Command
     extend DescendantTracker
 
-    attr_reader :argv
-    attr_reader :env
-    attr_reader :logger
-    attr_reader :opts
-    attr_reader :readers
-    attr_reader :writer
+    attr_reader :argv, :env, :logger, :opts, :readers, :writer
 
     def self.name
       raise NotImplementedError, "#{self.class} must implement #{self.class}.name"
@@ -22,7 +21,7 @@ module FlatKit
     end
 
     def self.names
-      children.map { |c| c.name }
+      children.map(&:name)
     end
 
     def self.for(name)
@@ -48,7 +47,7 @@ module FlatKit
   end
 end
 
-require 'flat_kit/command/cat'
-require 'flat_kit/command/merge'
-require 'flat_kit/command/sort'
-require 'flat_kit/command/stats'
+require "flat_kit/command/cat"
+require "flat_kit/command/merge"
+require "flat_kit/command/sort"
+require "flat_kit/command/stats"

data/lib/flat_kit/descendant_tracker.rb

@@ -1,17 +1,20 @@
-require 'set'
+# frozen_string_literal: true
+
+require "set"
 
 module FlatKit
+  # Internal: A module to track descendants of a class
+  #
   module DescendantTracker
     def inherited(klass)
       super
       return unless klass.instance_of?(Class)
-      self.children << klass
+
+      children << klass
     end
 
     def children
-      unless defined? @_children
-        @_children = Set.new
-      end
+      @_children = Set.new unless defined? @_children
       @_children
     end
 
@@ -24,7 +27,7 @@ module FlatKit
       end
     end
 
-    # 
+    #
     # Find all the children that return truthy from the given method with args
     #
     def find_children(method, *args)

data/lib/flat_kit/error.rb

@@ -1,4 +1,8 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: A Base error class for all FlatKit errors
+  #
   class Error < ::StandardError
     class UnknownFormat < ::FlatKit::Error; end
   end

data/lib/flat_kit/event_emitter.rb

@@ -1,9 +1,12 @@
+# frozen_string_literal: true
+
 module FlatKit
-  # A simplified Observable class for use internally
+  # Internal: A simplified Observable class for use internally
   #
   module EventEmitter
     def add_listener(listener)
       raise ::NoMethodError, "#{listener} does not resond to #on_event" unless listener.respond_to?(:on_event)
+
       self._listeners ||= []
       self._listeners << listener unless _listeners.include?(listener)
     end
@@ -27,7 +30,7 @@ module FlatKit
     end
 
     def _listeners
-      @_listeners ||= Array.new
+      @_listeners ||= []
     end
   end
 end

data/lib/flat_kit/field_stats.rb

@@ -1,7 +1,11 @@
+# frozen_string_literal: true
+
 module FlatKit
-  # Collect stats on a single field. We may not know what the field data type is
-  # to start with, so collect a bunch of values until we have the threshold, and
-  # then calculte states based upon the data types determined by the guess
+  # Internal: Collect stats on a single field.
+  #
+  # We may not know what the field data type is to start with, so collect a
+  # bunch of values until we have the threshold, and then calculte states based
+  # upon the data types determined by the guess
   #
   class FieldStats
     DEFAULT_GUESS_THRESHOLD = 1000
@@ -9,7 +13,7 @@ module FlatKit
     CORE_STATS = :core
     CARDINALITY_STATS = :cardinality
 
-    ALL_STATS = [ CORE_STATS, CARDINALITY_STATS ]
+    ALL_STATS = [CORE_STATS, CARDINALITY_STATS].freeze
 
     EXPORT_FIELDS = %w[
       name
@@ -36,14 +40,13 @@ module FlatKit
       total_count
       null_percent
       unknown_percent
-    ]
-
+    ].freeze
 
-    attr_reader :type_counts
-    attr_reader :field_type
-    attr_reader :name
+    attr_reader :type_counts, :field_type, :name, :out_of_type_count
 
-    def initialize(name:, stats_to_collect: CORE_STATS, type: ::FlatKit::FieldType::GuessType, guess_threshold: DEFAULT_GUESS_THRESHOLD)
+    def initialize(name:, stats_to_collect: CORE_STATS,
+                   type: ::FlatKit::FieldType::GuessType,
+                   guess_threshold: DEFAULT_GUESS_THRESHOLD)
       @name              = name
       @field_type        = type
       @guess_threshold   = guess_threshold
@@ -56,9 +59,14 @@ module FlatKit
 
       @stats_to_collect.each do |collection_set|
         next if ALL_STATS.include?(collection_set)
-        raise ArgumentError, "#{collection_set} is not a valid stats collection set, must be one of #{ALL_STATS.map { |s| s.to_s }.join(", ") }"
+
+        valid_sets = ALL_STATS.map(&:to_s).join(", ")
+
+        raise ArgumentError, "#{collection_set} is not a valid stats collection set, must be one of #{valid_sets}"
       end
-      raise ArgumentError, "type: must be FieldType subclasses - not #{type}" unless type.kind_of?(Class) && (type.superclass == ::FlatKit::FieldType)
+      return if type.is_a?(Class) && (type.superclass == ::FlatKit::FieldType)
+
+      raise ArgumentError, "type: must be FieldType subclasses - not #{type}"
     end
 
     def field_type_determined?
@@ -68,14 +76,12 @@ module FlatKit
     def update(value)
       update_type_count(value)
 
-      if field_type_determined? then
+      if field_type_determined?
         update_stats(value)
       else
         @values << value
 
-        if @values.size >= @guess_threshold then
-          resolve_guess
-        end
+        resolve_guess if @values.size >= @guess_threshold
       end
     end
 
@@ -167,12 +173,9 @@ module FlatKit
       stats.count + @out_of_type_count
     end
 
-    def out_of_type_count
-      @out_of_type_count
-    end
-
     def null_percent
       return 0 if total_count.zero?
+
       ((null_count.to_f / total_count) * 100.0).truncate(2)
     end
 
@@ -182,15 +185,16 @@ module FlatKit
 
     def unknown_percent
       return 0 if total_count.zero?
+
       ((unknown_count.to_f / total_count) * 100.0).truncate(2)
     end
 
     def to_hash
       resolve_guess
 
-      Hash.new.tap do |h|
+      {}.tap do |h|
         EXPORT_FIELDS.each do |n|
-          h[n] = self.send(n)
+          h[n] = send(n)
         end
       end
     end
@@ -209,7 +213,7 @@ module FlatKit
 
     def update_stats(value)
       coerced_value = @field_type.coerce(value)
-      if coerced_value == FieldType::CoerceFailure then
+      if coerced_value == FieldType::CoerceFailure
         @out_of_type_count += 1
         return
       end
@@ -221,15 +225,16 @@ module FlatKit
     def update_type_count(value)
       guess = FieldType.best_guess(value)
       type_counts[guess] += 1
-      return guess
+      guess
     end
 
     def resolve_guess
       return if field_type_determined?
-      best_guess_type, _best_guess_count = type_counts.max_by { |k, v| v }
+
+      best_guess_type, _best_guess_count = type_counts.max_by { |_k, v| v }
       @field_type = best_guess_type
       @stats = StatType.for(@field_type).new(collecting_frequencies: collecting_frequencies?)
-      if @field_type == ::FlatKit::FieldType::StringType then
+      if @field_type == ::FlatKit::FieldType::StringType
         @length_stats = ::FlatKit::StatType::NumericalStats.new(collecting_frequencies: collecting_frequencies?)
       end
       @values.each do |v|

data/lib/flat_kit/field_type/boolean_type.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module FlatKit
   class FieldType
+    # Internal: Implemenation of the boolean type and coercion to the type
+    #
     class BooleanType < FieldType
-
       TRUTHY_REGEX = /\A(true|t|1|yes|y|on)\Z/i
       FALSEY_REGEX = /\A(false|f|0|no|n|off)\Z/i
       REGEX        = Regexp.union(TRUTHY_REGEX, FALSEY_REGEX)
@@ -12,16 +15,15 @@ module FlatKit
 
       def self.matches?(data)
         case data
-        when TrueClass
-          true
-        when FalseClass
+        when TrueClass, FalseClass
           true
         when String
           REGEX.match?(data)
         when Integer
           return true if data.zero?
           return true if data == 1
-          return false
+
+          false
         else
           false
         end
@@ -36,10 +38,12 @@ module FlatKit
         when Numeric
           return false if data.zero?
           return true  if data == 1
+
           CoerceFailure
         when String
           return true  if TRUTHY_REGEX.match?(data)
           return false if FALSEY_REGEX.match?(data)
+
           CoerceFailure
         end
       end

data/lib/flat_kit/field_type/date_type.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module FlatKit
   class FieldType
-    # Representing the type of data which only includes data up to the day
-    # resolution
+    # Internal: Representing the type of data which only includes data up to
+    # the day resolution
+    #
     class DateType < FieldType
-
       # %Y   4 digit year
       # %y   2 didigt year (%Y mod 100) (00..99)
       # %m   month of year zero padded
@@ -49,9 +51,8 @@ module FlatKit
 
           # other formats
           "%Y-%j",
-          "%a %b %d %Y"
-       ]
-
+          "%a %b %d %Y",
+        ].freeze
       end
 
       # https://en.wikipedia.org/wiki/Date_format_by_country
@@ -146,7 +147,7 @@ module FlatKit
 
       def self.matches?(data)
         coerced = coerce(data)
-        return coerced.kind_of?(Date)
+        coerced.is_a?(Date)
       end
 
       def self.coerce(data)
@@ -156,20 +157,21 @@ module FlatKit
         when Date
           data
         when String
-          coerced_data = CoerceFailure
-          parse_formats.each do |format|
-            begin
-              coerced_data = Date.strptime(data, format)
-              break
-            rescue => _
-              false
-            end
-          end
-          coerced_data
+          try_parse(data)
         else
           CoerceFailure
         end
       end
+
+      def self.try_parse(data)
+        parse_formats.each do |format|
+          coerced_data = Date.strptime(data, format)
+          return coerced_data
+        rescue StandardError => _e
+          false
+        end
+        CoerceFailure
+      end
     end
   end
 end

data/lib/flat_kit/field_type/float_type.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module FlatKit
   class FieldType
+    # Internal: Represeting floating point data and conversion to it
+    #
     class FloatType < FieldType
-
       def self.type_name
         "float"
       end
@@ -14,12 +17,8 @@ module FlatKit
           false
         when String
           return false if IntegerType.matches?(data)
-          begin
-            Float(data)
-            true
-          rescue ArgumentError => _
-            false
-          end
+
+          maybe_float?(data)
         else
           false
         end
@@ -27,11 +26,18 @@ module FlatKit
 
       def self.coerce(data)
         Float(data)
-      rescue TypeError => _
+      rescue TypeError => _e
         CoerceFailure
-      rescue ArgumentError => _
+      rescue ArgumentError => _e
         CoerceFailure
       end
+
+      def self.maybe_float?(data)
+        Float(data)
+        true
+      rescue ArgumentError => _e
+        false
+      end
     end
   end
 end

data/lib/flat_kit/field_type/guess_type.rb

@@ -1,18 +1,21 @@
+# frozen_string_literal: true
+
 module FlatKit
   class FieldType
-    # GuessType is a field type where we don't know what type the field is, and
-    # it needs to be guessed. This is a sentinel type that doesn't match any
-    # data.
+    # Internal: GuessType is a field type where we don't know what type the
+    # field is, and it needs to be guessed. This is a sentinel type that doesn't
+    # match any data.
+    #
     class GuessType < FieldType
       def self.type_name
-        self.name
+        name
       end
 
-      def self.matches?(data)
+      def self.matches?(*)
         false
       end
 
-      def self.coerce(data)
+      def self.coerce(*)
         CoerceFailure
       end
     end

data/lib/flat_kit/field_type/integer_type.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module FlatKit
   class FieldType
+    # Internal: Class reprepseting the Integer type and coercian to it.
+    #
     class IntegerType < FieldType
-
       REGEX = /\A[-+]?\d+\Z/
 
       def self.type_name
@@ -23,12 +26,11 @@ module FlatKit
 
       def self.coerce(data)
         Integer(data)
-      rescue TypeError => _
+      rescue TypeError => _e
         CoerceFailure
-      rescue ArgumentError => _
+      rescue ArgumentError => _e
         CoerceFailure
       end
- 
     end
   end
 end

data/lib/flat_kit/field_type/null_type.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module FlatKit
   class FieldType
+    # Internal: Class reprepseting the null type and coercian to it.
+    #
     class NullType < FieldType
-
       REGEX = Regexp.union(/\A(null|nil)\Z/i, /\A\\N\Z/)
 
       def self.type_name
@@ -25,6 +28,7 @@ module FlatKit
           data
         when String
           return nil if REGEX.match?(data)
+
           CoerceFailure
         else
           CoerceFailure

data/lib/flat_kit/field_type/string_type.rb

@@ -1,20 +1,22 @@
+# frozen_string_literal: true
+
 module FlatKit
   class FieldType
-    # StringType is essentially a fallback - hence its lower weight than other
-    # types that might have string representations.
-    class StringType< FieldType
-
+    # Internal: StringType is essentially a fallback - hence its lower weight
+    # than other types that might have string representations.
+    #
+    class StringType < FieldType
       def self.type_name
         "string"
       end
 
       def self.matches?(data)
-        data.kind_of?(String)
+        data.is_a?(String)
       end
 
       def self.coerce(data)
         data.to_s
-      rescue => _
+      rescue StandardError => _e
         CoerceFailure
       end
     end

data/lib/flat_kit/field_type/timestamp_type.rb

@@ -1,15 +1,18 @@
+# frozen_string_literal: true
+
 module FlatKit
   class FieldType
+    # Internal: Type for all tiemstamps types more granular than Date.
+    #
     class TimestampType < FieldType
-
       def self.parse_formats
-        @timestamp_formats ||= [
+        @parse_formats ||= [
           "%Y-%m-%d %H:%M:%S.%NZ",
           "%Y-%m-%d %H:%M:%S.%N",
           "%Y-%m-%dT%H:%M:%S.%N%z", # w3cdtf
           "%Y-%m-%d %H:%M:%S",
           "%Y-%m-%dT%H:%M:%S%z",
-          "%Y-%m-%dT%H:%M:%SZ", 
+          "%Y-%m-%dT%H:%M:%SZ",
           "%Y%m%dT%H%M%S",
           "%a, %d %b %Y %H:%M:%S %z", # rfc2822, httpdate
         ].freeze
@@ -21,7 +24,7 @@ module FlatKit
 
       def self.matches?(data)
         coerced = coerce(data)
-        return coerced.kind_of?(Time)
+        coerced.is_a?(Time)
       end
 
       def self.coerce(data)
@@ -30,12 +33,10 @@ module FlatKit
           data
         when String
           parse_formats.each do |format|
-            begin
-              coerced_data = Time.strptime(data, format).utc
-              return coerced_data
-            rescue => _
-              # do nothing
-            end
+            coerced_data = Time.strptime(data, format).utc
+            return coerced_data
+          rescue StandardError => _e
+            # do nothing
           end
           CoerceFailure
         else