flat_kit 0.2.0 → 1.0.0

data/lib/flat_kit/command/stats.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module FlatKit
+  class Command
+    # Internal: The implementation of the stats command.
+    #
+    class Stats < ::FlatKit::Command
+      def self.name
+        "stats"
+      end
+
+      def self.description
+        "Collect and report stats on the inputfile"
+      end
+
+      def self.parser
+        ::Optimist::Parser.new do
+          banner Sort.description.to_s
+          banner ""
+
+          banner <<~BANNER
+            Given an input file collect basic statistics.
+
+            The statistics can vary based upon the datatype of the field.
+
+            Numeric fields will report the basic count, min, max, mean, standard deviation and sum.
+            Non-numeric fields that are comparable, like dates, will report count, min and max.
+            Other non-numeric fields will only report the count.
+
+            Adding --cardinality will report the count, and frequency of distinct values in the result.
+            This will allow for reporting the median value.
+
+            The fields upon which stats are collected may be selected with the --fields parameter.
+            By default statistics are collected on all fields.
+
+            The flatfile type(s) will be automatically determined by the file name.
+
+            The output can be dumped as a CSV, JSON or a a formated ascii table.
+
+          BANNER
+
+          banner <<~USAGE
+
+            Usage:
+              fk stats --everything file.json
+              fk stats --select surname,given_name file.csv
+              fk stats --select surname,given_name --output-format json file.csv > stats.json
+              fk stats --select field1,field2 --output-format json input.csv
+              fk stats --select field1 file.json.gz -o stats.csv
+              gunzip -c file.json.gz | fk stats --input-format json --output-format text
+
+          USAGE
+
+          banner <<~OPTIONS
+
+            Options:
+
+          OPTIONS
+
+          opt :output, "Send the output to the given path instead of standard out.", default: "<stdout>"
+          opt :input_format, "Input format, csv or json", default: "auto", short: :none
+          opt :output_format, "Output format, csv or json", default: "auto", short: :none
+          opt :select, "The comma separted list of field(s) to report stats on", required: false, type: :string
+          opt :everything, "Show all statistics that are possible", default: false
+          opt :cardinality, "Show the cardinality of the fields, this requires additional memory", default: false
+        end
+      end
+
+      def parse
+        parser = self.class.parser
+        ::Optimist.with_standard_exception_handling(parser) do
+          opts = parser.parse(argv)
+          fields = ::FlatKit::Stats::AllFields
+          fields = CSV.parse_line(opts[:select]) if opts[:select]
+
+          stats = [FieldStats::CORE_STATS]
+          stats << FieldStats::CARDINALITY_STATS if opts[:cardinality] || opts[:everything]
+
+          paths = parser.leftovers
+          raise ::Optimist::CommandlineError, "1 and only 1 input file is allowed" if paths.size > 1
+
+          path = paths.first || "-" # default to stdin
+          @stats = ::FlatKit::Stats.new(input: path, input_fallback: opts[:input_format],
+                                        output: opts[:output], output_fallback: opts[:output_format],
+                                        fields_to_stat: fields, stats_to_collect: stats)
+        rescue ::FlatKit::Error => e
+          raise ::Optimist::CommandlineError, e.message
+        end
+      end
+
+      def call
+        @stats.call
+      end
+    end
+  end
+end

data/lib/flat_kit/command.rb

@@ -1,13 +1,12 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: The base class for all commands in the CLI
+  #
   class Command
     extend DescendantTracker
 
-    attr_reader :argv
-    attr_reader :env
-    attr_reader :logger
-    attr_reader :opts
-    attr_reader :readers
-    attr_reader :writer
+    attr_reader :argv, :env, :logger, :opts, :readers, :writer
 
     def self.name
       raise NotImplementedError, "#{self.class} must implement #{self.class}.name"
@@ -22,7 +21,7 @@ module FlatKit
     end
 
     def self.names
-      children.map { |c| c.name }
+      children.map(&:name)
     end
 
     def self.for(name)
@@ -48,6 +47,7 @@ module FlatKit
   end
 end
 
-require 'flat_kit/command/cat'
-require 'flat_kit/command/merge'
-require 'flat_kit/command/sort'
+require "flat_kit/command/cat"
+require "flat_kit/command/merge"
+require "flat_kit/command/sort"
+require "flat_kit/command/stats"

data/lib/flat_kit/descendant_tracker.rb

@@ -1,17 +1,20 @@
-require 'set'
+# frozen_string_literal: true
+
+require "set"
 
 module FlatKit
+  # Internal: A module to track descendants of a class
+  #
   module DescendantTracker
     def inherited(klass)
       super
       return unless klass.instance_of?(Class)
-      self.children << klass
+
+      children << klass
     end
 
     def children
-      unless defined? @_children
-        @_children = Set.new
-      end
+      @_children = Set.new unless defined? @_children
       @_children
     end
 
@@ -23,5 +26,14 @@ module FlatKit
         child_klass.send(method, *args)
       end
     end
+
+    #
+    # Find all the children that return truthy from the given method with args
+    #
+    def find_children(method, *args)
+      children.select do |child_klass|
+        child_klass.send(method, *args)
+      end
+    end
   end
 end

data/lib/flat_kit/error.rb

@@ -1,4 +1,8 @@
+# frozen_string_literal: true
+
 module FlatKit
+  # Internal: A Base error class for all FlatKit errors
+  #
   class Error < ::StandardError
     class UnknownFormat < ::FlatKit::Error; end
   end

data/lib/flat_kit/event_emitter.rb

@@ -1,9 +1,12 @@
+# frozen_string_literal: true
+
 module FlatKit
-  # A simplified Observable class for use internally
+  # Internal: A simplified Observable class for use internally
   #
   module EventEmitter
     def add_listener(listener)
       raise ::NoMethodError, "#{listener} does not resond to #on_event" unless listener.respond_to?(:on_event)
+
       self._listeners ||= []
       self._listeners << listener unless _listeners.include?(listener)
     end
@@ -20,14 +23,14 @@ module FlatKit
       _listeners.clear
     end
 
-    def notify_listeners(name:, data:)
+    def notify_listeners(name:, data:, meta: nil)
       _listeners.each do |l|
-        l.on_event(name: name, data: data)
+        l.on_event(name: name, data: data, meta: meta)
       end
     end
 
     def _listeners
-      @_listeners ||= Array.new
+      @_listeners ||= []
     end
   end
 end

data/lib/flat_kit/field_stats.rb

@@ -0,0 +1,246 @@
+# frozen_string_literal: true
+
+module FlatKit
+  # Internal: Collect stats on a single field.
+  #
+  # We may not know what the field data type is to start with, so collect a
+  # bunch of values until we have the threshold, and then calculte states based
+  # upon the data types determined by the guess
+  #
+  class FieldStats
+    DEFAULT_GUESS_THRESHOLD = 1000
+
+    CORE_STATS = :core
+    CARDINALITY_STATS = :cardinality
+
+    ALL_STATS = [CORE_STATS, CARDINALITY_STATS].freeze
+
+    EXPORT_FIELDS = %w[
+      name
+      type
+      count
+      max
+      mean
+      min
+      stddev
+      sum
+      mode
+      unique_count
+
+      max_length
+      mean_length
+      min_length
+      stddev_length
+      mode_length
+      unique_count_lengths
+
+      null_count
+      unknown_count
+      out_of_type_count
+      total_count
+      null_percent
+      unknown_percent
+    ].freeze
+
+    attr_reader :type_counts, :field_type, :name, :out_of_type_count
+
+    def initialize(name:, stats_to_collect: CORE_STATS,
+                   type: ::FlatKit::FieldType::GuessType,
+                   guess_threshold: DEFAULT_GUESS_THRESHOLD)
+      @name              = name
+      @field_type        = type
+      @guess_threshold   = guess_threshold
+      @type_counts       = Hash.new(0)
+      @out_of_type_count = 0
+      @values            = []
+      @stats             = nil
+      @length_stats      = nil
+      @stats_to_collect  = [stats_to_collect].flatten
+
+      @stats_to_collect.each do |collection_set|
+        next if ALL_STATS.include?(collection_set)
+
+        valid_sets = ALL_STATS.map(&:to_s).join(", ")
+
+        raise ArgumentError, "#{collection_set} is not a valid stats collection set, must be one of #{valid_sets}"
+      end
+      return if type.is_a?(Class) && (type.superclass == ::FlatKit::FieldType)
+
+      raise ArgumentError, "type: must be FieldType subclasses - not #{type}"
+    end
+
+    def field_type_determined?
+      @field_type != ::FlatKit::FieldType::GuessType
+    end
+
+    def update(value)
+      update_type_count(value)
+
+      if field_type_determined?
+        update_stats(value)
+      else
+        @values << value
+
+        resolve_guess if @values.size >= @guess_threshold
+      end
+    end
+
+    def collecting_frequencies?
+      @stats_to_collect.include?(CARDINALITY_STATS)
+    end
+
+    def type
+      @field_type.type_name
+    end
+
+    def count
+      stats.count
+    end
+
+    def max
+      stats.max if stats.respond_to?(:max)
+    end
+
+    def mean
+      stats.mean if stats.respond_to?(:mean)
+    end
+
+    def min
+      stats.min if stats.respond_to?(:min)
+    end
+
+    def stddev
+      stats.stddev if stats.respond_to?(:stddev)
+    end
+
+    def sum
+      stats.sum if stats.respond_to?(:sum)
+    end
+
+    def mode
+      stats.mode if collecting_frequencies?
+    end
+
+    def unique_count
+      stats.unique_count if collecting_frequencies?
+    end
+
+    def unique_values
+      stats.unique_values if collecting_frequencies?
+    end
+
+    def frequencies
+      stats.frequencies if collecting_frequencies?
+    end
+
+    def min_length
+      length_stats.min if @length_stats
+    end
+
+    def max_length
+      length_stats.max if @length_stats
+    end
+
+    def mean_length
+      length_stats.mean if @length_stats
+    end
+
+    def stddev_length
+      length_stats.stddev if @length_stats
+    end
+
+    def mode_length
+      length_stats.mode if @length_stats && collecting_frequencies?
+    end
+
+    def unique_count_lengths
+      length_stats.unique_count if @length_stats && collecting_frequencies?
+    end
+
+    def unique_values_lengths
+      length_stats.unique_values if @length_stats && collecting_frequencies?
+    end
+
+    def length_frequencies
+      length_stats.frequencies if @length_stats && collecting_frequencies?
+    end
+
+    def null_count
+      type_counts[FieldType::NullType]
+    end
+
+    def total_count
+      stats.count + @out_of_type_count
+    end
+
+    def null_percent
+      return 0 if total_count.zero?
+
+      ((null_count.to_f / total_count) * 100.0).truncate(2)
+    end
+
+    def unknown_count
+      type_counts[FieldType::UnknownType]
+    end
+
+    def unknown_percent
+      return 0 if total_count.zero?
+
+      ((unknown_count.to_f / total_count) * 100.0).truncate(2)
+    end
+
+    def to_hash
+      resolve_guess
+
+      {}.tap do |h|
+        EXPORT_FIELDS.each do |n|
+          h[n] = send(n)
+        end
+      end
+    end
+
+    private
+
+    def stats
+      resolve_guess
+      @stats
+    end
+
+    def length_stats
+      resolve_guess
+      @length_stats
+    end
+
+    def update_stats(value)
+      coerced_value = @field_type.coerce(value)
+      if coerced_value == FieldType::CoerceFailure
+        @out_of_type_count += 1
+        return
+      end
+
+      @stats.update(coerced_value)
+      @length_stats.update(coerced_value.to_s.length) if @length_stats
+    end
+
+    def update_type_count(value)
+      guess = FieldType.best_guess(value)
+      type_counts[guess] += 1
+      guess
+    end
+
+    def resolve_guess
+      return if field_type_determined?
+
+      best_guess_type, _best_guess_count = type_counts.max_by { |_k, v| v }
+      @field_type = best_guess_type
+      @stats = StatType.for(@field_type).new(collecting_frequencies: collecting_frequencies?)
+      if @field_type == ::FlatKit::FieldType::StringType
+        @length_stats = ::FlatKit::StatType::NumericalStats.new(collecting_frequencies: collecting_frequencies?)
+      end
+      @values.each do |v|
+        update_stats(v)
+      end
+      @values.clear
+    end
+  end
+end

data/lib/flat_kit/field_type/boolean_type.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module FlatKit
+  class FieldType
+    # Internal: Implemenation of the boolean type and coercion to the type
+    #
+    class BooleanType < FieldType
+      TRUTHY_REGEX = /\A(true|t|1|yes|y|on)\Z/i
+      FALSEY_REGEX = /\A(false|f|0|no|n|off)\Z/i
+      REGEX        = Regexp.union(TRUTHY_REGEX, FALSEY_REGEX)
+
+      def self.type_name
+        "boolean"
+      end
+
+      def self.matches?(data)
+        case data
+        when TrueClass, FalseClass
+          true
+        when String
+          REGEX.match?(data)
+        when Integer
+          return true if data.zero?
+          return true if data == 1
+
+          false
+        else
+          false
+        end
+      end
+
+      def self.coerce(data)
+        case data
+        when TrueClass
+          true
+        when FalseClass
+          false
+        when Numeric
+          return false if data.zero?
+          return true  if data == 1
+
+          CoerceFailure
+        when String
+          return true  if TRUTHY_REGEX.match?(data)
+          return false if FALSEY_REGEX.match?(data)
+
+          CoerceFailure
+        end
+      end
+    end
+  end
+end

data/lib/flat_kit/field_type/date_type.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+module FlatKit
+  class FieldType
+    # Internal: Representing the type of data which only includes data up to
+    # the day resolution
+    #
+    class DateType < FieldType
+      # %Y   4 digit year
+      # %y   2 didigt year (%Y mod 100) (00..99)
+      # %m   month of year zero padded
+      # %-m  month of year no-padding
+      # %B   Full month name
+      # %b   Abbreviated month name
+      # %^b  uppercased month name
+      # %d   day of month zero padded
+      # %-d  day of moneth not padded
+      # %e   day of month blank padded
+      # %j   day of year zero padded
+
+      # parse formats are not the same as print formats as parsing does not deal
+      # with flags and widths
+      def self.parse_formats
+        @parse_formats ||= [
+          # YMD formats
+          "%Y-%m-%d",
+          "%Y%m%d",
+          "%Y/%m/%d",
+          "%Y %m %d.",
+
+          # DMY formats
+          "%d %B %Y",
+          "%d %b %Y",
+          "%d-%b-%Y",
+          "%d/%b/%Y",
+          "%d-%m-%Y",
+          "%d-%m-%y",
+          "%d %b, %Y",
+          "%d %b,%Y",
+          "%d %B, %Y",
+          "%d %B,%Y",
+
+          # MDY formats
+          "%m/%d/%Y",
+          "%m-%d-%Y",
+          "%m/%d/%y",
+          "%m-%d-%y",
+
+          "%B %d, %Y",
+          "%b %d, %Y",
+
+          # other formats
+          "%Y-%j",
+          "%a %b %d %Y",
+        ].freeze
+      end
+
+      # https://en.wikipedia.org/wiki/Date_format_by_country
+      # List of formats culled from the above - not using all as it is
+      # definitely a performance issue at the moment
+      # def self.known_formats
+      #   @known_formats ||= [
+      #     # YMD formats
+      #     "%Y-%m-%d",
+      #     "%Y%m%d",
+      #     "%Y/%m/%d",
+      #     "%Y.%m.%d",
+      #     "%Y.%m.%d.",
+      #     "%Y %m %d.",
+      #     "%Y %b %d",
+      #     "%Y %b %-d",
+      #     "%Y %B %-d",
+      #     "%Y %B %d",
+      #     "%Y-%m%d",
+      #     "%Y. %m. %-d.",
+      #     "%Y. %m. %d.",
+      #     "%Y.%-m.%-d.",
+      #     "%Y.%-m.%-d",
+      #     "%Y, %d %B",
+      #     "%Y, %d %b",
+      #
+      #     "%y.%-m.%-d",
+      #     "%y.%-m.%-d.",
+      #     "%y.%m.%d.",
+      #     "%y.%m.%d",
+      #     "%y/%m/%d",
+      #
+      #     # DMY formats
+      #     "%-d %b %Y",
+      #     "%-d %B %Y",
+      #     "%-d-%-m-%Y",
+      #     "%-d. %-m. %Y",
+      #     "%-d. %-m. %Y.",
+      #     "%-d. %B %Y",
+      #     "%-d. %B %Y.",
+      #     "%-d.%-m.%Y",
+      #     "%-d.%-m.%Y.",
+      #     "%-d.%m.%Y.",
+      #     "%-d.%m.%Y",
+      #     "%-d.%b.%Y",
+      #     "%-d.%B.%Y",
+      #     "%-d/%-m %Y",
+      #     "%-d/%-m/%Y",
+      #     "%d %B %Y",
+      #     "%d %b %Y",
+      #     "%d-%m-%Y",
+      #     "%d-%b-%Y",
+      #     "%d-%B-%Y",
+      #     "%d.%m.%Y",
+      #     "%d/%m %Y",
+      #     "%d/%m/%Y",
+      #
+      #     "%-d.%b.%y",
+      #     "%-d.%B.%y",
+      #     "%-d.%-m.%y",
+      #     "%-d/%-m-%y",
+      #     "%-d/%-m/%y",
+      #     "%d/%m/%y",
+      #     "%d-%m-%y",
+      #     "%d.%m.%y",
+      #     "%d%m%y",
+      #
+      #     # MDY formats
+      #     "%-m/%-d/%Y",
+      #     "%m/%d/%Y",
+      #     "%m-%d-%Y",
+      #     "%b-%d-%Y",
+      #     "%B %-d, %Y",
+      #     "%B %-d. %Y",
+      #     "%B %d, %Y",
+      #     "%B-%d-%Y",
+      #     "%B/%d/%Y",
+      #
+      #     "%-m/%-d/%y",
+      #
+      #     # other formats
+      #     "%Y-%j",
+      #     "%Y%m",
+      #     "%Y-%m",
+      #     "%Y %m",
+      #   ]
+      # end
+
+      def self.type_name
+        "date"
+      end
+
+      def self.matches?(data)
+        coerced = coerce(data)
+        coerced.is_a?(Date)
+      end
+
+      def self.coerce(data)
+        case data
+        when DateTime
+          CoerceFailure
+        when Date
+          data
+        when String
+          try_parse(data)
+        else
+          CoerceFailure
+        end
+      end
+
+      def self.try_parse(data)
+        parse_formats.each do |format|
+          coerced_data = Date.strptime(data, format)
+          return coerced_data
+        rescue StandardError => _e
+          false
+        end
+        CoerceFailure
+      end
+    end
+  end
+end
+
+
+__END__
+