flat_kit 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90aee146f0054a6afc302716d7bcbac4e013e0ec67bd09d390de326d9d090a27
-  data.tar.gz: b34138e0095b751ff2e29322c3899f7c4df54b5bd9649e2c2c9d550966225acb
+  metadata.gz: 8ee02f6b5e9ed51f565da86c04a0d4600cc0af271e3c8b90f67e22f00ff450fd
+  data.tar.gz: ee58c4ee864c91dc2e11c429891a5c4a7455541f3747f745aa8de0203b6c2142
 SHA512:
-  metadata.gz: 27f23b37c07b0702152904ffe4062269a091005dd112a570f374b902cedb9c489d5802e3d7cac366a275da435a5f1d38b25f3d20e34b8e12f3185330bdd65b0e
-  data.tar.gz: 9f11eda352f16db26ec54793836ff0e5153fb624659989d710789b7afb07a46be284ccbe235e649fb0bcb676327a040fc35f91a63c20c9772cccb3e50517e150
+  metadata.gz: 1cbcb7d15633b06818d0647935475f1e87f9944baa2c0c8450fb9bb54bcad3c4f82a178c5ef3eb43529f05ef61091958f1eee751291dcf45f3632a18c2b0bfeb
+  data.tar.gz: ff78cd8f3e0795da93f50fc796b209a996a10b4fda7524ebf0878895ca467bbb178e2f4bf6da9547d08ba7c5fff1c8791a22c5f4341a59f569af0b22e8d8858c

data/HISTORY.md

@@ -1,4 +1,10 @@
 # FlatKit Changelog
+## Version 0.3.0
+
+* Changing the event listening api to include meta data about the event
+* Add field type detection
+* Add a 'stats' command to generate stats about the data file
+
 ## Version 0.2.0
 
 * add in event listening to allow for additional integrations

data/Manifest.txt

@@ -5,15 +5,28 @@ Manifest.txt
 README.md
 Rakefile
 bin/fk
+examples/stream-active-record-to-csv.rb
 lib/flat_kit.rb
 lib/flat_kit/cli.rb
 lib/flat_kit/command.rb
 lib/flat_kit/command/cat.rb
 lib/flat_kit/command/merge.rb
 lib/flat_kit/command/sort.rb
+lib/flat_kit/command/stats.rb
 lib/flat_kit/descendant_tracker.rb
 lib/flat_kit/error.rb
 lib/flat_kit/event_emitter.rb
+lib/flat_kit/field_stats.rb
+lib/flat_kit/field_type.rb
+lib/flat_kit/field_type/boolean_type.rb
+lib/flat_kit/field_type/date_type.rb
+lib/flat_kit/field_type/float_type.rb
+lib/flat_kit/field_type/guess_type.rb
+lib/flat_kit/field_type/integer_type.rb
+lib/flat_kit/field_type/null_type.rb
+lib/flat_kit/field_type/string_type.rb
+lib/flat_kit/field_type/timestamp_type.rb
+lib/flat_kit/field_type/unknown_type.rb
 lib/flat_kit/format.rb
 lib/flat_kit/input.rb
 lib/flat_kit/input/file.rb
@@ -31,11 +44,17 @@ lib/flat_kit/merge_tree.rb
 lib/flat_kit/output.rb
 lib/flat_kit/output/file.rb
 lib/flat_kit/output/io.rb
+lib/flat_kit/position.rb
 lib/flat_kit/reader.rb
 lib/flat_kit/record.rb
 lib/flat_kit/sentinel_internal_node.rb
 lib/flat_kit/sentinel_leaf_node.rb
 lib/flat_kit/sort.rb
+lib/flat_kit/stat_type.rb
+lib/flat_kit/stat_type/nominal_stats.rb
+lib/flat_kit/stat_type/numerical_stats.rb
+lib/flat_kit/stat_type/ordinal_stats.rb
+lib/flat_kit/stats.rb
 lib/flat_kit/writer.rb
 lib/flat_kit/xsv.rb
 lib/flat_kit/xsv/format.rb
@@ -47,6 +66,15 @@ tasks/extension.rake
 tasks/man.rake
 tasks/this.rb
 test/device_dataset.rb
+test/field_type/test_boolean_type.rb
+test/field_type/test_date_type.rb
+test/field_type/test_float_type.rb
+test/field_type/test_guess_type.rb
+test/field_type/test_integer_type.rb
+test/field_type/test_null_type.rb
+test/field_type/test_string_type.rb
+test/field_type/test_timestamp_type.rb
+test/field_type/test_unknown_type.rb
 test/input/test_file.rb
 test/input/test_io.rb
 test/jsonl/test_format.rb
@@ -55,8 +83,14 @@ test/jsonl/test_record.rb
 test/jsonl/test_writer.rb
 test/output/test_file.rb
 test/output/test_io.rb
+test/run
+test/stat_type/test_nominal_stats.rb
+test/stat_type/test_numerical_stats.rb
+test/stat_type/test_ordinal_stats.rb
 test/test_conversions.rb
 test/test_event_emitter.rb
+test/test_field_stats.rb
+test/test_field_type.rb
 test/test_format.rb
 test/test_helper.rb
 test/test_merge.rb

data/Rakefile

@@ -10,6 +10,7 @@ This.ruby_gemspec do |spec|
   spec.add_dependency('oj', '~> 3.0')
   spec.add_dependency('optimist', '~> 3.0')
 
+  spec.add_development_dependency( 'faker'    , '~> 2.16')
   spec.add_development_dependency( 'rake'     , '~> 13.0')
   spec.add_development_dependency( 'minitest' , '~> 5.11' )
   spec.add_development_dependency( 'minitest-focus' , '~> 1.2' )

data/examples/stream-active-record-to-csv.rb

@@ -0,0 +1,42 @@
+#!/usr/bin/env
+
+#------------------------------------------------------------------------------
+# This is an example to show how to stream an active record scope to a CSV file
+# using FlatKit.
+#------------------------------------------------------------------------------
+
+require 'flat_kit'     # gem 'flat_kit'
+require 'progress_bar' # gem 'progress-bar'
+
+# get an appropriate scope from one of your models - or any scope for that
+# matter
+scope      = MyActiveRecordModel.all
+
+# Output to a file that is csv, and automatically gzipped
+#
+output_csv = ::FlatKit::Xsv::Writer.new(destination: "export.csv.gz")
+
+# handy progress bar
+bar        = ProgressBar.new(scope.count)
+
+# using active record in batches to not pull all the recors from the database at
+# once
+#
+#  https://api.rubyonrails.org/classes/ActiveRecord/Batches.html#method-i-find_each
+scope.find_each do |record|
+
+  # generate an XSV Record by pulling hte attributes out of the active record
+  # model. You may also want to generate a hash from a query or something
+  # along those lines. In any case pass in a Hash to complete_structured_data:
+  # and nil to data.
+  xsv_record = ::FlatKit::Xsv::Record.new(data: nil, complete_structured_data: record.attributes)
+
+  # FlatKit will automatically handle writing out the header line based upon
+  # the fields in the first record.
+  output_csv.write(xsv_record)
+
+  bar.increment!
+end
+
+# close the output file explicitly
+output_csv.close

data/lib/flat_kit.rb

@@ -1,11 +1,14 @@
 module FlatKit
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end
 require 'flat_kit/error'
+require 'flat_kit/descendant_tracker'
 require 'flat_kit/logger'
 require 'flat_kit/event_emitter'
-require 'flat_kit/descendant_tracker'
+
+require 'flat_kit/field_type'
 require 'flat_kit/format'
+require 'flat_kit/position'
 require 'flat_kit/record'
 require 'flat_kit/reader'
 require 'flat_kit/writer'
@@ -16,6 +19,10 @@ require 'flat_kit/xsv'
 require 'flat_kit/jsonl'
 require 'flat_kit/merge'
 require 'flat_kit/sort'
+require 'flat_kit/stats'
+
+require 'flat_kit/stat_type'
+require 'flat_kit/field_stats'
 
 require 'flat_kit/merge_tree'
 require 'flat_kit/internal_node'

data/lib/flat_kit/cli.rb

@@ -72,7 +72,18 @@ module FlatKit
       ::FlatKit.logger.debug argv
 
       command_name  = argv.shift
+      if command_name.downcase == "help" then
+        parser.educate
+        exit 0
+      end
+
       command_klass = FlatKit::Command.for(command_name)
+      if command_klass.nil? then
+        $stdout.puts "ERROR: Unknown command '#{command_name}'"
+        parser.educate
+        exit 0
+      end
+
       command       = command_klass.new(argv: argv, logger: ::FlatKit.logger, env: env)
       command.call
     end

data/lib/flat_kit/command.rb

@@ -51,3 +51,4 @@ end
 require 'flat_kit/command/cat'
 require 'flat_kit/command/merge'
 require 'flat_kit/command/sort'
+require 'flat_kit/command/stats'

data/lib/flat_kit/command/stats.rb

@@ -0,0 +1,94 @@
+module FlatKit
+  class Command
+    class Stats < ::FlatKit::Command
+
+      def self.name
+        "stats"
+      end
+
+      def self.description
+        "Collect and report stats on the inputfile"
+      end
+
+      def self.parser
+        ::Optimist::Parser.new do
+          banner "#{Sort.description}"
+          banner ""
+
+          banner <<~BANNER
+          Given an input file collect basic statistics.
+
+          The statistics can vary based upon the datatype of the field.
+
+          Numeric fields will report the basic count, min, max, mean, standard deviation and sum.
+          Non-numeric fields that are comparable, like dates, will report count, min and max.
+          Other non-numeric fields will only report the count.
+
+          Adding --cardinality will report the count, and frequency of distinct values in the result.
+          This will allow for reporting the median value.
+
+          The fields upon which stats are collected may be selected with the --fields parameter.
+          By default statistics are collected on all fields.
+
+          The flatfile type(s) will be automatically determined by the file name.
+
+          The output can be dumped as a CSV, JSON or a a formated ascii table.
+
+          BANNER
+
+          banner <<~USAGE
+
+          Usage:
+            fk stats --everything file.json
+            fk stats --select surname,given_name file.csv
+            fk stats --select surname,given_name --output-format json file.csv > stats.json
+            fk stats --select field1,field2 --output-format json input.csv
+            fk stats --select field1 file.json.gz -o stats.csv
+            gunzip -c file.json.gz | fk stats --input-format json --output-format text
+
+          USAGE
+
+          banner <<~OPTIONS
+
+          Options:
+
+          OPTIONS
+
+          opt :output, "Send the output to the given path instead of standard out.", default: "<stdout>"
+          opt :input_format, "Input format, csv or json", default: "auto", short: :none
+          opt :output_format, "Output format, csv or json", default: "auto", short: :none
+          opt :select, "The comma separted list of field(s) to report stats on", required: false, type: :string
+          opt :everything, "Show all statistics that are possible", default: false
+          opt :cardinality, "Show the cardinality of the fields, this requires additional memory", default: false
+        end
+      end
+
+      def parse
+        parser = self.class.parser
+        ::Optimist::with_standard_exception_handling(parser) do
+          begin
+            opts = parser.parse(argv)
+            fields = ::FlatKit::Stats::AllFields
+            fields = CSV.parse_line(opts[:select]) if opts[:select]
+
+            stats = [FieldStats::CORE_STATS]
+            stats << FieldStats::CARDINALITY_STATS if opts[:cardinality] || opts[:everything]
+
+            paths = parser.leftovers
+            raise ::Optimist::CommandlineError, "1 and only 1 input file is allowed" if paths.size > 1
+            path = paths.first || "-" # default to stdin
+            @stats = ::FlatKit::Stats.new(input: path, input_fallback: opts[:input_format],
+                                         output: opts[:output], output_fallback: opts[:output_format],
+                                         fields_to_stat: fields, stats_to_collect: stats)
+          rescue ::FlatKit::Error => e
+            raise ::Optimist::CommandlineError, e.message
+          end
+        end
+      end
+
+      def call
+        @stats.call
+      end
+    end
+  end
+end

data/lib/flat_kit/descendant_tracker.rb

@@ -23,5 +23,14 @@ module FlatKit
         child_klass.send(method, *args)
       end
     end
+
+    # 
+    # Find all the children that return truthy from the given method with args
+    #
+    def find_children(method, *args)
+      children.select do |child_klass|
+        child_klass.send(method, *args)
+      end
+    end
   end
 end

data/lib/flat_kit/event_emitter.rb

@@ -20,9 +20,9 @@ module FlatKit
       _listeners.clear
     end
 
-    def notify_listeners(name:, data:)
+    def notify_listeners(name:, data:, meta: nil)
       _listeners.each do |l|
-        l.on_event(name: name, data: data)
+        l.on_event(name: name, data: data, meta: meta)
       end
     end
 

data/lib/flat_kit/field_stats.rb

@@ -0,0 +1,241 @@
+module FlatKit
+  # Collect stats on a single field. We may not know what the field data type is
+  # to start with, so collect a bunch of values until we have the threshold, and
+  # then calculte states based upon the data types determined by the guess
+  #
+  class FieldStats
+    DEFAULT_GUESS_THRESHOLD = 1000
+
+    CORE_STATS = :core
+    CARDINALITY_STATS = :cardinality
+
+    ALL_STATS = [ CORE_STATS, CARDINALITY_STATS ]
+
+    EXPORT_FIELDS = %w[
+      name
+      type
+      count
+      max
+      mean
+      min
+      stddev
+      sum
+      mode
+      unique_count
+
+      max_length
+      mean_length
+      min_length
+      stddev_length
+      mode_length
+      unique_count_lengths
+
+      null_count
+      unknown_count
+      out_of_type_count
+      total_count
+      null_percent
+      unknown_percent
+    ]
+
+
+    attr_reader :type_counts
+    attr_reader :field_type
+    attr_reader :name
+
+    def initialize(name:, stats_to_collect: CORE_STATS, type: ::FlatKit::FieldType::GuessType, guess_threshold: DEFAULT_GUESS_THRESHOLD)
+      @name              = name
+      @field_type        = type
+      @guess_threshold   = guess_threshold
+      @type_counts       = Hash.new(0)
+      @out_of_type_count = 0
+      @values            = []
+      @stats             = nil
+      @length_stats      = nil
+      @stats_to_collect  = [stats_to_collect].flatten
+
+      @stats_to_collect.each do |collection_set|
+        next if ALL_STATS.include?(collection_set)
+        raise ArgumentError, "#{collection_set} is not a valid stats collection set, must be one of #{ALL_STATS.map { |s| s.to_s }.join(", ") }"
+      end
+      raise ArgumentError, "type: must be FieldType subclasses - not #{type}" unless type.kind_of?(Class) && (type.superclass == ::FlatKit::FieldType)
+    end
+
+    def field_type_determined?
+      @field_type != ::FlatKit::FieldType::GuessType
+    end
+
+    def update(value)
+      update_type_count(value)
+
+      if field_type_determined? then
+        update_stats(value)
+      else
+        @values << value
+
+        if @values.size >= @guess_threshold then
+          resolve_guess
+        end
+      end
+    end
+
+    def collecting_frequencies?
+      @stats_to_collect.include?(CARDINALITY_STATS)
+    end
+
+    def type
+      @field_type.type_name
+    end
+
+    def count
+      stats.count
+    end
+
+    def max
+      stats.max if stats.respond_to?(:max)
+    end
+
+    def mean
+      stats.mean if stats.respond_to?(:mean)
+    end
+
+    def min
+      stats.min if stats.respond_to?(:min)
+    end
+
+    def stddev
+      stats.stddev if stats.respond_to?(:stddev)
+    end
+
+    def sum
+      stats.sum if stats.respond_to?(:sum)
+    end
+
+    def mode
+      stats.mode if collecting_frequencies?
+    end
+
+    def unique_count
+      stats.unique_count if collecting_frequencies?
+    end
+
+    def unique_values
+      stats.unique_values if collecting_frequencies?
+    end
+
+    def frequencies
+      stats.frequencies if collecting_frequencies?
+    end
+
+    def min_length
+      length_stats.min if @length_stats
+    end
+
+    def max_length
+      length_stats.max if @length_stats
+    end
+
+    def mean_length
+      length_stats.mean if @length_stats
+    end
+
+    def stddev_length
+      length_stats.stddev if @length_stats
+    end
+
+    def mode_length
+      length_stats.mode if @length_stats && collecting_frequencies?
+    end
+
+    def unique_count_lengths
+      length_stats.unique_count if @length_stats && collecting_frequencies?
+    end
+
+    def unique_values_lengths
+      length_stats.unique_values if @length_stats && collecting_frequencies?
+    end
+
+    def length_frequencies
+      length_stats.frequencies if @length_stats && collecting_frequencies?
+    end
+
+    def null_count
+      type_counts[FieldType::NullType]
+    end
+
+    def total_count
+      stats.count + @out_of_type_count
+    end
+
+    def out_of_type_count
+      @out_of_type_count
+    end
+
+    def null_percent
+      return 0 if total_count.zero?
+      ((null_count.to_f / total_count) * 100.0).truncate(2)
+    end
+
+    def unknown_count
+      type_counts[FieldType::UnknownType]
+    end
+
+    def unknown_percent
+      return 0 if total_count.zero?
+      ((unknown_count.to_f / total_count) * 100.0).truncate(2)
+    end
+
+    def to_hash
+      resolve_guess
+
+      Hash.new.tap do |h|
+        EXPORT_FIELDS.each do |n|
+          h[n] = self.send(n)
+        end
+      end
+    end
+
+    private
+
+    def stats
+      resolve_guess
+      @stats
+    end
+
+    def length_stats
+      resolve_guess
+      @length_stats
+    end
+
+    def update_stats(value)
+      coerced_value = @field_type.coerce(value)
+      if coerced_value == FieldType::CoerceFailure then
+        @out_of_type_count += 1
+        return
+      end
+
+      @stats.update(coerced_value)
+      @length_stats.update(coerced_value.to_s.length) if @length_stats
+    end
+
+    def update_type_count(value)
+      guess = FieldType.best_guess(value)
+      type_counts[guess] += 1
+      return guess
+    end
+
+    def resolve_guess
+      return if field_type_determined?
+      best_guess_type, _best_guess_count = type_counts.max_by { |k, v| v }
+      @field_type = best_guess_type
+      @stats = StatType.for(@field_type).new(collecting_frequencies: collecting_frequencies?)
+      if @field_type == ::FlatKit::FieldType::StringType then
+        @length_stats = ::FlatKit::StatType::NumericalStats.new(collecting_frequencies: collecting_frequencies?)
+      end
+      @values.each do |v|
+        update_stats(v)
+      end
+      @values.clear
+    end
+  end
+end