data_miner 2.1.0 → 2.1.1

data/CHANGELOG

@@ -1,3 +1,10 @@
+2.1.1 / 2012-05-16
+
+* Enhancements
+
+  * Keep row counts from before and after runs.
+  * If DataMiner.per_column_statistics?, keep per-column statistics from before and after runs.
+
 2.1.0 / 2012-05-10
 
 * Enhancements

data/lib/data_miner.rb

@@ -96,6 +96,16 @@ class DataMiner
     end
   end
 
+  # Whether per-column stats like max, min, average, standard deviation, etc are enabled.
+  def per_column_statistics?
+    @per_column_statistics == true
+  end
+
+  # Turn on or off per-column stats.
+  def per_column_statistics=(boolean)
+    @per_column_statistics = boolean
+  end
+
   class << self
     delegate(*DataMiner.instance_methods(false), :to => :instance)
   end

data/lib/data_miner/run.rb

@@ -1,6 +1,8 @@
 require 'aasm'
 require 'active_record_inline_schema'
 
+require 'data_miner/run/column_statistic'
+
 class DataMiner
   # A record of what happened when you ran a data miner script.
   #
@@ -57,6 +59,14 @@ class DataMiner
     col :stopped_at, :type => :datetime
     col :updated_at, :type => :datetime
     col :error, :type => :text
+    col :row_count_before, :type => :integer
+    col :row_count_after, :type => :integer
+    add_index :model_name
+    add_index :aasm_state
+
+    validates_presence_of :model_name
+
+    has_many :column_statistics, :class_name => 'DataMiner::Run::ColumnStatistic'
 
     include ::AASM
     aasm_initial_state INITIAL_STATE
@@ -68,11 +78,16 @@ class DataMiner
     aasm_event(:skip)    { transitions :from => :limbo, :to => :skipped }
     aasm_event(:fail)    { transitions :from => :limbo, :to => :failed }
 
-    validates_presence_of :model_name
-
     # @private
     def start
+      model = model_name.constantize
+      if model.table_exists?
+        self.row_count_before = model.count
+      end
       save!
+      if DataMiner.per_column_statistics?
+        ColumnStatistic.before self
+      end
       begin
         catch :data_miner_succeed do
           yield
@@ -85,6 +100,10 @@ class DataMiner
         fail!
         raise $!
       ensure
+        self.row_count_after = model.count
+        if DataMiner.per_column_statistics?
+          ColumnStatistic.after self
+        end
         self.stopped_at = ::Time.now
         save!
         DataMiner.logger.info %{[data_miner] #{model_name} #{aasm_current_state.to_s.upcase} (#{(stopped_at-created_at).round(2)}s)}
@@ -92,6 +111,31 @@ class DataMiner
       self
     end
 
+    # Get the column statistics for a particular column before or after this run.
+    #
+    # @param [String] column_name The column you want to know about.
+    # @param ["before","after"] period Whether you want to know about before or after the run.
+    #
+    # @return [ColumnStatistic]
+    def column_statistics_for(column_name, period)
+      column_name = column_name.to_s
+      period = period.to_s
+      model = model_name.constantize
+      if existing = column_statistics.where(:column_name => column_name, :period => period).first
+        existing
+      elsif model.table_exists?
+        unless model.column_names.include?(column_name)
+          raise ::ArgumentError, %{[data_miner] Nonexistent column #{column_name.inspect} on #{model_name}}
+        end
+        blank = ColumnStatistic.new
+        blank.run = self
+        blank.model_name = model_name
+        blank.period = period
+        blank.column_name = column_name
+        blank
+      end
+    end
+
     # @private
     def as_lock
       database_name = Run.connection.instance_variable_get(:@config).try(:[], :database)

data/lib/data_miner/run/column_statistic.rb

@@ -0,0 +1,87 @@
+class DataMiner
+  class Run < ::ActiveRecord::Base
+    # If +DataMiner.per_column_statistics?+, this model keeps per-column stats like max, min, average, standard deviation, etc.
+    #
+    # Each +DataMiner::Run+ will have two of these for every column; a "before" and an "after".
+    class ColumnStatistic < ::ActiveRecord::Base
+      class << self
+        # @private
+        def before(run)
+          period run, 'before'
+        end
+
+        # @private
+        def after(run)
+          period run, 'after'
+        end
+
+        private
+
+        def period(run, period)
+          unless table_exists?
+            auto_upgrade!
+          end
+          model = run.model_name.constantize
+          return unless model.table_exists?
+          model.column_names.each do |column_name|
+            column_statistic = new
+            column_statistic.run = run
+            column_statistic.model_name = run.model_name
+            column_statistic.period = period
+            column_statistic.column_name = column_name
+            column_statistic.perform_calculations
+            column_statistic.save!
+          end
+          nil
+        end
+
+      end
+
+      NUMERIC = [
+        :integer,
+        :float,
+        :decimal,
+      ]
+
+      self.table_name = 'data_miner_run_column_statistics'
+
+      belongs_to :run, :class_name => 'DataMiner::Run'
+
+      col :run_id, :type => :integer
+      col :model_name
+      col :period
+      col :column_name
+      col :null_count, :type => :integer
+      col :max
+      col :min
+      col :average, :type => :float
+      col :standard_deviation, :type => :float
+      col :sum, :type => :float
+      add_index :run_id
+      add_index :model_name
+
+      # @private
+      def perform_calculations
+        model = run.model_name.constantize
+
+        self.null_count = model.where("#{model.connection.quote_column_name(column_name)} IS NULL").count
+        self.max = calculate(:MAX).inspect
+        self.min = calculate(:MIN).inspect
+
+        column = model.columns_hash[column_name]
+        if NUMERIC.include?(column.type)
+          self.average = calculate :AVG
+          self.standard_deviation = calculate :STDDEV
+          self.sum = calculate :SUM
+        end
+      end
+
+      private
+
+      def calculate(operation)
+        model = run.model_name.constantize
+        model.connection.select_value "SELECT #{operation}(#{model.connection.quote_column_name(column_name)}) FROM #{model.quoted_table_name}"
+      end
+    end
+  end
+end

data/lib/data_miner/script.rb

@@ -213,6 +213,9 @@ class DataMiner
         Script.current_stack.clear
       end
       Script.current_stack << model_name
+      unless Run.table_exists?
+        Run.auto_upgrade!
+      end
       run = Run.new
       run.model_name = model_name
       run.start do

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '2.1.0'
+  VERSION = '2.1.1'
 end

data/test/helper.rb

@@ -11,7 +11,7 @@ require 'minitest/reporters'
 MiniTest::Unit.runner = MiniTest::SuiteRunner.new
 MiniTest::Unit.runner.reporters << MiniTest::Reporters::SpecReporter.new
 
-cmd = %{mysql -u root -ppassword -e "drop database data_miner_test; create database data_miner_test charset utf8"}
+cmd = %{mysql -u root -ppassword -e "DROP DATABASE data_miner_test; CREATE DATABASE data_miner_test CHARSET utf8"}
 $stderr.puts "Running `#{cmd}`..."
 system cmd
 $stderr.puts "Done."
@@ -28,6 +28,71 @@ ActiveRecord::Base.establish_connection(
   'password' => 'password'
 )
 
+ActiveRecord::Base.mass_assignment_sanitizer = :strict
+
 require 'data_miner'
 DataMiner::Run.auto_upgrade!
+DataMiner::Run::ColumnStatistic.auto_upgrade!
 DataMiner::Run.clear_locks
+
+PETS = File.expand_path('../support/pets.csv', __FILE__)
+PETS_FUNNY = File.expand_path('../support/pets_funny.csv', __FILE__)
+COLOR_DICTIONARY_ENGLISH = File.expand_path('../support/pet_color_dictionary.en.csv', __FILE__)
+COLOR_DICTIONARY_SPANISH = File.expand_path('../support/pet_color_dictionary.es.csv', __FILE__)
+BREEDS = File.expand_path('../support/breeds.xls', __FILE__)
+
+class Pet < ActiveRecord::Base
+  self.primary_key = "name"
+  col :name
+  col :breed_id
+  col :color_id
+  col :age, :type => :integer
+  col :age_units
+  col :weight, :type => :float
+  col :weight_units
+  col :height, :type => :integer
+  col :height_units
+  col :favorite_food
+  col :command_phrase
+  belongs_to :breed
+  data_miner do
+    process :auto_upgrade!
+    process :run_data_miner_on_parent_associations!
+    import("A list of pets", :url => "file://#{PETS}") do
+      key :name
+      store :age, :units_field_name => 'age_units'
+      store :breed_id, :field_name => :breed, :nullify_blank_strings => true
+      store :color_id, :field_name => :color, :dictionary => { :url => "file://#{COLOR_DICTIONARY_ENGLISH}", :input => :input, :output => :output }
+      store :weight, :from_units => :pounds, :to_units => :kilograms
+      store :favorite_food, :nullify_blank_strings => true
+      store :command_phrase
+      store :height, :units => :centimetres
+    end
+  end
+end
+
+class Breed < ActiveRecord::Base
+  class << self
+    def update_average_age!
+      # make sure pet is populated
+      Pet.run_data_miner!
+      update_all %{breeds.average_age = (SELECT AVG(pets.age) FROM pets WHERE pets.breed_id = breeds.name)}
+    end
+  end
+  self.primary_key = "name"
+  col :name
+  col :average_age, :type => :float
+  data_miner do
+    process :auto_upgrade!
+    import("A list of breeds", :url => "file://#{BREEDS}") do
+      key :name, :field_name => 'Breed name'
+    end
+    process :update_average_age!
+  end
+end
+
+ActiveRecord::Base.descendants.each do |model|
+  model.attr_accessible nil
+end
+
+Pet.auto_upgrade!

data/test/support/pets.csv

@@ -3,3 +3,4 @@ Pierre,Tabby,GO,4,years,4.4,30,tomato,"eh"
 Jerry,Beagle,BR/BL,5,years,10,30,cheese,"che"
 Amigo,Spanish Lizarto,GR/BU,17,years,"   ",3,crickets,"    "
 Johnny,Beagle,BR/BL,2,years,20,45,"   ",
+Nemo,,,,,,,,

data/test/test_data_miner.rb

@@ -1,73 +1,12 @@
 # -*- encoding: utf-8 -*-
 require 'helper'
 
-PETS = File.expand_path('../support/pets.csv', __FILE__)
-PETS_FUNNY = File.expand_path('../support/pets_funny.csv', __FILE__)
-COLOR_DICTIONARY_ENGLISH = File.expand_path('../support/pet_color_dictionary.en.csv', __FILE__)
-COLOR_DICTIONARY_SPANISH = File.expand_path('../support/pet_color_dictionary.es.csv', __FILE__)
-BREEDS = File.expand_path('../support/breeds.xls', __FILE__)
-
-class Pet < ActiveRecord::Base
-  self.primary_key = "name"
-  col :name
-  col :breed_id
-  col :color_id
-  col :age, :type => :integer
-  col :age_units
-  col :weight, :type => :float
-  col :weight_units
-  col :height, :type => :integer
-  col :height_units
-  col :favorite_food
-  col :command_phrase
-  belongs_to :breed
-  data_miner do
-    process :auto_upgrade!
-    process :run_data_miner_on_parent_associations!
-    import("A list of pets", :url => "file://#{PETS}") do
-      key :name
-      store :age, :units_field_name => 'age_units'
-      store :breed_id, :field_name => :breed
-      store :color_id, :field_name => :color, :dictionary => { :url => "file://#{COLOR_DICTIONARY_ENGLISH}", :input => :input, :output => :output }
-      store :weight, :from_units => :pounds, :to_units => :kilograms
-      store :favorite_food, :nullify_blank_strings => true
-      store :command_phrase
-      store :height, :units => :centimetres
-    end
-  end
-end
-
-class Breed < ActiveRecord::Base
-  class << self
-    def update_average_age!
-      # make sure pet is populated
-      Pet.run_data_miner!
-      update_all %{breeds.average_age = (SELECT AVG(pets.age) FROM pets WHERE pets.breed_id = breeds.name)}
-    end
-  end
-  self.primary_key = "name"
-  col :name
-  col :average_age, :type => :float
-  data_miner do
-    process :auto_upgrade!
-    import("A list of breeds", :url => "file://#{BREEDS}") do
-      key :name, :field_name => 'Breed name'
-    end
-    process :update_average_age!
-  end
-end
-
-ActiveRecord::Base.mass_assignment_sanitizer = :strict
-ActiveRecord::Base.descendants.each do |model|
-  model.attr_accessible nil
-end
-
-Pet.auto_upgrade!
-
 describe DataMiner do
   describe "when used to import example data about pets" do
     before do
       Pet.delete_all
+      DataMiner::Run.delete_all
+      DataMiner::Run::ColumnStatistic.delete_all
     end
     it "it does not depend on mass-assignment" do
       lambda do
@@ -155,5 +94,10 @@ describe DataMiner do
       Pet.find('Amigo').weight.must_be_nil
       Pet.find('Amigo').weight_units.must_be_nil
     end
+    it "keeps a row count before and after" do
+      Pet.run_data_miner!
+      Pet.data_miner_runs.first.row_count_before.must_equal 0
+      Pet.data_miner_runs.first.row_count_after.must_equal 5
+    end
   end
 end

data/test/test_data_miner_run_column_statistic.rb

@@ -0,0 +1,43 @@
+# -*- encoding: utf-8 -*-
+require 'helper'
+
+describe DataMiner::Run::ColumnStatistic do
+  describe "when advanced statistics are enabled" do
+    before do
+      DataMiner.per_column_statistics = true
+      Pet.delete_all
+      DataMiner::Run.delete_all
+      DataMiner::Run::ColumnStatistic.delete_all
+    end
+
+    after do
+      DataMiner.per_column_statistics = false
+    end
+
+    it "keeps null count" do
+      Pet.run_data_miner!
+
+      Pet.data_miner_runs.first.column_statistics_for(:breed_id, :before).null_count.must_equal 0
+      Pet.data_miner_runs.first.column_statistics_for(:breed_id, :after).null_count.must_equal 1
+
+      Pet.data_miner_runs.first.column_statistics_for(:command_phrase, :before).null_count.must_equal 0
+      Pet.data_miner_runs.first.column_statistics_for(:command_phrase, :after).null_count.must_equal 0
+    end
+
+    it "keeps max and min (as strings)" do
+      Pet.run_data_miner!
+      Pet.data_miner_runs.first.column_statistics_for(:age, :before).max.must_equal 'nil'
+      Pet.data_miner_runs.first.column_statistics_for(:age, :after).max.must_equal '17'
+    end
+
+    it "keeps average and stddev" do
+      Pet.run_data_miner!
+
+      Pet.data_miner_runs.first.column_statistics_for(:age, :before).average.must_be_nil
+      Pet.data_miner_runs.first.column_statistics_for(:age, :after).average.must_equal 7.0
+
+      Pet.data_miner_runs.first.column_statistics_for(:age, :before).standard_deviation.must_be_nil
+      Pet.data_miner_runs.first.column_statistics_for(:age, :after).standard_deviation.must_equal 5.8737
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_miner
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-10 00:00:00.000000000 Z
+date: 2012-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: remote_table
@@ -147,6 +147,7 @@ files:
 - lib/data_miner/attribute.rb
 - lib/data_miner/dictionary.rb
 - lib/data_miner/run.rb
+- lib/data_miner/run/column_statistic.rb
 - lib/data_miner/script.rb
 - lib/data_miner/step.rb
 - lib/data_miner/step/import.rb
@@ -160,6 +161,7 @@ files:
 - test/support/pets.csv
 - test/support/pets_funny.csv
 - test/test_data_miner.rb
+- test/test_data_miner_run_column_statistic.rb
 - test/test_earth_import.rb
 - test/test_earth_tap.rb
 - test/test_safety.rb
@@ -196,6 +198,7 @@ test_files:
 - test/support/pets.csv
 - test/support/pets_funny.csv
 - test/test_data_miner.rb
+- test/test_data_miner_run_column_statistic.rb
 - test/test_earth_import.rb
 - test/test_earth_tap.rb
 - test/test_safety.rb