data_miner 2.1.1 → 2.1.2

data/CHANGELOG

@@ -1,3 +1,14 @@
+2.1.2 / 2012-05-22
+
+* Breaking changes
+
+  * DataMiner::Run#column_statistics_for(column_name, period) -> {initial,final}_column_statistics(column_name)
+  * Remove std dev from statistics kept
+
+* Enhancements
+
+  * Added zero count and null count
+
 2.1.1 / 2012-05-16
 
 * Enhancements

data/lib/data_miner/run/column_statistic.rb

@@ -2,22 +2,10 @@ class DataMiner
   class Run < ::ActiveRecord::Base
     # If +DataMiner.per_column_statistics?+, this model keeps per-column stats like max, min, average, standard deviation, etc.
     #
-    # Each +DataMiner::Run+ will have two of these for every column; a "before" and an "after".
+    # Each +DataMiner::Run+ will have two of these for every column; an "initial" and a "final"
     class ColumnStatistic < ::ActiveRecord::Base
       class << self
-        # @private
-        def before(run)
-          period run, 'before'
-        end
-
-        # @private
-        def after(run)
-          period run, 'after'
-        end
-
-        private
-
-        def period(run, period)
+        def take(run)
           unless table_exists?
             auto_upgrade!
           end
@@ -27,9 +15,8 @@ class DataMiner
             column_statistic = new
             column_statistic.run = run
             column_statistic.model_name = run.model_name
-            column_statistic.period = period
             column_statistic.column_name = column_name
-            column_statistic.perform_calculations
+            column_statistic.take_statistics
             column_statistic.save!
           end
           nil
@@ -49,36 +36,40 @@ class DataMiner
 
       col :run_id, :type => :integer
       col :model_name
-      col :period
       col :column_name
       col :null_count, :type => :integer
+      col :zero_count, :type => :integer
+      col :blank_count, :type => :integer
       col :max
       col :min
       col :average, :type => :float
-      col :standard_deviation, :type => :float
       col :sum, :type => :float
+      col :created_at, :type => :datetime
       add_index :run_id
       add_index :model_name
 
       # @private
-      def perform_calculations
+      def take_statistics
         model = run.model_name.constantize
 
         self.null_count = model.where("#{model.connection.quote_column_name(column_name)} IS NULL").count
-        self.max = calculate(:MAX).inspect
-        self.min = calculate(:MIN).inspect
+        
+        self.max = calc(:MAX).inspect
+        self.min = calc(:MIN).inspect
 
         column = model.columns_hash[column_name]
         if NUMERIC.include?(column.type)
-          self.average = calculate :AVG
-          self.standard_deviation = calculate :STDDEV
-          self.sum = calculate :SUM
+          self.zero_count = model.where(column_name => 0).count
+          self.average = calc :AVG
+          self.sum = calc :SUM
+        elsif column.type == :string
+          self.blank_count = model.where("LENGTH(TRIM(#{model.connection.quote_column_name(column_name)})) = 0").count
         end
       end
 
       private
 
-      def calculate(operation)
+      def calc(operation)
         model = run.model_name.constantize
         model.connection.select_value "SELECT #{operation}(#{model.connection.quote_column_name(column_name)}) FROM #{model.quoted_table_name}"
       end

data/lib/data_miner/run.rb

@@ -86,7 +86,7 @@ class DataMiner
       end
       save!
       if DataMiner.per_column_statistics?
-        ColumnStatistic.before self
+        ColumnStatistic.take self
       end
       begin
         catch :data_miner_succeed do
@@ -102,38 +102,31 @@ class DataMiner
       ensure
         self.row_count_after = model.count
         if DataMiner.per_column_statistics?
-          ColumnStatistic.after self
+          ColumnStatistic.take self
         end
-        self.stopped_at = ::Time.now
+        self.stopped_at = ::Time.now.utc
         save!
         DataMiner.logger.info %{[data_miner] #{model_name} #{aasm_current_state.to_s.upcase} (#{(stopped_at-created_at).round(2)}s)}
       end
       self
     end
 
-    # Get the column statistics for a particular column before or after this run.
+    # Get the column statistics for a particular column before this run started.
     #
     # @param [String] column_name The column you want to know about.
-    # @param ["before","after"] period Whether you want to know about before or after the run.
     #
     # @return [ColumnStatistic]
-    def column_statistics_for(column_name, period)
-      column_name = column_name.to_s
-      period = period.to_s
-      model = model_name.constantize
-      if existing = column_statistics.where(:column_name => column_name, :period => period).first
-        existing
-      elsif model.table_exists?
-        unless model.column_names.include?(column_name)
-          raise ::ArgumentError, %{[data_miner] Nonexistent column #{column_name.inspect} on #{model_name}}
-        end
-        blank = ColumnStatistic.new
-        blank.run = self
-        blank.model_name = model_name
-        blank.period = period
-        blank.column_name = column_name
-        blank
-      end
+    def initial_column_statistics(column_name)
+      column_statistics.where(:column_name => column_name.to_s).first
+    end
+
+    # Get the column statistics for a particular column after this run finished.
+    #
+    # @param [String] column_name The column you want to know about.
+    #
+    # @return [ColumnStatistic]
+    def final_column_statistics(column_name)
+      column_statistics.where(:column_name => column_name.to_s).last
     end
 
     # @private

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '2.1.1'
+  VERSION = '2.1.2'
 end

data/test/test_data_miner_run_column_statistic.rb

@@ -8,6 +8,7 @@ describe DataMiner::Run::ColumnStatistic do
       Pet.delete_all
       DataMiner::Run.delete_all
       DataMiner::Run::ColumnStatistic.delete_all
+      Pet.run_data_miner!
     end
 
     after do
@@ -15,29 +16,35 @@ describe DataMiner::Run::ColumnStatistic do
     end
 
     it "keeps null count" do
-      Pet.run_data_miner!
-
-      Pet.data_miner_runs.first.column_statistics_for(:breed_id, :before).null_count.must_equal 0
-      Pet.data_miner_runs.first.column_statistics_for(:breed_id, :after).null_count.must_equal 1
+      Pet.data_miner_runs.first.initial_column_statistics(:breed_id).null_count.must_equal 0
+      Pet.data_miner_runs.first.final_column_statistics(:breed_id).null_count.must_equal 1
 
-      Pet.data_miner_runs.first.column_statistics_for(:command_phrase, :before).null_count.must_equal 0
-      Pet.data_miner_runs.first.column_statistics_for(:command_phrase, :after).null_count.must_equal 0
+      Pet.data_miner_runs.first.initial_column_statistics(:command_phrase).null_count.must_equal 0
+      Pet.data_miner_runs.first.final_column_statistics(:command_phrase).null_count.must_equal 0
     end
 
     it "keeps max and min (as strings)" do
-      Pet.run_data_miner!
-      Pet.data_miner_runs.first.column_statistics_for(:age, :before).max.must_equal 'nil'
-      Pet.data_miner_runs.first.column_statistics_for(:age, :after).max.must_equal '17'
+      Pet.data_miner_runs.first.initial_column_statistics(:age).max.must_equal 'nil'
+      Pet.data_miner_runs.first.final_column_statistics(:age).max.must_equal '17'
     end
 
-    it "keeps average and stddev" do
-      Pet.run_data_miner!
+    it "keeps average and sum" do
+      Pet.data_miner_runs.first.initial_column_statistics(:age).average.must_be_nil
+      Pet.data_miner_runs.first.final_column_statistics(:age).average.must_equal 7.0
 
-      Pet.data_miner_runs.first.column_statistics_for(:age, :before).average.must_be_nil
-      Pet.data_miner_runs.first.column_statistics_for(:age, :after).average.must_equal 7.0
+      Pet.data_miner_runs.first.initial_column_statistics(:age).sum.must_be_nil
+      Pet.data_miner_runs.first.final_column_statistics(:age).sum.must_equal 28.0
+    end
+
+    it "keeps blank (empty string) count" do
+      Pet.data_miner_runs.first.initial_column_statistics(:command_phrase).blank_count.must_equal 0
+      Pet.data_miner_runs.first.final_column_statistics(:command_phrase).blank_count.must_equal 3
+    end
 
-      Pet.data_miner_runs.first.column_statistics_for(:age, :before).standard_deviation.must_be_nil
-      Pet.data_miner_runs.first.column_statistics_for(:age, :after).standard_deviation.must_equal 5.8737
+    it "keeps zero count" do
+      Pet.data_miner_runs.first.initial_column_statistics(:age).zero_count.must_equal 0
+      Pet.data_miner_runs.first.final_column_statistics(:age).zero_count.must_equal 0
     end
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_miner
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-16 00:00:00.000000000 Z
+date: 2012-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: remote_table