data_miner 2.5.2 → 3.0.0.alpha

data/lib/data_miner/run/column_statistic.rb

@@ -1,78 +0,0 @@
-class DataMiner
-  class Run < ::ActiveRecord::Base
-    # If +DataMiner.per_column_statistics?+, this model keeps per-column stats like max, min, average, standard deviation, etc.
-    #
-    # Each +DataMiner::Run+ will have two of these for every column; an "initial" and a "final"
-    class ColumnStatistic < ::ActiveRecord::Base
-      class << self
-        def take(run)
-          unless table_exists?
-            auto_upgrade!
-          end
-          model = run.model_name.constantize
-          return unless model.table_exists?
-          model.column_names.each do |column_name|
-            column_statistic = new
-            column_statistic.run = run
-            column_statistic.model_name = run.model_name
-            column_statistic.column_name = column_name
-            column_statistic.take_statistics
-            column_statistic.save!
-          end
-          nil
-        end
-
-      end
-
-      NUMERIC = [
-        :integer,
-        :float,
-        :decimal,
-      ]
-
-      self.table_name = 'data_miner_run_column_statistics'
-
-      belongs_to :run, :class_name => 'DataMiner::Run'
-
-      col :run_id, :type => :integer
-      col :model_name
-      col :column_name
-      col :null_count, :type => :integer
-      col :zero_count, :type => :integer
-      col :blank_count, :type => :integer
-      col :max
-      col :min
-      col :average, :type => :float
-      col :sum, :type => :float
-      col :created_at, :type => :datetime
-      add_index :run_id
-      add_index :model_name
-
-      # @private
-      def take_statistics
-        model = run.model_name.constantize
-
-        self.null_count = model.where("#{model.connection.quote_column_name(column_name)} IS NULL").count
-        
-        self.max = calc(:MAX).inspect
-        self.min = calc(:MIN).inspect
-
-        column = model.columns_hash[column_name]
-        if NUMERIC.include?(column.type)
-          self.zero_count = model.where(column_name => 0).count
-          self.average = calc :AVG
-          self.sum = calc :SUM
-        elsif column.type == :string
-          self.blank_count = model.where("LENGTH(TRIM(#{model.connection.quote_column_name(column_name)})) = 0").count
-        end
-      end
-
-      private
-
-      def calc(operation)
-        model = run.model_name.constantize
-        model.connection.select_value "SELECT #{operation}(#{model.connection.quote_column_name(column_name)}) FROM #{model.quoted_table_name}"
-      end
-    end
-  end
-end

data/lib/data_miner/unit_converter.rb

@@ -1,12 +0,0 @@
-class DataMiner
-  class UnitConverter
-    class << self
-      def load(type)
-        if type
-          require "data_miner/unit_converter/#{type}"
-          const_get(type.to_s.camelcase).new
-        end
-      end
-    end
-  end
-end

data/lib/data_miner/unit_converter/alchemist.rb

@@ -1,11 +0,0 @@
-require 'alchemist'
-
-class DataMiner
-  class UnitConverter
-    class Alchemist < UnitConverter
-      def convert(value, from, to)
-        value.to_f.send(from).to.send(to).to_f
-      end
-    end
-  end
-end

data/lib/data_miner/unit_converter/conversions.rb

@@ -1,11 +0,0 @@
-require 'conversions'
-
-class DataMiner
-  class UnitConverter
-    class Conversions < UnitConverter
-      def convert(value, from, to)
-        value.to_f.convert from, to
-      end
-    end
-  end
-end

data/test/data_miner/step/test_import.rb

@@ -1,35 +0,0 @@
-require 'helper'
-init_database
-require 'earth'
-
-require 'earth/residence'
-require 'earth/electricity'
-require 'earth/hospitality'
-
-class PetBlue < ActiveRecord::Base
-  data_miner do
-    import 'fake', :url => 'fake' do
-      key :id
-    end
-  end
-end
-PetBlue.auto_upgrade!
-
-describe DataMiner::Step::Import do
-  describe '#table_has_autoincrementing_primary_key?' do
-    it "recognizes auto-increment primary keys" do
-      PetBlue.data_miner_script.steps.first.send(:table_has_autoincrementing_primary_key?).must_equal true
-    end
-    it "recognizes that not all integer primary keys are auto-increment" do
-      [
-        ElectricUtility,
-        ResidentialEnergyConsumptionSurveyResponse,
-        CommercialBuildingEnergyConsumptionSurveyResponse,
-      ].each do |model|
-        model.data_miner_script.steps.select { |s| s.is_a?(DataMiner::Step::Import) }.each do |import_step|
-          import_step.send(:table_has_autoincrementing_primary_key?).must_equal false
-        end
-      end
-    end
-  end
-end

data/test/data_miner/unit_converter/test_alchemist.rb

@@ -1,20 +0,0 @@
-require 'helper'
-
-describe 'DataMiner::UnitConverter::Alchemist' do
-  before do
-    @original_converter = DataMiner.unit_converter
-    DataMiner.unit_converter = :alchemist
-  end
-
-  after do
-    DataMiner.unit_converter = @original_converter
-  end
-
-  describe '#convert' do
-    it 'converts a value from one unit to another' do
-      value = DataMiner.unit_converter.convert 3.5, :kilograms, :pounds
-      assert value.is_a?(Float)
-      value.must_be_close_to 7.71617918
-    end
-  end
-end

data/test/data_miner/unit_converter/test_conversions.rb

@@ -1,20 +0,0 @@
-require 'helper'
-
-describe 'DataMiner::UnitConverter::Conversions' do
-  before do
-    @original_converter = DataMiner.unit_converter
-    DataMiner.unit_converter = :conversions
-  end
-
-  after do
-    DataMiner.unit_converter = @original_converter
-  end
-
-  describe '#convert' do
-    it 'converts a value from one unit to another' do
-      value = DataMiner.unit_converter.convert 3.5, :kilograms, :pounds
-      assert value.is_a?(Float)
-      value.must_be_close_to 7.71617918
-    end
-  end
-end

data/test/support/data_miner_with_conversions.rb

@@ -1,16 +0,0 @@
-require 'helper'
-
-require 'conversions'
-Conversions.register :years, :years, 1
-
-describe 'DataMiner with Conversions' do
-  before do
-    init_database(:conversions)
-    init_models
-    Pet.run_data_miner!
-  end
-
-  it 'converts convertible units' do
-    Pet.find('Pierre').weight.must_be_close_to 4.4.pounds.to(:kilograms)
-  end
-end

data/test/support/data_miner_without_unit_converter.rb

@@ -1,51 +0,0 @@
-require 'helper'
-
-class MyPet < ActiveRecord::Base
-  PETS = File.expand_path('../pets.csv', __FILE__)
-  COLOR_DICTIONARY_ENGLISH = File.expand_path('../pet_color_dictionary.en.csv', __FILE__)
-
-  self.primary_key = "name"
-  col :name
-  col :color_id
-  col :age, :type => :integer
-  col :age_units
-  col :weight, :type => :float
-  col :weight_units
-  col :height, :type => :integer
-  col :height_units
-  col :favorite_food
-  col :command_phrase
-
-  data_miner do
-    process :auto_upgrade!
-    import("A list of pets", :url => "file://#{PETS}") do
-      key :name
-      store :age
-      store :color_id, :field_name => :color, :dictionary => { :url => "file://#{COLOR_DICTIONARY_ENGLISH}", :input => :input, :output => :output }
-      store :weight
-      store :favorite_food, :nullify_blank_strings => true
-      store :command_phrase
-      store :height, :units => :centimetres
-    end
-  end
-end
-
-describe 'DataMiner with Conversions' do
-  it 'happens when DataMiner.unit_converter is nil' do
-    DataMiner.unit_converter.must_be_nil
-  end
-
-  it 'converts convertible units' do
-    init_database(nil)
-    MyPet.run_data_miner!
-    MyPet.find('Pierre').weight.must_equal 4.4
-  end
-
-  it 'raises an error if conversions are attempted' do
-    init_database(nil)
-    lambda do
-      init_models
-      Pet.run_data_miner!
-    end.must_raise DataMiner::Attribute::NoConverterSet
-  end
-end

data/test/test_data_miner_run_column_statistic.rb

@@ -1,52 +0,0 @@
-# -*- encoding: utf-8 -*-
-require 'helper'
-init_database
-init_models
-
-describe DataMiner::Run::ColumnStatistic do
-  describe "when advanced statistics are enabled" do
-    before do
-      DataMiner.per_column_statistics = true
-      Pet.delete_all
-      DataMiner::Run.delete_all
-      DataMiner::Run::ColumnStatistic.delete_all
-      Pet.run_data_miner!
-    end
-
-    after do
-      DataMiner.per_column_statistics = false
-    end
-
-    it "keeps null count" do
-      Pet.data_miner_runs.first.initial_column_statistics(:breed_id).null_count.must_equal 0
-      Pet.data_miner_runs.first.final_column_statistics(:breed_id).null_count.must_equal 1
-
-      Pet.data_miner_runs.first.initial_column_statistics(:command_phrase).null_count.must_equal 0
-      Pet.data_miner_runs.first.final_column_statistics(:command_phrase).null_count.must_equal 0
-    end
-
-    it "keeps max and min (as strings)" do
-      Pet.data_miner_runs.first.initial_column_statistics(:age).max.must_equal 'nil'
-      Pet.data_miner_runs.first.final_column_statistics(:age).max.must_include '17'
-    end
-
-    it "keeps average and sum" do
-      Pet.data_miner_runs.first.initial_column_statistics(:age).average.must_be_nil
-      Pet.data_miner_runs.first.final_column_statistics(:age).average.must_equal 7.0
-
-      Pet.data_miner_runs.first.initial_column_statistics(:age).sum.must_be_nil
-      Pet.data_miner_runs.first.final_column_statistics(:age).sum.must_equal 28.0
-    end
-
-    it "keeps blank (empty string) count" do
-      Pet.data_miner_runs.first.initial_column_statistics(:command_phrase).blank_count.must_equal 0
-      Pet.data_miner_runs.first.final_column_statistics(:command_phrase).blank_count.must_equal 3
-    end
-
-    it "keeps zero count" do
-      Pet.data_miner_runs.first.initial_column_statistics(:age).zero_count.must_equal 0
-      Pet.data_miner_runs.first.final_column_statistics(:age).zero_count.must_equal 0
-    end
-
-  end
-end

data/test/test_earth_import.rb

@@ -1,26 +0,0 @@
-# -*- encoding: utf-8 -*-
-require 'helper'
-init_database
-require 'earth'
-
-# use earth, which has a plethora of real-world data_miner blocks
-Earth.init :locality, :pet, :load_data_miner => true, :apply_schemas => true
-
-describe DataMiner do
-  describe "being used by the Earth library's import steps" do
-    describe "for pets" do
-      it "can pull breed and species" do
-        Breed.run_data_miner!
-        Breed.find('Golden Retriever').species.must_equal Species.find('dog')
-      end
-    end
-    describe "for localities" do
-      it "can handle non-latin characters" do
-        Country.run_data_miner!
-        Country.find('DE').name.must_equal 'Germany'
-        Country.find('AX').name.must_equal 'Åland Islands'
-        Country.find('CI').name.must_equal "Côte d'Ivoire"
-      end
-    end
-  end
-end

data/test/test_safety.rb

@@ -1,84 +0,0 @@
-# -*- encoding: utf-8 -*-
-require 'helper'
-init_database
-init_models
-require 'earth'
-
-require 'lock_method'
-DataMiner::Run.lock_method :start
-
-# use earth, which has a plethora of real-world data_miner blocks
-Earth.init :locality, :pet, :load_data_miner => true, :apply_schemas => true
-
-describe DataMiner do
-  describe "when being run in a multi-threaded environment" do
-    before do
-      @old_thread_abort_on_exception = Thread.abort_on_exception
-      Thread.abort_on_exception = false
-    end
-
-    after do
-      Thread.abort_on_exception = @old_thread_abort_on_exception
-    end
-
-    it "tries not to duplicate data" do
-      Breed.delete_all
-      Breed.run_data_miner!
-      reference_count = Breed.count
-      Breed.delete_all
-      threads = (0..2).map do |i|
-        Thread.new do
-          # $stderr.write "Thread #{i} starting\n"
-          Breed.run_data_miner!
-          # $stderr.write "Thread #{i} done\n"
-        end
-      end
-      exceptions = []
-      threads.each do |t|
-        begin
-          t.join
-        rescue
-          exceptions << $!
-        end
-      end
-      exceptions.length.must_equal 2
-      exceptions.each do |exception|
-        exception.must_be_kind_of LockMethod::Locked
-      end
-      Breed.count.must_equal reference_count
-    end
-
-    it "allows you to clear locks if necessary" do
-      threads = (0..2).map do |i|
-        Thread.new do
-          # $stderr.write "Thread #{i} starting\n"
-          case i
-          when 0
-            Breed.run_data_miner! 
-          when 1
-            sleep 0.3
-            DataMiner::Run.clear_locks
-            Breed.run_data_miner!
-          when 2
-            # i will hit a lock!
-            sleep 0.6
-            Breed.run_data_miner!
-          end
-          # $stderr.write "Thread #{i} done\n"
-        end
-      end
-      exceptions = []
-      threads.each do |t|
-        begin
-          t.join
-        rescue
-          exceptions << $!
-        end
-      end
-      exceptions.length.must_equal 1
-      exceptions.each do |exception|
-        exception.must_be_kind_of LockMethod::Locked
-      end
-    end
-  end
-end

data/test/test_unit_conversion.rb

@@ -1,16 +0,0 @@
-require 'helper'
-
-describe 'DataMiner unit conversion' do
-  it "blows up if you don't specify a converter" do
-    output = `ruby -I#{File.dirname(__FILE__)} #{File.expand_path('../support/data_miner_without_unit_converter.rb', __FILE__)}`
-    refute $?.success?, output
-  end
-  it 'can convert with alchemist' do
-    output = `ruby -I#{File.dirname(__FILE__)} #{File.expand_path('../support/data_miner_with_alchemist.rb', __FILE__)}`
-    assert $?.success?, output
-  end
-  it 'can convert with conversions' do
-    output = `ruby -I#{File.dirname(__FILE__)} #{File.expand_path('../support/data_miner_with_conversions.rb', __FILE__)}`
-    assert $?.success?, output
-  end
-end