data_miner 2.5.2 → 3.0.0.alpha

data/lib/data_miner/script.rb

@@ -202,7 +202,7 @@ class DataMiner
     # @note Normally you should use +Country.run_data_miner!+
     # @note A primitive "call stack" is kept that will prevent infinite loops. So, if Country's data miner script calls Province's AND vice-versa, each one will only be run once.
     #
-    # @return [DataMiner::Run]
+    # @return nil
     def start
       model_name = model.name
       # $stderr.write "0 - #{model_name}\n"
@@ -217,17 +217,11 @@ class DataMiner
         Script.current_stack.clear
       end
       Script.current_stack << model_name
-      unless Run.table_exists?
-        Run.auto_upgrade!
-      end
-      run = Run.new
-      run.model_name = model_name
-      run.start do
-        steps.each do |step|
-          step.start
-          model.reset_column_information
-        end
+      steps.each do |step|
+        step.start
+        model.reset_column_information
       end
+      nil
     end
         
     private

data/lib/data_miner/step/import.rb

@@ -22,24 +22,24 @@ class DataMiner
 
       # @private
       def initialize(script, description, settings, &blk)
-        settings = settings.symbolize_keys
-        if settings.has_key?(:table)
+        settings = settings.stringify_keys
+        if settings.has_key?('table')
           raise ::ArgumentError, %{[data_miner] :table is no longer an allowed setting.}
         end
-        if (errata_settings = settings[:errata]) and not errata_settings.is_a?(::Hash)
+        if (errata_settings = settings['errata']) and not errata_settings.is_a?(::Hash)
           raise ::ArgumentError, %{[data_miner] :errata must be a hash of initialization settings to Errata}
         end
         @script = script
         @attributes = ::ActiveSupport::OrderedHash.new
-        @validate_query = !!settings[:validate]
+        @validate_query = !!settings['validate']
         @description = description
-        if settings.has_key? :errata
-          errata_settings = settings[:errata].symbolize_keys
-          errata_settings[:responder] ||= model
-          settings[:errata] = errata_settings
+        if settings.has_key? 'errata'
+          errata_settings = settings['errata'].stringify_keys
+          errata_settings['responder'] ||= model
+          settings['errata'] = errata_settings
         end
         @table_settings = settings.dup
-        @table_settings[:streaming] = true
+        @table_settings['streaming'] = true
         @table_mutex = ::Mutex.new
         instance_eval(&blk)
       end
@@ -48,17 +48,17 @@ class DataMiner
       #
       # @see DataMiner::Attribute The actual Attribute class.
       #
-      # @param [Symbol] attr_name The name of the local model column.
+      # @param [String] attr_name The name of the local model column.
       # @param [optional, Hash] attr_options Options that will be passed to +DataMiner::Attribute.new+
       # @option attr_options [*] anything Any option for +DataMiner::Attribute+.
       #
       # @return [nil]
-      def store(attr_name, attr_options = {})
-        attr_name = attr_name.to_sym
+      def store(attr_name, attr_options = {}, &blk)
+        attr_name = attr_name.to_s
         if attributes.has_key? attr_name
           raise "You should only call store or key once for #{model.name}##{attr_name}"
         end
-        attributes[attr_name] = DataMiner::Attribute.new self, attr_name, attr_options
+        attributes[attr_name] = DataMiner::Attribute.new self, attr_name, attr_options, &blk
       end
 
       # Store data into a model column AND use it as the key.
@@ -67,13 +67,13 @@ class DataMiner
       #
       # Enables idempotency. In other words, you can run the data miner script multiple times, get updated data, and not get duplicate rows.
       #
-      # @param [Symbol] attr_name The name of the local model column.
+      # @param [String] attr_name The name of the local model column.
       # @param [optional, Hash] attr_options Options that will be passed to +DataMiner::Attribute.new+
       # @option attr_options [*] anything Any option for +DataMiner::Attribute+.
       #
       # @return [nil]
       def key(attr_name, attr_options = {})
-        attr_name = attr_name.to_sym
+        attr_name = attr_name.to_s
         if attributes.has_key? attr_name
           raise "You should only call store or key once for #{model.name}##{attr_name}"
         end
@@ -83,9 +83,8 @@ class DataMiner
 
       # @private
       def start
-        upsert_enabled? ? save_with_upsert : save_with_activerecord
+        upsert_enabled? ? save_with_upsert : save_with_find_or_initialize
         refresh
-
         nil
       end
 
@@ -101,13 +100,28 @@ class DataMiner
         (not validate?) and (storing_primary_key? or table_has_autoincrementing_primary_key?)
       end
 
+      def count_every
+        @count_every ||= ENV.fetch('DATA_MINER_COUNT_EVERY', -1).to_i
+      end
+
       def save_with_upsert
         c = model.connection_pool.checkout
+        attrs_except_key = attributes.except(@key).values
+        count = 0
         Upsert.stream(c, model.table_name) do |upsert|
           table.each do |row|
+            $stderr.puts "#{count}..." if count_every > 0 and count % count_every == 0
+            count += 1
             selector = @key ? { @key => attributes[@key].read(row) } : { model.primary_key => nil }
-            document = attributes.except(@key).inject({}) do |memo, (_, attr)|
-              memo.merge! attr.updates(row)
+            document = attrs_except_key.inject({}) do |memo, attr|
+              attr.updates(row).each do |k, v|
+                case memo[k]
+                when ::Hash
+                  memo[k] = memo[k].merge v
+                else
+                  memo[k] = v
+                end
+              end
               memo
             end
             upsert.row selector, document
@@ -116,8 +130,11 @@ class DataMiner
         model.connection_pool.checkin c
       end
 
-      def save_with_activerecord
+      def save_with_find_or_initialize
+        count = 0
         table.each do |row|
+          $stderr.puts "#{count}..." if count_every > 0 and count % count_every == 0
+          count += 1
           record = @key ? model.send("find_or_initialize_by_#{@key}", attributes[@key].read(row)) : model.new
           attributes.each { |_, attr| attr.set_from_row record, row }
           record.save!
@@ -125,7 +142,7 @@ class DataMiner
       end
 
       def table_has_autoincrementing_primary_key?
-        return @table_has_autoincrementing_primary_key_query.first if @table_has_autoincrementing_primary_key_query.is_a?(Array)
+        return @table_has_autoincrementing_primary_key_query if defined?(@table_has_autoincrementing_primary_key_query)
         c = model.connection_pool.checkout
         answer = if (pk = model.primary_key) and model.columns_hash[pk].type == :integer
           case c.adapter_name
@@ -143,14 +160,12 @@ class DataMiner
           end
         end
         model.connection_pool.checkin c
-        @table_has_autoincrementing_primary_key_query = [answer]
-        answer
+        @table_has_autoincrementing_primary_key_query = answer
       end
 
       def storing_primary_key?
-        return @storing_primary_key_query.first if @storing_primary_key_query.is_a?(Array)
-        @storing_primary_key_query = [model.primary_key && attributes.has_key?(model.primary_key.to_sym)]
-        @storing_primary_key_query.first
+        return @storing_primary_key_query if defined?(@storing_primary_key_query)
+        @storing_primary_key_query = model.primary_key && attributes.has_key?(model.primary_key)
       end
 
       def table
@@ -161,7 +176,6 @@ class DataMiner
 
       def refresh
         @table = nil
-        attributes.each { |_, attr| attr.refresh }
         nil
       end
     end

data/lib/data_miner/step/sql.rb

@@ -43,7 +43,7 @@ class DataMiner
           ActiveRecord::Base.connection.execute statement
         else
           tmp_path = UnixUtils.curl url
-          send config[:adapter], tmp_path
+          send config['adapter'], tmp_path
           File.unlink tmp_path
         end
       end
@@ -55,24 +55,24 @@ class DataMiner
           ActiveRecord::Base.connection_config
         else
           ActiveRecord::Base.connection_pool.spec.config
-        end
+        end.stringify_keys
       end
 
       def mysql(path)
-        connect = if config[:socket]
-          [ '--socket', config[:socket] ]
+        connect = if config['socket']
+          [ '--socket', config['socket'] ]
         else
-          [ '--host', config.fetch(:host, '127.0.0.1'), '--port', config.fetch(:port, 3306).to_s ]
+          [ '--host', config.fetch('host', '127.0.0.1'), '--port', config.fetch('port', 3306).to_s ]
         end
         
         argv = [
           'mysql',
           '--compress',
-          '--user', config[:username],
-          "-p#{config[:password]}",
+          '--user', config['username'],
+          "-p#{config['password']}",
           connect,
           '--default-character-set', 'utf8',
-          config[:database]
+          config['database']
         ].flatten
 
         File.open(path) do |f|
@@ -97,7 +97,7 @@ class DataMiner
         argv = [
           'psql',
           '--quiet',
-          '--dbname', config[:database],
+          '--dbname', config['database'],
           '--file',   path
         ].flatten
         
@@ -113,7 +113,7 @@ class DataMiner
       def sqlite3(path)
         argv = [
           'sqlite3',
-          config[:database]
+          config['database']
         ]
         File.open(path) do |f|
           pid = POSIX::Spawn.spawn(*(argv+[{:in => f}]))

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '2.5.2'
+  VERSION = '3.0.0.alpha'
 end

data/test/data_miner/step/test_sql.rb

@@ -2,37 +2,33 @@
 require 'helper'
 init_database
 
-class BreedBlue < ActiveRecord::Base
-  self.table_name = 'breeds'
-  self.primary_key = 'name'
+class StateBlue < ActiveRecord::Base
+  self.table_name = 'states'
+  self.primary_key = 'postal_abbreviation'
   data_miner do
-    sql "Brighter Planet's list of breeds (as a URL)", 'http://data.brighterplanet.com/breeds.sql'
+    sql "Brighter Planet's list of states (as a URL)", 'http://data.brighterplanet.com/states.sql'
   end
 end
 
-class BreedRed < ActiveRecord::Base
-  self.table_name = 'breeds'
-  self.primary_key = 'name'
+class StateRed < ActiveRecord::Base
+  self.table_name = 'states'
+  self.primary_key = 'postal_abbreviation'
   data_miner do
-    sql "Brighter Planet's list of breeds (as a URL)", 'http://data.brighterplanet.com/breeds.sql'
-    sql "Mess up weights", %{UPDATE breeds SET weight = 999}
+    sql "Brighter Planet's list of states (as a URL)", 'http://data.brighterplanet.com/states.sql'
+    sql "Mess up weights", %{UPDATE states SET name = 'Foobar'}
   end
 end
 
 describe DataMiner::Step::Sql do
   before do
-    BreedBlue.delete_all rescue nil
+    StateBlue.delete_all rescue nil
   end
   it "can be provided as a URL" do
-    BreedBlue.run_data_miner!
-    BreedBlue.where(:name => 'Affenpinscher').count.must_equal 1
-    BreedBlue.where(:name => 'Württemberger').count.must_equal 1
-    BreedBlue.find('Afghan Hound').weight.must_be_close_to 24.9476
+    StateBlue.run_data_miner!
+    StateBlue.where(:name => 'Wisconsin').count.must_equal 1
   end
   it "can be provided as a string" do
-    BreedRed.run_data_miner!
-    BreedRed.where(:name => 'Affenpinscher').count.must_equal 1
-    BreedRed.where(:name => 'Württemberger').count.must_equal 1
-    BreedRed.find('Afghan Hound').weight.must_be_close_to 999
+    StateRed.run_data_miner!
+    StateRed.find('NJ').name.must_equal 'Foobar'
   end
 end

data/test/data_miner/test_attribute.rb

@@ -1,36 +1,4 @@
 require 'helper'
 
 describe DataMiner::Attribute do
-  before do
-    DataMiner.unit_converter = :alchemist
-  end
-
-  describe '#convert?' do
-    it 'returns true if from_units is set' do
-      attribute = DataMiner::Attribute.new :foo, 'bar', :from_units => :pounds, :to_units => :kilograms
-      assert attribute.send(:convert?)
-    end
-    it 'returns true if to_units and units_field_name are set' do
-      attribute = DataMiner::Attribute.new :foo, 'bar', :units_field_name => 'bar', :to_units => :kilograms
-      assert attribute.send(:convert?)
-    end
-    it 'returns true if to_units and units_field_number are set' do
-      attribute = DataMiner::Attribute.new :foo, 'bar', :units_field_number => 3, :to_units => :kilograms
-      assert attribute.send(:convert?)
-    end
-    it 'returns false if units_field_name only is set' do
-      attribute = DataMiner::Attribute.new :foo, 'bar', :units_field_name => 'bar'
-      refute attribute.send(:convert?)
-    end
-    it 'returns false if units_field_number only is set' do
-      attribute = DataMiner::Attribute.new :foo, 'bar', :units_field_number => 'bar'
-      refute attribute.send(:convert?)
-    end
-    it 'raises if no converter and units are used' do
-      DataMiner.unit_converter = nil
-      lambda {
-        DataMiner::Attribute.new :foo, 'bar', :from_units => :pounds, :to_units => :kilograms
-      }.must_raise ArgumentError, /unit_converter/
-    end
-  end
 end

data/test/helper.rb

@@ -15,14 +15,15 @@ MiniTest::Reporters.use!
 require 'active_record'
 require 'logger'
 ActiveRecord::Base.logger = Logger.new $stderr
-ActiveRecord::Base.logger.level = Logger::INFO
-# ActiveRecord::Base.logger.level = Logger::DEBUG
+ActiveRecord::Base.logger.level = (ENV['VERBOSE'] == 'true') ? Logger::DEBUG : Logger::INFO
 
 ActiveRecord::Base.mass_assignment_sanitizer = :strict
 
+require 'active_record_inline_schema'
+
 require 'data_miner'
 
-def init_database(unit_converter = :conversions)
+def init_database
   case ENV['DATABASE']
   when /postgr/i
     system %{dropdb test_data_miner}
@@ -46,12 +47,6 @@ def init_database(unit_converter = :conversions)
       'password' => 'password'
     )
   end
-
-  DataMiner::Run.auto_upgrade!
-  DataMiner::Run::ColumnStatistic.auto_upgrade!
-  DataMiner::Run.clear_locks
-
-  DataMiner.unit_converter = unit_converter
 end
 
 def init_models

data/test/support/data_miner_with_alchemist.rb

@@ -2,12 +2,8 @@ require 'helper'
 
 describe 'DataMiner with Alchemist' do
   before do
-    init_database(:alchemist)
+    init_database
     init_models
     Pet.run_data_miner!
   end
-
-  it 'converts convertible units' do
-    Pet.find('Pierre').weight.must_be_close_to 4.4.pounds.to.kilograms.to_f
-  end
 end

data/test/support/pet.rb

@@ -9,26 +9,27 @@ class Pet < ActiveRecord::Base
   col :breed_id
   col :color_id
   col :age, :type => :integer
-  col :age_units
   col :weight, :type => :float
-  col :weight_units
   col :height, :type => :float
-  col :height_units
   col :favorite_food
   col :command_phrase
+  col :emphatic_command_phrase
   belongs_to :breed
   data_miner do
     process :auto_upgrade!
     process :run_data_miner_on_parent_associations!
     import("A list of pets", :url => "file://#{PETS}") do
       key :name
-      store :age, :units_field_name => 'age_units'
-      store :breed_id, :field_name => :breed, :nullify_blank_strings => true
-      store :color_id, :field_name => :color, :dictionary => { :url => "file://#{COLOR_DICTIONARY_ENGLISH}", :input => :input, :output => :output }
-      store :weight, :from_units => :pounds, :to_units => :kilograms
-      store :favorite_food, :nullify_blank_strings => true
+      store :age
+      store :breed_id, :field_name => :breed
+      store :color_id, :field_name => :color, :dictionary => RemoteTable.new("file://#{COLOR_DICTIONARY_ENGLISH}").inject({}) { |memo, row| memo[row['input']] = row['output']; memo }
+      store :weight
+      store :favorite_food
       store :command_phrase
-      store :height, :units => :millimetres
+      store :height
+      store :emphatic_command_phrase do |row|
+        (row['command_phrase'] + "!!!!!") if row['command_phrase']
+      end
     end
   end
 end

data/test/support/pet2.rb

@@ -15,7 +15,7 @@ class Pet2 < ActiveRecord::Base
     end
     import("Breed numbers based on license number", :url => "file://#{BREED_BY_LICENSE_NUMBER}") do
       key :license_number
-      store :breed_id, :field_name => :breed, :nullify_blank_strings => true
+      store :breed_id, :field_name => :breed
     end
   end
 end

data/test/support/pets.csv

@@ -2,5 +2,5 @@ license_number,name,breed,color,age,age_units,weight,height,favorite_food,comman
 111,Pierre,Tabby,GO,4,years,4.4,"3.000,5",tomato,"eh"
 222,Jerry,Beagle,BR/BL,5,years,10,"3,000.0",cheese,"che"
 333,Amigo,Spanish Lizarto,GR/BU,17,years,"   ","300,5",crickets,"    "
-444,Johnny,Beagle,BR/BL,2,years,20,"4,000","   ",
-555,Nemo,,,,,,,,
+444,Johnny,Beagle,BR/BL,2,years,20,"4,000","   "," oh   ok "
+555,Nemo,,,,,,,,