data_miner 2.2.0 → 2.3.0

data/CHANGELOG

@@ -1,3 +1,9 @@
+2.3.0 / 2012-06-21
+
+* Enhancements
+
+  * Using https://github.com/seamusabshere/upsert to speed up import steps when possible.
+
 2.2.0 / 2012-06-11
 
 * Breaking changes

data/data_miner.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'activesupport', '>=2.3.4'
   s.add_runtime_dependency 'errata', '>=1.0.1'
   s.add_runtime_dependency 'remote_table', '>=1.2.2'
+  s.add_runtime_dependency 'upsert'
 
   s.add_development_dependency 'dkastner-alchemist'
   s.add_development_dependency 'conversions'
@@ -31,7 +32,16 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'lock_method'
   s.add_development_dependency 'minitest'
   s.add_development_dependency 'minitest-reporters'
-  s.add_development_dependency 'mysql2'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'yard'
+  if RUBY_PLATFORM == 'java'
+    s.add_development_dependency 'jruby-openssl'
+    s.add_development_dependency 'activerecord-jdbcsqlite3-adapter'
+    s.add_development_dependency 'activerecord-jdbcmysql-adapter'
+    s.add_development_dependency 'activerecord-jdbcpostgresql-adapter'
+  else
+    s.add_development_dependency 'sqlite3'
+    s.add_development_dependency 'mysql2'
+    s.add_development_dependency 'pg'
+  end
 end

data/lib/data_miner/attribute.rb

@@ -25,13 +25,7 @@ class DataMiner
         errors
       end
     end
-    
-    def number_column?
-      return @number_column_query[0] if @number_column_query.is_a?(Array)
-      @number_column_query = [model.columns_hash[name.to_s].number?]
-      @number_column_query[0]
-    end
-    
+
     VALID_OPTIONS = [
       :from_units,
       :to_units,
@@ -211,22 +205,34 @@ class DataMiner
       end
     end
 
-    # @private
+    # # @private
+    # TODO make sure that nil handling is replicated when using upsert
     def set_from_row(local_record, remote_row)
       previously_nil = local_record.send(name).nil?
       currently_nil = false
-
       if previously_nil or overwrite
         new_value = read remote_row
         local_record.send "#{name}=", new_value
         currently_nil = new_value.nil?
       end
-
       if not currently_nil and persist_units? and (final_to_units = (to_units || read_units(remote_row)))
         local_record.send "#{name}_units=", final_to_units
       end
     end
 
+    # @private
+    def updates(remote_row)
+      v = read remote_row
+      if persist_units?
+        v_units = unless v.nil?
+          to_units || read_units(remote_row)
+        end
+        { name => v, "#{name}_units" => v_units }
+      else
+        { name => v }
+      end
+    end
+
     # @private
     def read(row)
       if matcher and matcher_output = matcher.match(row)
@@ -316,7 +322,7 @@ class DataMiner
     def refresh
       @dictionary = nil
     end
-        
+
     private
 
     def model
@@ -324,9 +330,15 @@ class DataMiner
     end
 
     def text_column?
-      return @text_column_query[0] if @text_column_query.is_a?(Array)
+      return @text_column_query.first if @text_column_query.is_a?(Array)
       @text_column_query = [model.columns_hash[name.to_s].text?]
-      @text_column_query[0]
+      @text_column_query.first
+    end
+    
+    def number_column?
+      return @number_column_query.first if @number_column_query.is_a?(Array)
+      @number_column_query = [model.columns_hash[name.to_s].number?]
+      @number_column_query.first
     end
 
     def static?

data/lib/data_miner/step/import.rb

@@ -1,5 +1,6 @@
 require 'errata'
 require 'remote_table'
+require 'upsert'
 
 class DataMiner
   class Step
@@ -84,10 +85,25 @@ class DataMiner
 
       # @private
       def start
-        table.each do |row|
-          record = model.send "find_or_initialize_by_#{@key}", attributes[@key].read(row)
-          attributes.each { |_, attr| attr.set_from_row record, row }
-          record.save!
+        if storing_primary_key? or table_has_autoincrementing_primary_key?
+          c = ActiveRecord::Base.connection_pool.checkout
+          Upsert.stream(c, model.table_name) do |upsert|
+            table.each do |row|
+              selector = { @key => attributes[@key].read(row) }
+              document = attributes.except(@key).inject({}) do |memo, (_, attr)|
+                memo.merge! attr.updates(row)
+                memo
+              end
+              upsert.row selector, document
+            end
+          end
+          ActiveRecord::Base.connection_pool.checkin c
+        else
+          table.each do |row|
+            record = model.send "find_or_initialize_by_#{@key}", attributes[@key].read(row)
+            attributes.each { |_, attr| attr.set_from_row record, row }
+            record.save!
+          end
         end
         refresh
         nil
@@ -95,6 +111,21 @@ class DataMiner
 
       private
 
+      def table_has_autoincrementing_primary_key?
+        return @table_has_autoincrementing_primary_key_query.first if @table_has_autoincrementing_primary_key_query.is_a?(Array)
+        answer = model.columns.any? do |column|
+          column.primary and column.sql_type =~ /\bint/i
+        end
+        @table_has_autoincrementing_primary_key_query = [answer]
+        answer
+      end
+
+      def storing_primary_key?
+        return @storing_primary_key_query.first if @storing_primary_key_query.is_a?(Array)
+        @storing_primary_key_query = [attributes.has_key?(model.primary_key.to_sym)]
+        @storing_primary_key_query.first
+      end
+
       def table
         @table || @table_mutex.synchronize do
           @table ||= ::RemoteTable.new(@table_settings)

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '2.2.0'
+  VERSION = '2.3.0'
 end

data/test/helper.rb

@@ -1,7 +1,9 @@
 require 'rubygems'
 require 'bundler/setup'
 
-if Bundler.definition.specs['ruby-debug19'].first or Bundler.definition.specs['ruby-debug'].first
+if Bundler.definition.specs['debugger'].first
+  require 'debugger'
+elsif Bundler.definition.specs['ruby-debug'].first
   require 'ruby-debug'
 end
 
@@ -16,12 +18,46 @@ require 'logger'
 ActiveRecord::Base.logger = Logger.new $stderr
 ActiveRecord::Base.logger.level = Logger::INFO
 # ActiveRecord::Base.logger.level = Logger::DEBUG
-ActiveRecord::Base.establish_connection(
-  'adapter' => 'mysql2',
-  'database' => 'data_miner_test',
-  'username' => 'root',
-  'password' => 'password'
-)
+
+case ENV['DATABASE']
+when /postgr/i
+  createdb_bin = ENV['TEST_CREATEDB_BIN'] || 'createdb'
+  dropdb_bin = ENV['TEST_DROPDB_BIN'] || 'dropdb'
+  username = ENV['TEST_POSTGRES_USERNAME'] || `whoami`.chomp
+  # password = ENV['TEST_POSTGRES_PASSWORD'] || 'password'
+  database = ENV['TEST_POSTGRES_DATABASE'] || 'data_miner_test'
+  system %{#{dropdb_bin} #{database}}
+  system %{#{createdb_bin} #{database}}
+  ActiveRecord::Base.establish_connection(
+    'adapter' => 'postgresql',
+    'encoding' => 'utf8',
+    'database' => database,
+    'username' => username
+    # 'password' => password
+  )
+when /sqlite/i
+  ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":memory:")
+else
+  bin = ENV['TEST_MYSQL_BIN'] || 'mysql'
+  username = ENV['TEST_MYSQL_USERNAME'] || 'root'
+  password = ENV['TEST_MYSQL_PASSWORD'] || 'password'
+  database = ENV['TEST_MYSQL_DATABASE'] || 'data_miner_test'
+  cmd = "#{bin} -u #{username} -p#{password}"
+  `#{cmd} -e 'show databases'`
+  unless $?.success?
+    $stderr.puts "Skipping mysql tests because `#{cmd}` doesn't work"
+    exit 0
+  end
+  system %{#{cmd} -e "drop database #{database}"}
+  system %{#{cmd} -e "create database #{database}"}
+  ActiveRecord::Base.establish_connection(
+    'adapter' => (RUBY_PLATFORM == 'java' ? 'mysql' : 'mysql2'),
+    'encoding' => 'utf8',
+    'database' => database,
+    'username' => username,
+    'password' => password
+  )
+end
 
 ActiveRecord::Base.mass_assignment_sanitizer = :strict
 
@@ -43,7 +79,9 @@ end
 def init_models
   require 'support/breed'
   require 'support/pet'
+  require 'support/pet2'
   Pet.auto_upgrade!
+  Pet2.auto_upgrade!
 
   ActiveRecord::Base.descendants.each do |model|
     model.attr_accessible nil

data/test/support/breed.rb

@@ -5,7 +5,7 @@ class Breed < ActiveRecord::Base
     def update_average_age!
       # make sure pet is populated
       Pet.run_data_miner!
-      update_all %{breeds.average_age = (SELECT AVG(pets.age) FROM pets WHERE pets.breed_id = breeds.name)}
+      update_all %{"average_age" = (SELECT AVG("pets"."age") FROM "pets" WHERE "pets"."breed_id" = "breeds"."name")}
     end
   end
   self.primary_key = "name"

data/test/support/breed_by_license_number.csv

@@ -0,0 +1,2 @@
+license_number,breed
+222,Beagle-Basset

data/test/support/pet2.rb

@@ -0,0 +1,21 @@
+BREED_BY_LICENSE_NUMBER = File.expand_path('../breed_by_license_number.csv', __FILE__)
+
+class Pet2 < ActiveRecord::Base
+  self.primary_key = "name"
+  col :name
+  col :breed_id
+  col :license_number, :type => :integer
+
+  data_miner do
+    process :auto_upgrade!
+    process :run_data_miner_on_parent_associations!
+    import("A list of pets", :url => "file://#{PETS}") do
+      key :name
+      store :license_number
+    end
+    import("Breed numbers based on license number", :url => "file://#{BREED_BY_LICENSE_NUMBER}") do
+      key :license_number
+      store :breed_id, :field_name => :breed, :nullify_blank_strings => true
+    end
+  end
+end

data/test/support/pets.csv

@@ -1,6 +1,6 @@
-name,breed,color,age,age_units,weight,height,favorite_food,command_phrase
-Pierre,Tabby,GO,4,years,4.4,"3.000,5",tomato,"eh"
-Jerry,Beagle,BR/BL,5,years,10,"3,000.0",cheese,"che"
-Amigo,Spanish Lizarto,GR/BU,17,years,"   ","300,5",crickets,"    "
-Johnny,Beagle,BR/BL,2,years,20,"4,000","   ",
-Nemo,,,,,,,,
+license_number,name,breed,color,age,age_units,weight,height,favorite_food,command_phrase
+111,Pierre,Tabby,GO,4,years,4.4,"3.000,5",tomato,"eh"
+222,Jerry,Beagle,BR/BL,5,years,10,"3,000.0",cheese,"che"
+333,Amigo,Spanish Lizarto,GR/BU,17,years,"   ","300,5",crickets,"    "
+444,Johnny,Beagle,BR/BL,2,years,20,"4,000","   ",
+555,Nemo,,,,,,,,

data/test/test_data_miner.rb

@@ -108,5 +108,9 @@ describe DataMiner do
       Pet.data_miner_runs.first.row_count_before.must_equal 0
       Pet.data_miner_runs.first.row_count_after.must_equal 5
     end
+    it "can import based on keys other than the primary key" do
+      Pet2.run_data_miner!
+      Pet2.find('Jerry').breed_id.must_equal 'Beagle-Basset'
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_miner
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-11 00:00:00.000000000 Z
+date: 2012-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -109,6 +109,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.2.2
+- !ruby/object:Gem::Dependency
+  name: upsert
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: dkastner-alchemist
   requirement: !ruby/object:Gem::Requirement
@@ -222,7 +238,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mysql2
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -238,7 +254,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: yard
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -254,7 +270,39 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: yard
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -306,11 +354,13 @@ files:
 - test/data_miner/unit_converter/test_conversions.rb
 - test/helper.rb
 - test/support/breed.rb
+- test/support/breed_by_license_number.csv
 - test/support/breeds.xls
 - test/support/data_miner_with_alchemist.rb
 - test/support/data_miner_with_conversions.rb
 - test/support/data_miner_without_unit_converter.rb
 - test/support/pet.rb
+- test/support/pet2.rb
 - test/support/pet_color_dictionary.en.csv
 - test/support/pet_color_dictionary.es.csv
 - test/support/pets.csv
@@ -352,11 +402,13 @@ test_files:
 - test/data_miner/unit_converter/test_conversions.rb
 - test/helper.rb
 - test/support/breed.rb
+- test/support/breed_by_license_number.csv
 - test/support/breeds.xls
 - test/support/data_miner_with_alchemist.rb
 - test/support/data_miner_with_conversions.rb
 - test/support/data_miner_without_unit_converter.rb
 - test/support/pet.rb
+- test/support/pet2.rb
 - test/support/pet_color_dictionary.en.csv
 - test/support/pet_color_dictionary.es.csv
 - test/support/pets.csv