data_miner 2.3.0 → 2.3.1

data/CHANGELOG

@@ -1,3 +1,10 @@
+2.3.1 / 2012-06-22
+
+* Bug fixes
+
+  * Avoid duplicate rows by correctly identifying auto-increment versus non-auto-increment integer primary keys
+  * Properly sort per-row column statistics
+
 2.3.0 / 2012-06-21
 
 * Enhancements

data/Rakefile

@@ -11,6 +11,7 @@ end
 
 task :test_separately do
   Dir[File.expand_path('../test/**/test_*.rb', __FILE__)].each do |path|
+    sleep 0.5
     system "rake test TEST=#{path}"
   end
 end

data/data_miner.gemspec

@@ -18,12 +18,12 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   
   s.add_runtime_dependency 'aasm'
-  s.add_runtime_dependency 'active_record_inline_schema'
+  s.add_runtime_dependency 'active_record_inline_schema', '>=0.5.6'
   s.add_runtime_dependency 'activerecord', '>=2.3.4'
   s.add_runtime_dependency 'activesupport', '>=2.3.4'
   s.add_runtime_dependency 'errata', '>=1.0.1'
-  s.add_runtime_dependency 'remote_table', '>=1.2.2'
-  s.add_runtime_dependency 'upsert'
+  s.add_runtime_dependency 'remote_table', '>=2.0.2'
+  s.add_runtime_dependency 'upsert', '>=0.3.1'
 
   s.add_development_dependency 'dkastner-alchemist'
   s.add_development_dependency 'conversions'

data/lib/data_miner/run.rb

@@ -66,7 +66,7 @@ class DataMiner
 
     validates_presence_of :model_name
 
-    has_many :column_statistics, :class_name => 'DataMiner::Run::ColumnStatistic'
+    has_many :column_statistics, :class_name => 'DataMiner::Run::ColumnStatistic', :order => 'id ASC'
 
     include ::AASM
     aasm_initial_state INITIAL_STATE

data/lib/data_miner/step/import.rb

@@ -113,9 +113,23 @@ class DataMiner
 
       def table_has_autoincrementing_primary_key?
         return @table_has_autoincrementing_primary_key_query.first if @table_has_autoincrementing_primary_key_query.is_a?(Array)
-        answer = model.columns.any? do |column|
-          column.primary and column.sql_type =~ /\bint/i
+        c = ActiveRecord::Base.connection_pool.checkout
+        answer = if (pk = model.primary_key) and model.columns_hash[pk].type == :integer
+          case c.adapter_name
+          when /mysql/i
+            extra = c.select_value %{SELECT EXTRA FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = #{c.quote(c.current_database)} AND TABLE_NAME = #{c.quote(model.table_name)} AND COLUMN_NAME = #{c.quote(pk)}}
+            extra.to_s.include?('auto_increment')
+          when /postgres/i
+            column_default = c.select_value %{SELECT COLUMN_DEFAULT FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = #{c.quote(model.table_name)} AND COLUMN_NAME = #{c.quote(pk)}}
+            column_default.to_s.include?('nextval')
+          when /sqlite/i
+            # FIXME doesn't work
+            # row = c.select_rows("PRAGMA table_info(#{model.quoted_table_name})").detect { |r| r[1] == pk }
+            # row[2] == 'INTEGER' and row[3] == 1 and row[5] == 1
+            true
+          end
         end
+        ActiveRecord::Base.connection_pool.checkin c
         @table_has_autoincrementing_primary_key_query = [answer]
         answer
       end

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '2.3.0'
+  VERSION = '2.3.1'
 end

data/test/data_miner/step/test_import.rb

@@ -0,0 +1,33 @@
+require 'helper'
+init_database
+require 'earth'
+
+Earth.init :residence, :electricity, :hospitality, :load_data_miner => true, :apply_schemas => true
+
+class PetBlue < ActiveRecord::Base
+  data_miner do
+    import 'fake', :url => 'fake' do
+      key :id
+    end
+  end
+end
+PetBlue.auto_upgrade!
+
+describe DataMiner::Step::Import do
+  describe '#table_has_autoincrementing_primary_key?' do
+    it "recognizes auto-increment primary keys" do
+      PetBlue.data_miner_script.steps.first.send(:table_has_autoincrementing_primary_key?).must_equal true
+    end
+    it "recognizes that not all integer primary keys are auto-increment" do
+      [
+        ElectricUtility,
+        ResidentialEnergyConsumptionSurveyResponse,
+        CommercialBuildingEnergyConsumptionSurveyResponse,
+      ].each do |model|
+        model.data_miner_script.steps.select { |s| s.is_a?(DataMiner::Step::Import) }.each do |import_step|
+          import_step.send(:table_has_autoincrementing_primary_key?).must_equal false
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -19,55 +19,34 @@ ActiveRecord::Base.logger = Logger.new $stderr
 ActiveRecord::Base.logger.level = Logger::INFO
 # ActiveRecord::Base.logger.level = Logger::DEBUG
 
-case ENV['DATABASE']
-when /postgr/i
-  createdb_bin = ENV['TEST_CREATEDB_BIN'] || 'createdb'
-  dropdb_bin = ENV['TEST_DROPDB_BIN'] || 'dropdb'
-  username = ENV['TEST_POSTGRES_USERNAME'] || `whoami`.chomp
-  # password = ENV['TEST_POSTGRES_PASSWORD'] || 'password'
-  database = ENV['TEST_POSTGRES_DATABASE'] || 'data_miner_test'
-  system %{#{dropdb_bin} #{database}}
-  system %{#{createdb_bin} #{database}}
-  ActiveRecord::Base.establish_connection(
-    'adapter' => 'postgresql',
-    'encoding' => 'utf8',
-    'database' => database,
-    'username' => username
-    # 'password' => password
-  )
-when /sqlite/i
-  ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":memory:")
-else
-  bin = ENV['TEST_MYSQL_BIN'] || 'mysql'
-  username = ENV['TEST_MYSQL_USERNAME'] || 'root'
-  password = ENV['TEST_MYSQL_PASSWORD'] || 'password'
-  database = ENV['TEST_MYSQL_DATABASE'] || 'data_miner_test'
-  cmd = "#{bin} -u #{username} -p#{password}"
-  `#{cmd} -e 'show databases'`
-  unless $?.success?
-    $stderr.puts "Skipping mysql tests because `#{cmd}` doesn't work"
-    exit 0
-  end
-  system %{#{cmd} -e "drop database #{database}"}
-  system %{#{cmd} -e "create database #{database}"}
-  ActiveRecord::Base.establish_connection(
-    'adapter' => (RUBY_PLATFORM == 'java' ? 'mysql' : 'mysql2'),
-    'encoding' => 'utf8',
-    'database' => database,
-    'username' => username,
-    'password' => password
-  )
-end
-
 ActiveRecord::Base.mass_assignment_sanitizer = :strict
 
 require 'data_miner'
 
 def init_database(unit_converter = :conversions)
-  cmd = %{mysql -u root -ppassword -e "DROP DATABASE data_miner_test; CREATE DATABASE data_miner_test CHARSET utf8"}
-  $stderr.puts "Running `#{cmd}`..."
-  system cmd
-  $stderr.puts "Done."
+  case ENV['DATABASE']
+  when /postgr/i
+    system %{dropdb test_data_miner}
+    system %{createdb test_data_miner}
+    ActiveRecord::Base.establish_connection(
+      'adapter' => 'postgresql',
+      'encoding' => 'utf8',
+      'database' => 'test_data_miner',
+      'username' => `whoami`.chomp
+    )
+  when /sqlite/i
+    ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":memory:")
+  else
+    system %{mysql -u root -ppassword -e "DROP DATABASE test_data_miner"}
+    system %{mysql -u root -ppassword -e "CREATE DATABASE test_data_miner CHARSET utf8"}
+    ActiveRecord::Base.establish_connection(
+      'adapter' => (RUBY_PLATFORM == 'java' ? 'mysql' : 'mysql2'),
+      'encoding' => 'utf8',
+      'database' => 'test_data_miner',
+      'username' => 'root',
+      'password' => 'password'
+    )
+  end
 
   DataMiner::Run.auto_upgrade!
   DataMiner::Run::ColumnStatistic.auto_upgrade!

data/test/test_data_miner_run_column_statistic.rb

@@ -27,7 +27,7 @@ describe DataMiner::Run::ColumnStatistic do
 
     it "keeps max and min (as strings)" do
       Pet.data_miner_runs.first.initial_column_statistics(:age).max.must_equal 'nil'
-      Pet.data_miner_runs.first.final_column_statistics(:age).max.must_equal '17'
+      Pet.data_miner_runs.first.final_column_statistics(:age).max.must_include '17'
     end
 
     it "keeps average and sum" do

data/test/test_earth_import.rb

@@ -1,7 +1,6 @@
 # -*- encoding: utf-8 -*-
 require 'helper'
 init_database
-init_models
 require 'earth'
 
 # use earth, which has a plethora of real-world data_miner blocks

data/test/test_earth_tap.rb

@@ -1,7 +1,6 @@
 # -*- encoding: utf-8 -*-
 require 'helper'
 init_database
-init_models
 require 'earth'
 
 # use earth, which has a plethora of real-world data_miner blocks

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_miner
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.3.1
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-21 00:00:00.000000000 Z
+date: 2012-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -36,7 +36,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.5.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -44,7 +44,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.5.6
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -100,7 +100,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: 2.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -108,7 +108,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: 2.0.2
 - !ruby/object:Gem::Dependency
   name: upsert
   requirement: !ruby/object:Gem::Requirement
@@ -116,7 +116,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -124,7 +124,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.1
 - !ruby/object:Gem::Dependency
   name: dkastner-alchemist
   requirement: !ruby/object:Gem::Requirement
@@ -349,6 +349,7 @@ files:
 - lib/data_miner/unit_converter/alchemist.rb
 - lib/data_miner/unit_converter/conversions.rb
 - lib/data_miner/version.rb
+- test/data_miner/step/test_import.rb
 - test/data_miner/test_attribute.rb
 - test/data_miner/unit_converter/test_alchemist.rb
 - test/data_miner/unit_converter/test_conversions.rb
@@ -397,6 +398,7 @@ specification_version: 3
 summary: Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import
   XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models.
 test_files:
+- test/data_miner/step/test_import.rb
 - test/data_miner/test_attribute.rb
 - test/data_miner/unit_converter/test_alchemist.rb
 - test/data_miner/unit_converter/test_conversions.rb