data_miner 1.3.8 → 2.0.1

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '1.3.8'
+  VERSION = '2.0.1'
 end

data/test/helper.rb

@@ -1,30 +1,32 @@
 require 'rubygems'
-require 'bundler'
-Bundler.setup
-require 'test/unit'
-require 'shoulda'
-require 'mini_record'
-require 'logger'
-# require 'ruby-debug'
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-require 'data_miner'
-class Test::Unit::TestCase
-end
-
-test_log = File.open('test.log', 'w')
-test_log.sync = true
-DataMiner.logger = Logger.new test_log
+require 'bundler/setup'
 
-# because some of the test files reference it
-require 'errata'
+if Bundler.definition.specs['ruby-debug19'].first or Bundler.definition.specs['ruby-debug'].first
+  require 'ruby-debug'
+end
 
-ENV['WIP'] = 'true' if ENV['ALL'] == 'true'
+require 'minitest/spec'
+require 'minitest/autorun'
+require 'minitest/reporters'
+MiniTest::Unit.runner = MiniTest::SuiteRunner.new
+MiniTest::Unit.runner.reporters << MiniTest::Reporters::SpecReporter.new
 
-$:.push File.dirname(__FILE__)
-require 'support/test_database'
+cmd = %{mysql -u root -ppassword -e "drop database data_miner_test; create database data_miner_test charset utf8"}
+$stderr.puts "Running `#{cmd}`..."
+system cmd
+$stderr.puts "Done."
 
-ActiveSupport::Inflector.inflections do |inflect|
-  inflect.uncountable %w{ aircraft aircraft_deux census_division_deux census_division_trois }
-end
+require 'active_record'
+require 'logger'
+ActiveRecord::Base.logger = Logger.new $stderr
+ActiveRecord::Base.logger.level = Logger::INFO
+# ActiveRecord::Base.logger.level = Logger::DEBUG
+ActiveRecord::Base.establish_connection(
+  'adapter' => 'mysql2',
+  'database' => 'data_miner_test',
+  'username' => 'root',
+  'password' => 'password'
+)
 
+require 'data_miner'
+DataMiner::Run.auto_upgrade!

data/test/support/pet_color_dictionary.en.csv

@@ -0,0 +1,5 @@
+input,output
+WH/RD,white/red
+BR/BL,brown/black
+GR/BU,green/blue
+GO,gold

data/test/support/pet_color_dictionary.es.csv

@@ -0,0 +1,5 @@
+input,output
+WH/RD,blanco/rojo
+BR/BL,morron/negro
+GR/BU,verde/azul
+GO,oro

data/test/support/pets.csv

@@ -0,0 +1,5 @@
+name,breed,color,age
+Pierre,Tabby,GO,4
+Jerry,Beagle,BR/BL,5
+Amigo,Spanish Lizarto,GR/BU,17
+Johnny,Beagle,BR/BL,2

data/test/support/pets_funny.csv

@@ -0,0 +1,4 @@
+name,breed,color
+Pierre,Rhino,GO
+Jerry,Badger,BR/BL
+Amigo,Dinosaur,GR/BU

data/test/test_data_miner.rb

@@ -0,0 +1,103 @@
+# -*- encoding: utf-8 -*-
+require 'helper'
+
+PETS = File.expand_path('../support/pets.csv', __FILE__)
+PETS_FUNNY = File.expand_path('../support/pets_funny.csv', __FILE__)
+COLOR_DICTIONARY_ENGLISH = File.expand_path('../support/pet_color_dictionary.en.csv', __FILE__)
+COLOR_DICTIONARY_SPANISH = File.expand_path('../support/pet_color_dictionary.es.csv', __FILE__)
+BREEDS = File.expand_path('../support/breeds.xls', __FILE__)
+
+class Pet < ActiveRecord::Base
+  self.primary_key = "name"
+  col :name
+  col :breed_id
+  col :color_id
+  col :age, :type => :integer
+  belongs_to :breed
+  data_miner do
+    process :auto_upgrade!
+    process :run_data_miner_on_parent_associations!
+    import("A list of pets", :url => "file://#{PETS}") do
+      key :name
+      store :age
+      store :breed_id, :field_name => :breed
+      store :color_id, :field_name => :color, :dictionary => { :url => "file://#{COLOR_DICTIONARY_ENGLISH}", :input => :input, :output => :output }
+    end
+  end
+end
+
+class Breed < ActiveRecord::Base
+  class << self
+    def update_average_age!
+      # make sure pet is populated
+      Pet.run_data_miner!
+      update_all %{breeds.average_age = (SELECT AVG(pets.age) FROM pets WHERE pets.breed_id = breeds.name)}
+    end
+  end
+  self.primary_key = "name"
+  col :name
+  col :average_age, :type => :float
+  data_miner do
+    process :auto_upgrade!
+    import("A list of breeds", :url => "file://#{BREEDS}") do
+      key :name, :field_name => 'Breed name'
+    end
+    process :update_average_age!
+  end
+end
+
+Pet.auto_upgrade!
+
+describe DataMiner do
+  describe "when used to import example data about pets" do
+    before do
+      Pet.delete_all
+    end
+    it "is idempotent given a key" do
+      Pet.run_data_miner!
+      first_count = Pet.count
+      Pet.run_data_miner!
+      Pet.count.must_equal first_count
+    end
+    it "can map fields in the source file to columns in the database" do
+      Pet.run_data_miner!
+      Pet.find('Jerry').breed_id.must_equal 'Beagle'
+    end
+    it "can use a dictionary to translate source data" do
+      Pet.run_data_miner!
+      Pet.find('Jerry').color_id.must_equal 'brown/black'
+    end
+    it "refreshes the dictionary for every run" do
+      Pet.run_data_miner!
+      Pet.find('Jerry').color_id.must_equal 'brown/black'
+      begin
+        FileUtils.mv COLOR_DICTIONARY_ENGLISH, "#{COLOR_DICTIONARY_ENGLISH}.bak"
+        FileUtils.cp COLOR_DICTIONARY_SPANISH, COLOR_DICTIONARY_ENGLISH # oops! somebody swapped in a spanish dictionary
+        Pet.run_data_miner!
+        Pet.find('Jerry').color_id.must_equal 'morron/negro'
+      ensure
+        FileUtils.mv "#{COLOR_DICTIONARY_ENGLISH}.bak", COLOR_DICTIONARY_ENGLISH
+      end
+    end
+    it "refreshes the data source for every run" do
+      Pet.run_data_miner!
+      Pet.find('Jerry').breed_id.must_equal 'Beagle'
+      begin
+        FileUtils.mv PETS, "#{PETS}.bak"
+        FileUtils.cp PETS_FUNNY, PETS # oops! somebody swapped in a funny data source
+        Pet.run_data_miner!
+        Pet.find('Jerry').breed_id.must_equal 'Badger'
+      ensure
+        FileUtils.mv "#{PETS}.bak", PETS
+      end
+    end
+    it "provides :run_data_miner_on_parent_associations!" do
+      Pet.run_data_miner!
+      Pet.find('Jerry').breed.must_equal Breed.find('Beagle')
+    end
+    it "runs class methods" do
+      Breed.run_data_miner!
+      Breed.find('Beagle').average_age.must_equal((5+2)/2.0)
+    end
+  end
+end

data/test/test_earth_import.rb

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+require 'helper'
+require 'earth'
+
+# use earth, which has a plethora of real-world data_miner blocks
+Earth.init :locality, :pet, :load_data_miner => true, :apply_schemas => true
+
+describe DataMiner do
+  describe "being used by the Earth library's import steps" do
+    describe "for pets" do
+      it "can pull breed and species" do
+        Breed.run_data_miner!
+        Breed.find('Golden Retriever').species.must_equal Species.find('dog')
+      end
+    end
+    describe "for localities" do
+      it "can handle non-latin characters" do
+        Country.run_data_miner!
+        Country.find('DE').name.must_equal 'Germany'
+        Country.find('AX').name.must_equal 'Åland Islands'
+        Country.find('CI').name.must_equal "Côte d'Ivoire"
+      end
+    end
+  end
+end

data/test/test_earth_tap.rb

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+require 'helper'
+require 'earth'
+
+# use earth, which has a plethora of real-world data_miner blocks
+Earth.init :locality, :pet, :load_data_miner => false, :apply_schemas => true
+
+DataMiner.run %w{Country Breed}
+
+describe DataMiner do
+  describe "being used by the Earth library's tap steps" do
+    describe "for pets" do
+      it "can pull breed and species" do
+        Breed.find('Golden Retriever').species.must_equal Species.find('dog')
+      end
+    end
+    describe "for localities" do
+      it "can handle non-latin characters" do
+        Country.find('DE').name.must_equal 'Germany'
+        Country.find('AX').name.must_equal 'Åland Islands'
+        Country.find('CI').name.must_equal "Côte d'Ivoire"
+      end
+    end
+  end
+end

data/test/test_safety.rb

@@ -0,0 +1,43 @@
+# -*- encoding: utf-8 -*-
+require 'helper'
+require 'earth'
+
+# use earth, which has a plethora of real-world data_miner blocks
+Earth.init :locality, :pet, :load_data_miner => true, :apply_schemas => true
+
+describe DataMiner do
+  describe "when being run in a multi-threaded environment" do
+    it "tries not to duplicate data" do
+      begin
+        old_thread_abort_on_exception = Thread.abort_on_exception
+        Thread.abort_on_exception = false
+        Breed.delete_all
+        Breed.run_data_miner!
+        reference_count = Breed.count
+        Breed.delete_all
+        threads = (0..2).map do |i|
+          Thread.new do
+            $stderr.write "Thread #{i} starting\n"
+            Breed.run_data_miner!
+            $stderr.write "Thread #{i} done\n"
+          end
+        end
+        exceptions = []
+        threads.each do |t|
+          begin
+            t.join
+          rescue
+            exceptions << $!
+          end
+        end
+        exceptions.length.must_equal 2
+        exceptions.each do |exception|
+          exception.must_be_kind_of LockMethod::Locked
+        end
+        Breed.count.must_equal reference_count
+      ensure
+        Thread.abort_on_exception = old_thread_abort_on_exception
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_miner
 version: !ruby/object:Gem::Version
-  version: 1.3.8
+  version: 2.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -11,11 +11,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-10 00:00:00.000000000 Z
+date: 2012-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: remote_table
-  requirement: &2164605920 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -23,10 +23,15 @@ dependencies:
         version: 1.2.2
   type: :runtime
   prerelease: false
-  version_requirements: *2164605920
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.2
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &2164605140 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -34,10 +39,15 @@ dependencies:
         version: 2.3.4
   type: :runtime
   prerelease: false
-  version_requirements: *2164605140
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.3.4
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2164604520 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -45,10 +55,15 @@ dependencies:
         version: 2.3.4
   type: :runtime
   prerelease: false
-  version_requirements: *2164604520
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.3.4
 - !ruby/object:Gem::Dependency
   name: conversions
-  requirement: &2164619940 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -56,21 +71,15 @@ dependencies:
         version: 1.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *2164619940
-- !ruby/object:Gem::Dependency
-  name: blockenspiel
-  requirement: &2164619140 !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.3.2
-  type: :runtime
-  prerelease: false
-  version_requirements: *2164619140
+        version: 1.4.4
 - !ruby/object:Gem::Dependency
   name: errata
-  requirement: &2164618260 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -78,73 +87,60 @@ dependencies:
         version: 1.0.1
   type: :runtime
   prerelease: false
-  version_requirements: *2164618260
-- !ruby/object:Gem::Dependency
-  name: mini_record-compat
-  requirement: &2164617640 !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: *2164617640
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
-  name: loose_tight_dictionary
-  requirement: &2164616980 !ruby/object:Gem::Requirement
+  name: active_record_inline_schema
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.0.5
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
-  version_requirements: *2164616980
-- !ruby/object:Gem::Dependency
-  name: test-unit
-  requirement: &2164616480 !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: *2164616480
 - !ruby/object:Gem::Dependency
-  name: shoulda
-  requirement: &2164615900 !ruby/object:Gem::Requirement
+  name: aasm
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
-  version_requirements: *2164615900
-- !ruby/object:Gem::Dependency
-  name: mysql
-  requirement: &2164615420 !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: *2164615420
 - !ruby/object:Gem::Dependency
-  name: rake
-  requirement: &2164614880 !ruby/object:Gem::Requirement
+  name: lock_method
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 0.5.1
+  type: :runtime
   prerelease: false
-  version_requirements: *2164614880
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.1
 description: Mine remote data into your ActiveRecord models. You can also convert
   units.
 email:
@@ -153,7 +149,6 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .document
 - .gitignore
 - CHANGELOG
 - Gemfile
@@ -164,24 +159,24 @@ files:
 - lib/data_miner.rb
 - lib/data_miner/active_record_extensions.rb
 - lib/data_miner/attribute.rb
-- lib/data_miner/config.rb
 - lib/data_miner/dictionary.rb
-- lib/data_miner/import.rb
-- lib/data_miner/process.rb
 - lib/data_miner/run.rb
-- lib/data_miner/tap.rb
+- lib/data_miner/script.rb
+- lib/data_miner/step.rb
+- lib/data_miner/step/import.rb
+- lib/data_miner/step/process.rb
+- lib/data_miner/step/tap.rb
 - lib/data_miner/version.rb
 - test/helper.rb
-- test/support/aircraft.rb
-- test/support/airport.rb
-- test/support/automobile_fuel_type.rb
-- test/support/automobile_variant.rb
-- test/support/country.rb
-- test/support/test_database.rb
-- test/test_data_miner_attribute.rb
-- test/test_data_miner_process.rb
-- test/test_old_syntax.rb
-- test/test_tap.rb
+- test/support/breeds.xls
+- test/support/pet_color_dictionary.en.csv
+- test/support/pet_color_dictionary.es.csv
+- test/support/pets.csv
+- test/support/pets_funny.csv
+- test/test_data_miner.rb
+- test/test_earth_import.rb
+- test/test_earth_tap.rb
+- test/test_safety.rb
 homepage: https://github.com/seamusabshere/data_miner
 licenses: []
 post_install_message: 
@@ -202,20 +197,19 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: data_miner
-rubygems_version: 1.8.15
+rubygems_version: 1.8.21
 signing_key: 
 specification_version: 3
 summary: Mine remote data into your ActiveRecord models.
 test_files:
 - test/helper.rb
-- test/support/aircraft.rb
-- test/support/airport.rb
-- test/support/automobile_fuel_type.rb
-- test/support/automobile_variant.rb
-- test/support/country.rb
-- test/support/test_database.rb
-- test/test_data_miner_attribute.rb
-- test/test_data_miner_process.rb
-- test/test_old_syntax.rb
-- test/test_tap.rb
+- test/support/breeds.xls
+- test/support/pet_color_dictionary.en.csv
+- test/support/pet_color_dictionary.es.csv
+- test/support/pets.csv
+- test/support/pets_funny.csv
+- test/test_data_miner.rb
+- test/test_earth_import.rb
+- test/test_earth_tap.rb
+- test/test_safety.rb
 has_rdoc: