data_miner 0.3.2 → 0.3.3

data/VERSION

@@ -1 +1 @@
-0.3.2
+0.3.3

data/data_miner.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.3.2"
+  s.version = "0.3.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-  s.date = %q{2010-02-25}
+  s.date = %q{2010-03-04}
   s.description = %q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
   s.email = %q{seamus@abshere.net}
   s.extra_rdoc_files = [

data/lib/data_miner/configuration.rb

@@ -40,7 +40,7 @@ module DataMiner
       run = target.runs.create! :started_at => Time.now
       finished = false
       begin
-        runnables.each(&:run)
+        runnables.each { |runnable| runnable.run(run) }
         finished = true
       ensure
         run.update_attributes! :ended_at => Time.now, :finished => finished
@@ -58,6 +58,7 @@ module DataMiner
       def run(options = {})
         classes.each do |klass|
           if options[:class_names].blank? or options[:class_names].include?(klass.name)
+            klass.delete_all if options[:from_scratch]
             klass.data_miner_config.run
           end
         end

data/lib/data_miner/import.rb

@@ -30,7 +30,7 @@ module DataMiner
       configuration.attributes[attr_name].options_for_import[self] = attr_options
     end
 
-    def run
+    def run(run)
       table.each_row do |row|
         if errata
           next if errata.rejects?(row)
@@ -48,6 +48,9 @@ module DataMiner
 
         Array.wrap(record_set).each do |record|
           attributes.values.each { |attr| attr.set_record_from_row(self, record, row) }
+          record.data_miner_touch_count ||= 0
+          record.data_miner_touch_count += 1
+          record.data_miner_last_run = run
           record.save!
         end
       end

data/lib/data_miner/process.rb

@@ -13,7 +13,7 @@ module DataMiner
       "Process(#{klass}) position #{position_in_run}"
     end
     
-    def run
+    def run(run)
       klass.send callback
       DataMiner.logger.info "ran #{inspect}"
     end

data/test/data_miner_test.rb

@@ -178,6 +178,7 @@ end
 
 class AutomobileMakeYear < ActiveRecord::Base
   set_primary_key :row_hash
+  belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
   
   belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'automobile_make_id'
   belongs_to :model_year, :class_name => 'AutomobileModelYear', :foreign_key => 'automobile_model_year_id'
@@ -240,6 +241,7 @@ end
 
 class AutomobileMakeFleetYear < ActiveRecord::Base
   set_primary_key :row_hash
+  belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
   belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'automobile_make_id'
   belongs_to :model_year, :class_name => 'AutomobileModelYear', :foreign_key => 'automobile_model_year_id'
   belongs_to :make_year, :class_name => 'AutomobileMakeYear', :foreign_key => 'automobile_make_year_id'
@@ -260,6 +262,7 @@ end
 
 class AutomobileModelYear < ActiveRecord::Base
   set_primary_key :year
+  belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
   
   has_many :make_years, :class_name => 'AutomobileMakeYear'
   has_many :variants, :class_name => 'AutomobileVariant'
@@ -275,6 +278,7 @@ end
 
 class AutomobileFuelType < ActiveRecord::Base
   set_primary_key :code
+  belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
   
   data_miner do
     unique_index 'code'
@@ -315,6 +319,7 @@ end
 
 class AutomobileModel < ActiveRecord::Base
   set_primary_key :row_hash
+  belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
   
   has_many :variants, :class_name => 'AutomobileVariant'
   belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'automobile_make_id'
@@ -326,6 +331,7 @@ end
 
 class AutomobileMake < ActiveRecord::Base
   set_primary_key :name
+  belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
 
   has_many :make_years, :class_name => 'AutomobileMakeYear'
   has_many :models, :class_name => 'AutomobileModel'
@@ -346,6 +352,7 @@ end
 
 class AutomobileVariant < ActiveRecord::Base
   set_primary_key :row_hash
+  belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
   
   belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'automobile_make_id'
   belongs_to :model, :class_name => 'AutomobileModel', :foreign_key => 'automobile_model_id'
@@ -534,6 +541,7 @@ end
 
 class Country < ActiveRecord::Base
   set_primary_key :iso_3166
+  belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
   
   data_miner do
     unique_index 'iso_3166'
@@ -554,6 +562,7 @@ end
 
 class Airport < ActiveRecord::Base
   set_primary_key :iata_code
+  belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
   belongs_to :country
   
   data_miner do
@@ -573,6 +582,7 @@ end
 
 class CensusRegion < ActiveRecord::Base
   set_primary_key :number
+  belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
   
   data_miner do
     unique_index 'number'
@@ -609,6 +619,7 @@ class DataMinerTest < Test::Unit::TestCase
     assert_raises DataMiner::MissingHashColumn do
       class IncompleteCountry < ActiveRecord::Base
         set_table_name 'countries'
+        belongs_to :data_miner_last_run, :class_name => 'DataMiner::Run'
         
         data_miner do
           # no unique index
@@ -630,7 +641,7 @@ class DataMinerTest < Test::Unit::TestCase
   end
   
   should "hash things if no unique index is listed" do
-    AutomobileVariant.data_miner_config.runnables[0].run
+    AutomobileVariant.data_miner_config.runnables[0].run(nil)
     assert AutomobileVariant.first.row_hash.present?
   end
   
@@ -655,7 +666,29 @@ class DataMinerTest < Test::Unit::TestCase
     assert (target.runs.last.ended_at - approx_ended_at).abs < 5 # seconds
   end
   
-  should "remove rows that have disappeared from the external data source" do
-    flunk "not implemented yet"
+  should "request a re-import from scratch" do
+    c = Country.new
+    c.iso_3166 = 'JUNK'
+    c.save!
+    assert Country.exists?(:iso_3166 => 'JUNK')
+    DataMiner.run :class_names => %w{ Country }, :from_scratch => true
+    assert !Country.exists?(:iso_3166 => 'JUNK')
+  end
+  
+  should "track how many times a row was touched" do
+    DataMiner.run :class_names => %w{ Country }, :from_scratch => true
+    assert_equal 1, Country.first.data_miner_touch_count
+    DataMiner.run :class_names => %w{ Country }
+    assert_equal 2, Country.first.data_miner_touch_count
+  end
+  
+  should "keep track of what the last import run that touched a row was" do
+    DataMiner.run :class_names => %w{ Country }, :from_scratch => true
+    a = DataMiner::Run.last
+    assert_equal a, Country.first.data_miner_last_run
+    DataMiner.run :class_names => %w{ Country }
+    b = DataMiner::Run.last
+    assert a != b
+    assert_equal b, Country.first.data_miner_last_run
   end
 end

data/test/test_helper.rb

@@ -29,6 +29,8 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.float    "longitude"
     t.datetime "created_at"
     t.datetime "updated_at"
+    t.integer 'data_miner_touch_count'
+    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE airports ADD PRIMARY KEY (iata_code);"
   
@@ -37,6 +39,8 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.string   "name"
     t.datetime "created_at"
     t.datetime "updated_at"
+    t.integer 'data_miner_touch_count'
+    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE countries ADD PRIMARY KEY (iso_3166);"
   
@@ -45,6 +49,8 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.datetime "updated_at"
     t.datetime "created_at"
     t.integer  "number"
+    t.integer 'data_miner_touch_count'
+    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE census_regions ADD PRIMARY KEY (number);"
   
@@ -78,6 +84,8 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.string   "carline_class_name"
     t.string   "speeds"
     t.string   "row_hash"
+    t.integer 'data_miner_touch_count'
+    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE automobile_variants ADD PRIMARY KEY (row_hash);"
   
@@ -88,6 +96,8 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.float    "emission_factor"
     t.float    "annual_distance"
     t.string   "code"
+    t.integer 'data_miner_touch_count'
+    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE automobile_fuel_types ADD PRIMARY KEY (code);"
 
@@ -105,6 +115,8 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.datetime "updated_at"
 
     t.string   "row_hash"
+    t.integer 'data_miner_touch_count'
+    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE automobile_make_fleet_years ADD PRIMARY KEY (row_hash);"
 
@@ -116,6 +128,8 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.float    "fuel_efficiency"
     t.integer  "volume"
     t.string   "row_hash"
+    t.integer 'data_miner_touch_count'
+    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE automobile_make_years ADD PRIMARY KEY (row_hash);"
 
@@ -125,6 +139,8 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.datetime "created_at"
     t.float    "fuel_efficiency"
     t.boolean  "major"
+    t.integer 'data_miner_touch_count'
+    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE automobile_makes ADD PRIMARY KEY (name);"
 
@@ -133,6 +149,8 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.datetime "updated_at"
     t.datetime "created_at"
     t.float    "fuel_efficiency"
+    t.integer 'data_miner_touch_count'
+    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE automobile_model_years ADD PRIMARY KEY (year);"
 
@@ -142,6 +160,8 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.datetime "updated_at"
     t.datetime "created_at"
     t.string   "row_hash"
+    t.integer 'data_miner_touch_count'
+    t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE automobile_models ADD PRIMARY KEY (row_hash);"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: data_miner
 version: !ruby/object:Gem::Version 
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-25 00:00:00 -05:00
+date: 2010-03-04 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency