data_miner 0.4.17 → 0.4.18

data/Rakefile

@@ -10,7 +10,7 @@ begin
     gem.email = "seamus@abshere.net"
     gem.homepage = "http://github.com/seamusabshere/data_miner"
     gem.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-    gem.add_dependency 'remote_table', '>=0.2.9'
+    gem.add_dependency 'remote_table', '>=0.2.10'
     gem.add_dependency 'activerecord', '>=2.3.4'
     gem.add_dependency 'activesupport', '>=2.3.4'
     gem.add_dependency 'andand', '>=1.3.1'

data/VERSION

@@ -1 +1 @@
-0.4.17
+0.4.18

data/data_miner.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.4.17"
+  s.version = "0.4.18"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-  s.date = %q{2010-04-15}
+  s.date = %q{2010-04-16}
   s.description = %q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
   s.email = %q{seamus@abshere.net}
   s.extra_rdoc_files = [
@@ -52,7 +52,7 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<remote_table>, [">= 0.2.9"])
+      s.add_runtime_dependency(%q<remote_table>, [">= 0.2.10"])
       s.add_runtime_dependency(%q<activerecord>, [">= 2.3.4"])
       s.add_runtime_dependency(%q<activesupport>, [">= 2.3.4"])
       s.add_runtime_dependency(%q<andand>, [">= 1.3.1"])
@@ -61,7 +61,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<blockenspiel>, [">= 0.3.2"])
       s.add_runtime_dependency(%q<log4r>, [">= 1.1.7"])
     else
-      s.add_dependency(%q<remote_table>, [">= 0.2.9"])
+      s.add_dependency(%q<remote_table>, [">= 0.2.10"])
       s.add_dependency(%q<activerecord>, [">= 2.3.4"])
       s.add_dependency(%q<activesupport>, [">= 2.3.4"])
       s.add_dependency(%q<andand>, [">= 1.3.1"])
@@ -71,7 +71,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<log4r>, [">= 1.1.7"])
     end
   else
-    s.add_dependency(%q<remote_table>, [">= 0.2.9"])
+    s.add_dependency(%q<remote_table>, [">= 0.2.10"])
     s.add_dependency(%q<activerecord>, [">= 2.3.4"])
     s.add_dependency(%q<activesupport>, [">= 2.3.4"])
     s.add_dependency(%q<andand>, [">= 1.3.1"])

data/lib/data_miner/attribute.rb

@@ -1,16 +1,16 @@
 module DataMiner
   class Attribute
-    attr_accessor :runnable
+    attr_accessor :step
     attr_accessor :name
     attr_accessor :options
 
-    delegate :resource, :to => :runnable
+    delegate :resource, :to => :step
 
-    def initialize(runnable, name, options = {})
+    def initialize(step, name, options = {})
       options.symbolize_keys!
       @options = options
 
-      @runnable = runnable
+      @step = step
       @name = name
     end
         

data/lib/data_miner/configuration.rb

@@ -2,23 +2,23 @@ module DataMiner
   class Configuration
     include Blockenspiel::DSL
     
-    attr_accessor :resource, :runnables, :runnable_counter, :attributes
+    attr_accessor :resource, :steps, :step_counter, :attributes
 
     def initialize(resource)
-      @runnables = Array.new
+      @steps = Array.new
       @resource = resource
-      @runnable_counter = 0
+      @step_counter = 0
       @attributes = HashWithIndifferentAccess.new
     end
     
     def process(method_name_or_block_description, &block)
-      runnables << DataMiner::Process.new(self, runnable_counter, method_name_or_block_description, &block)
-      self.runnable_counter += 1
+      steps << DataMiner::Process.new(self, step_counter, method_name_or_block_description, &block)
+      self.step_counter += 1
     end
 
     def clone(description, options = {})
-      runnables << DataMiner::Clone.new(self, runnable_counter, description, options)
-      self.runnable_counter += 1
+      steps << DataMiner::Clone.new(self, step_counter, description, options)
+      self.step_counter += 1
     end
 
     def import(*args, &block)
@@ -29,16 +29,19 @@ module DataMiner
       end
       options = args.last
         
-      runnable = DataMiner::Import.new self, runnable_counter, description, options
-      Blockenspiel.invoke block, runnable
-      runnables << runnable
-      self.runnable_counter += 1
+      step = DataMiner::Import.new self, step_counter, description, options
+      Blockenspiel.invoke block, step
+      steps << step
+      self.step_counter += 1
     end
 
     # Mine data for this class.
     def run(options = {})
       options.symbolize_keys!
       
+      return if DataMiner::Configuration.call_stack.include? resource.name
+      DataMiner::Configuration.call_stack.push resource.name
+      
       finished = false
       if DataMiner::Run.table_exists?
         run = DataMiner::Run.create! :started_at => Time.now, :resource_name => resource.name if DataMiner::Run.table_exists?
@@ -48,16 +51,17 @@ module DataMiner
       end
       resource.delete_all if options[:from_scratch]
       begin
-        runnables.each { |runnable| runnable.run run }
+        steps.each { |step| step.run run }
         finished = true
       ensure
         run.update_attributes! :ended_at => Time.now, :finished => finished if DataMiner::Run.table_exists?
+        DataMiner::Configuration.call_stack.clear if DataMiner::Configuration.call_stack.first == resource.name
       end
       nil
     end
     
-    def import_runnables
-      runnables.select { |runnable| runnable.is_a? Import }
+    def import_steps
+      steps.select { |step| step.is_a? Import }
     end
     
     def before_invoke
@@ -65,6 +69,7 @@ module DataMiner
     end
     
     def after_invoke
+      return unless resource.table_exists?
       make_sure_unit_definitions_make_sense
       suggest_missing_column_migrations
     end
@@ -79,8 +84,8 @@ module DataMiner
     ]
     
     def make_sure_unit_definitions_make_sense
-      import_runnables.each do |runnable|
-        runnable.attributes.each do |_, attribute|
+      import_steps.each do |step|
+        step.attributes.each do |_, attribute|
           if attribute.options.any? { |k, _| k.to_s =~ /unit/ } and COMPLETE_UNIT_DEFINITIONS.none? { |complete_definition| complete_definition.all? { |required_option| attribute.options[required_option].present? } }
             DataMiner.log_or_raise %{
 
@@ -111,8 +116,8 @@ You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence
         DataMiner.log_info "Not recording which run touched a row."
       end
       
-      import_runnables.each do |runnable|
-        runnable.attributes.each do |_, attribute|
+      import_steps.each do |step|
+        step.attributes.each do |_, attribute|
           DataMiner.log_or_raise "You can't have an attribute column that ends in _units (reserved): #{resource.table_name}.#{attribute.name}" if attribute.name.ends_with? '_units'
           unless resource.column_names.include? attribute.name
             missing_columns << attribute.name
@@ -160,6 +165,9 @@ On the other hand, if you're working directly with create_table, this might be h
     
     cattr_accessor :resource_names
     self.resource_names = Array.new
+    
+    cattr_accessor :call_stack
+    self.call_stack = Array.new
     class << self
       # Mine data. Defaults to all resource_names touched by DataMiner.
       #

data/lib/data_miner.rb

@@ -72,10 +72,7 @@ ActiveRecord::Base.class_eval do
   def self.data_miner(&block)
     DataMiner.start_logging
     
-    unless table_exists?
-      DataMiner.log_or_raise "Database table `#{table_name}` doesn't exist. DataMiner probably won't work properly until you run a migration or otherwise fix the schema."
-      return
-    end
+    DataMiner.log_info "Database table `#{table_name}` doesn't exist. It might be created in the data_miner block, but if it's not, DataMiner probably won't work properly until you run a migration or otherwise fix the schema." unless table_exists?
     
     DataMiner.resource_names.push self.name unless DataMiner.resource_names.include? self.name
 

data/test/data_miner_test.rb

@@ -488,6 +488,47 @@ class CensusDivision < ActiveRecord::Base
   end
 end
 
+class CrosscallingCensusRegion < ActiveRecord::Base
+  set_primary_key :number
+  
+  has_many :crosscalling_census_divisions
+  
+  data_miner do
+    process "derive ourselves from the census divisions table (i.e., cross call census divisions)" do
+      CrosscallingCensusDivision.run_data_miner!
+      connection.drop_table :crosscalling_census_regions rescue nil
+      connection.create_table :crosscalling_census_regions, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+        t.column :number, :integer
+        t.column :name, :string
+      end
+      connection.execute 'ALTER TABLE crosscalling_census_regions ADD PRIMARY KEY (number);'
+      connection.execute %{
+        INSERT IGNORE INTO crosscalling_census_regions(number, name)
+        SELECT crosscalling_census_divisions.census_region_number, crosscalling_census_divisions.census_region_name FROM crosscalling_census_divisions
+      }
+    end
+  end
+end
+
+class CrosscallingCensusDivision < ActiveRecord::Base
+  set_primary_key :number
+  
+  belongs_to :crosscalling_census_regions, :foreign_key => 'census_region_number'
+  
+  data_miner do
+    import "get a list of census divisions and their regions", :url => 'http://www.census.gov/popest/geographic/codes02.csv', :skip => 9, :select => lambda { |row| row['Division'].to_s.strip != 'X' and row['FIPS CODE STATE'].to_s.strip == 'X'} do
+      key 'number', :field_name => 'Division'
+      store 'name', :field_name => 'Name'
+      store 'census_region_number', :field_name => 'Region'
+      store 'census_region_name', :field_name => 'Region', :dictionary => { :input => 'number', :output => 'name', :url => 'http://data.brighterplanet.com/census_regions.csv' }
+    end
+    
+    process "make sure my parent object is set up (i.e., cross-call it)" do
+      CrosscallingCensusRegion.run_data_miner!
+    end
+  end
+end
+
 class ResidentialEnergyConsumptionSurveyResponse < ActiveRecord::Base
   set_primary_key :department_of_energy_identifier
   
@@ -906,6 +947,18 @@ class DataMinerTest < Test::Unit::TestCase
   end
     
   if ENV['FAST'] == 'true'
+    should "keep a call stack so that you can call run_data_miner! on a child" do
+      CrosscallingCensusDivision.run_data_miner!
+      assert CrosscallingCensusDivision.exists? :name => 'Mountain Division', :number => 8, :census_region_number => 4, :census_region_name => 'West Region'
+      assert CrosscallingCensusRegion.exists? :name => 'West Region', :number => 4
+    end
+    
+    should "keep a call stack so that you can call run_data_miner! on a parent" do
+      CrosscallingCensusRegion.run_data_miner!
+      assert CrosscallingCensusDivision.exists? :name => 'Mountain Division', :number => 8, :census_region_number => 4, :census_region_name => 'West Region'
+      assert CrosscallingCensusRegion.exists? :name => 'West Region', :number => 4
+    end
+    
     should "clone airports" do
       ClonedAirport.run_data_miner!
       assert ClonedAirport.count > 0
@@ -946,15 +999,15 @@ class DataMinerTest < Test::Unit::TestCase
     end
       
     should "hash things" do
-      AutomobileVariant.data_miner_config.runnables[0].run(nil)
+      AutomobileVariant.data_miner_config.steps[0].run(nil)
       assert AutomobileVariant.first.row_hash.present?
     end
   
     should "process a callback block instead of a method" do
       AutomobileVariant.delete_all
-      AutomobileVariant.data_miner_config.runnables[0].run(nil)
+      AutomobileVariant.data_miner_config.steps[0].run(nil)
       assert !AutomobileVariant.first.fuel_efficiency_city.present?
-      AutomobileVariant.data_miner_config.runnables.last.run(nil)
+      AutomobileVariant.data_miner_config.steps.last.run(nil)
       assert AutomobileVariant.first.fuel_efficiency_city.present?
     end
   

data/test/test_helper.rb

@@ -134,6 +134,19 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
   end
   execute 'ALTER TABLE census_divisions ADD PRIMARY KEY (number);'
   
+  create_table 'crosscalling_census_divisions', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.integer  'number'
+    t.string   'name'
+    t.datetime 'updated_at'
+    t.datetime 'created_at'
+    t.string   'census_region_name'
+    t.integer  'census_region_number'
+    
+    t.integer 'data_miner_touch_count'
+    t.integer 'data_miner_last_run_id'
+  end
+  execute 'ALTER TABLE crosscalling_census_divisions ADD PRIMARY KEY (number);'
+  
   create_table "automobile_variants", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
     t.float    "fuel_efficiency_city"
     t.float    "fuel_efficiency_highway"

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 4
-  - 17
-  version: 0.4.17
+  - 18
+  version: 0.4.18
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-15 00:00:00 -04:00
+date: 2010-04-16 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -28,8 +28,8 @@ dependencies:
         segments: 
         - 0
         - 2
-        - 9
-        version: 0.2.9
+        - 10
+        version: 0.2.10
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency