data_miner 1.1.8 → 1.2.0

data/README.rdoc

@@ -11,12 +11,7 @@ You define <tt>data_miner</tt> blocks in your ActiveRecord models. For example,
   class Country < ActiveRecord::Base
     set_primary_key :iso_3166_code
   
-    data_miner do
-      schema do
-        string   'iso_3166_code'
-        string   'name'
-      end
-    
+    data_miner do    
       import 'the official ISO country list',
              :url => 'http://www.iso.org/iso/list-en1-semic-3.txt',
              :skip => 2,
@@ -34,6 +29,36 @@ Now you can run:
   irb(main):001:0> Country.run_data_miner!
   => nil
 
+== Creating tables from scratch (changed in 1.2)
+
+We recommend using the <tt>create_table</tt> gem (https://github.com/seamusabshere/create_table)
+
+This replaces the <tt>schema</tt> method that was available before. It didn't make sense for <tt>data_miner</tt> to provide this natively.
+
+class Car < ActiveRecord::Base
+
+  # THE NEW WAY - depends on create_table gem, which is not required by default
+  # see the process step in the data_miner block where we actually call create_table!
+  create_table do
+    string :make
+    string :model
+  end
+  
+  data_miner do
+    # DEPRECATED - see above
+    # schema do
+    #   string :make
+    #   string :model
+    # end
+  
+    process "create the table, adding and removing columns as necessary" do
+      create_table!
+    end
+    
+    # [... other data mining steps]
+  end
+end
+
 ==Advanced usage
 
 This is how we linked together (http://data.brighterplanet.com/aircraft) the FAA's list of aircraft with the US Department of Transportations list of aircraft:
@@ -44,6 +69,36 @@ This is how we linked together (http://data.brighterplanet.com/aircraft) the FAA
     # table without breaking associations.
     set_primary_key :icao_code
   
+    # Use the create_table gem to define the database schema in-line.
+    # It will destructively and automatically add/remove columns.
+    # This is "OK" because you can always just re-run the import script to get the data back.
+    # PS. If you're using DataMapper, you don't need this
+    create_table do
+      string   'icao_code'
+      string   'manufacturer_name'
+      string   'name'
+      string   'bts_name'
+      string   'bts_aircraft_type_code'
+      string   'brighter_planet_aircraft_class_code'
+      string   'fuel_use_aircraft_name'
+      float    'm3'
+      string   'm3_units'
+      float    'm2'
+      string   'm2_units'
+      float    'm1'
+      string   'm1_units'
+      float    'endpoint_fuel'
+      string   'endpoint_fuel_units'
+      float    'seats'
+      float    'distance'
+      string   'distance_units'
+      float    'load_factor'
+      float    'freight_share'
+      float    'payload'
+      float    'weighting'
+      index    'bts_aircraft_type_code'
+    end
+  
     # A dictionary between BTS aircraft type codes and ICAO aircraft
     # codes that uses string similarity instead of exact matching.
     # This is preferable to typing everything out.
@@ -154,36 +209,6 @@ This is how we linked together (http://data.brighterplanet.com/aircraft) the FAA
         raise DataMiner::Skip unless DataMiner::Run.allowed? Aircraft
       end
     
-      # Define the database schema in-line.
-      # It will destructively and automatically add/remove columns.
-      # This is "OK" because you can always just re-run the import script to get the data back.
-      # PS. if we were using DataMapper, we wouldn't need this.
-      schema :options => 'ENGINE=InnoDB default charset=utf8' do
-        string   'icao_code'
-        string   'manufacturer_name'
-        string   'name'
-        string   'bts_name'
-        string   'bts_aircraft_type_code'
-        string   'brighter_planet_aircraft_class_code'
-        string   'fuel_use_aircraft_name'
-        float    'm3'
-        string   'm3_units'
-        float    'm2'
-        string   'm2_units'
-        float    'm1'
-        string   'm1_units'
-        float    'endpoint_fuel'
-        string   'endpoint_fuel_units'
-        float    'seats'
-        float    'distance'
-        string   'distance_units'
-        float    'load_factor'
-        float    'freight_share'
-        float    'payload'
-        float    'weighting'
-        index    'bts_aircraft_type_code'
-      end
-
       # The FAA publishes a document to help people identify aircraft by different names.
       ('A'..'Z').each do |letter|
         import( "ICAO aircraft codes starting with the letter #{letter} used by the FAA",

data/data_miner.gemspec

@@ -27,13 +27,14 @@ Gem::Specification.new do |s|
   s.add_dependency 'blockenspiel', '>=0.3.2'
   s.add_dependency 'taps', '>=0.3.11'
   s.add_dependency 'errata', '>=1.0.1'
+  s.add_development_dependency 'create_table', '>=0.0.2'
   s.add_development_dependency 'loose_tight_dictionary', ">=0.0.5"
   s.add_development_dependency 'test-unit'
   s.add_development_dependency 'shoulda'
   s.add_development_dependency 'mysql'
-  if RUBY_VERSION >= '1.9'
-    s.add_development_dependency 'ruby-debug19'
-  else
-    s.add_development_dependency 'ruby-debug'
-  end
+  # if RUBY_VERSION >= '1.9'
+  #   s.add_development_dependency 'ruby-debug19'
+  # else
+  #   s.add_development_dependency 'ruby-debug'
+  # end
 end

data/lib/data_miner/active_record_extensions.rb

@@ -6,8 +6,6 @@ class DataMiner
     def data_miner(options = {}, &blk)
       ::DataMiner.instance.start_logging
 
-      ::DataMiner.logger.debug "Database table `#{table_name}` doesn't exist. It might be created in the data_miner block, but if it's not, DataMiner probably won't work properly until you run a migration or otherwise fix the schema." unless table_exists?
-
       ::DataMiner.instance.resource_names.push self.name unless ::DataMiner.instance.resource_names.include? self.name
 
       # this is class_eval'ed here so that each ActiveRecord descendant has its own copy, or none at all
@@ -19,11 +17,6 @@ class DataMiner
         def self.run_data_miner!(options = {})
           data_miner_config.run options
         end
-        def self.execute_schema
-          if schema = data_miner_config.steps.detect { |s| s.instance_of?(::DataMiner::Schema) }
-            schema.run
-          end
-        end
       end
 
       if options[:append]

data/lib/data_miner/config.rb

@@ -18,13 +18,7 @@ class DataMiner
     # def attributes
     #   @attributes ||= {}
     # end
-    
-    def schema(create_table_options = {}, &blk)
-      step = Schema.new self, create_table_options
-      ::Blockenspiel.invoke blk, step
-      steps.push step
-    end
-    
+        
     def process(method_id_or_block_description, &blk)
       step = Process.new self, method_id_or_block_description, &blk
       steps.push step

data/lib/data_miner/process.rb

@@ -5,6 +5,8 @@ class DataMiner
     attr_reader :block_description
     attr_reader :blk
 
+    alias :description :block_description
+
     def initialize(config, method_id_or_block_description, &blk)
       @config = config
       if block_given?

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '1.1.8'
+  VERSION = '1.2.0'
 end

data/lib/data_miner.rb

@@ -26,7 +26,6 @@ class DataMiner
   autoload :Tap, 'data_miner/tap'
   autoload :Process, 'data_miner/process'
   autoload :Run, 'data_miner/run'
-  autoload :Schema, 'data_miner/schema'
   autoload :Verify, 'data_miner/verify'
   
   class << self

data/test/helper.rb

@@ -3,7 +3,8 @@ require 'bundler'
 Bundler.setup
 require 'test/unit'
 require 'shoulda'
-require 'ruby-debug'
+require 'create_table'
+# require 'ruby-debug'
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'data_miner'

data/test/support/automobile_variant.rb

@@ -42,8 +42,8 @@ class AutomobileVariant < ActiveRecord::Base
     }
 
     class ParserB
-      require 'slither'
-      ::Slither.define :fuel_economy_guide_b do |d|
+      require 'fixed_width'
+      ::FixedWidth.define :fuel_economy_guide_b do |d|
         d.rows do |row|
           row.trap { true } # there's only one section
           row.column 'active_year'      , 4,    :type => :integer  #   ACTIVE YEAR

data/test/test_old_syntax.rb

@@ -567,18 +567,22 @@ end
 class AutomobileMakeFleetYear < ActiveRecord::Base
   set_primary_key :name
 
+  create_table do
+    string "name"
+    string   "make_name"
+    string   "fleet"
+    integer   "year"
+    float    "fuel_efficiency"
+    string "fuel_efficiency_units"
+    integer  "volume"
+    string  "make_year_name"
+    datetime "created_at"
+    datetime "updated_at"
+  end
+
   data_miner do
-    schema :id => false do
-      string "name"
-      string   "make_name"
-      string   "fleet"
-      integer   "year"
-      float    "fuel_efficiency"
-      string "fuel_efficiency_units"
-      integer  "volume"
-      string  "make_year_name"
-      datetime "created_at"
-      datetime "updated_at"
+    process "create table" do
+      create_table!
     end
     
     process "finish if i tell you to" do
@@ -605,298 +609,307 @@ end
 
 class CensusDivisionTrois < ActiveRecord::Base
   set_primary_key :number_code
+  
+  create_table do
+    string  'number_code'
+    string   'name'
+    string   'census_region_name'
+    integer  'census_region_number'
+    index    'census_region_name', :name => 'homefry'
+    index   ['number_code', 'name', 'census_region_name', 'census_region_number']
+  end
+  
   data_miner do
-    schema :options => 'ENGINE=InnoDB default charset=utf8' do
-      string  'number_code'
-      string   'name'
-      string   'census_region_name'
-      integer  'census_region_number'
-      index    'census_region_name', :name => 'homefry'
-      index   ['number_code', 'name', 'census_region_name', 'census_region_number']
+    process "create table" do
+      create_table!
     end
   end
 end
 
 class CensusDivisionFour < ActiveRecord::Base
+  create_table do
+    string  'number_code'
+    string   'name'
+    string   'census_region_name'
+    integer  'census_region_number'
+    index    'census_region_name', :name => 'homefry'
+  end
+
   data_miner do
-    schema do
-      string  'number_code'
-      string   'name'
-      string   'census_region_name'
-      integer  'census_region_number'
-      index    'census_region_name', :name => 'homefry'
+    process "create table" do
+      create_table!
     end
   end
 end
 
 # todo: have somebody properly organize these
 class TestOldSyntax < Test::Unit::TestCase
-  # if ENV['WIP']
-  #   context 'with nullify option' do
-  #     should 'treat blank fields as null values' do
-  #       Aircraft.delete_all
-  #       Aircraft.data_miner_runs.delete_all
-  #       Aircraft.run_data_miner!
-  #       assert_greater_than 0, Aircraft.count
-  #       assert_false Aircraft.where(:brighter_planet_aircraft_class_code => nil).empty?
-  #     end
-  #   end
-  # end
-  # 
-  # if ENV['ALL'] == 'true'
-  #   should 'directly create a table for the model' do
-  #     if AutomobileMakeFleetYear.table_exists?
-  #       ActiveRecord::Base.connection.execute 'DROP TABLE automobile_make_fleet_years;'
-  #     end
-  #     AutomobileMakeFleetYear.execute_schema
-  #     assert AutomobileMakeFleetYear.table_exists?
-  #   end
-  # end
-  #   
-  # if ENV['ALL'] == 'true' or ENV['FAST'] == 'true'
-  #   should 'append to an existing config' do
-  #     AutomobileFuelType.class_eval do
-  #       data_miner :append => true do
-  #         import 'example1', :url => 'http://example1.com' do
-  #           key 'code'
-  #           store 'name'
-  #         end
-  #       end
-  #       data_miner :append => true do
-  #         import 'example2', :url => 'http://example2.com' do
-  #           key 'code'
-  #           store 'name'
-  #         end
-  #       end
-  #     end
-  #     assert_equal 'http://example1.com', AutomobileFuelType.data_miner_config.steps[-2].table.url
-  #     assert_equal 'http://example2.com', AutomobileFuelType.data_miner_config.steps[-1].table.url
-  #   end
-  #   
-  #   should 'override an existing data_miner configuration' do
-  #     AutomobileFuelType.class_eval do
-  #       data_miner do
-  #         import 'example', :url => 'http://example.com' do
-  #           key 'code'
-  #           store 'name'
-  #         end
-  #       end
-  #     end
-  #     assert_kind_of DataMiner::Import, AutomobileFuelType.data_miner_config.steps.first
-  #     assert_equal 'http://example.com', AutomobileFuelType.data_miner_config.steps.first.table.url
-  #   end
-  #   should "stop and finish if it gets a DataMiner::Finish" do
-  #     AutomobileMakeFleetYear.delete_all
-  #     AutomobileMakeFleetYear.data_miner_runs.delete_all
-  #     $force_finish = true
-  #     AutomobileMakeFleetYear.run_data_miner!
-  #     assert_equal 0, AutomobileMakeFleetYear.count
-  #     assert (AutomobileMakeFleetYear.data_miner_runs.count > 0)
-  #     assert AutomobileMakeFleetYear.data_miner_runs.all? { |run| run.finished? and not run.skipped and not run.killed? }
-  #     $force_finish = false
-  #     AutomobileMakeFleetYear.run_data_miner!
-  #     assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
-  #   end
-  #   
-  #   should "stop and register skipped if it gets a DataMiner::Skip" do
-  #     AutomobileMakeFleetYear.delete_all
-  #     AutomobileMakeFleetYear.data_miner_runs.delete_all
-  #     $force_skip = true
-  #     AutomobileMakeFleetYear.run_data_miner!
-  #     assert_equal 0, AutomobileMakeFleetYear.count
-  #     assert (AutomobileMakeFleetYear.data_miner_runs.count > 0)
-  #     assert AutomobileMakeFleetYear.data_miner_runs.all? { |run| run.skipped? and not run.finished? and not run.killed? }
-  #     $force_skip = false
-  #     AutomobileMakeFleetYear.run_data_miner!
-  #     assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
-  #   end
-  #   
-  #   should "eagerly enforce a schema" do
-  #     ActiveRecord::Base.connection.create_table 'census_division_trois', :force => true, :options => 'ENGINE=InnoDB default charset=utf8' do |t|
-  #       t.string   'name'
-  #       t.string   'census_region_name'
-  #       # t.integer  'census_region_number'
-  #     end
-  #     ActiveRecord::Base.connection.execute 'ALTER TABLE census_division_trois ADD INDEX (census_region_name)'
-  #     CensusDivisionTrois.reset_column_information
-  #     missing_columns = %w{ census_region_number }
-  # 
-  #     # sanity check
-  #     missing_columns.each do |column|
-  #       assert_false CensusDivisionTrois.column_names.include?(column)
-  #     end
-  #     assert_false ActiveRecord::Base.connection.indexes(CensusDivisionTrois.table_name).any? { |index| index.name == 'homefry' }
-  #     
-  #     3.times do
-  #       CensusDivisionTrois.run_data_miner!
-  #       missing_columns.each do |column|
-  #         assert CensusDivisionTrois.column_names.include?(column)
-  #       end
-  #       assert ActiveRecord::Base.connection.indexes(CensusDivisionTrois.table_name).any? { |index| index.name == 'homefry' }
-  #       assert_equal :string, CensusDivisionTrois.columns_hash[CensusDivisionTrois.primary_key].type
-  #     end
-  #   end
-  #   
-  #   should "let schemas work with default id primary keys" do
-  #     ActiveRecord::Base.connection.create_table 'census_division_fours', :force => true, :options => 'ENGINE=InnoDB default charset=utf8' do |t|
-  #       t.string   'name'
-  #       t.string   'census_region_name'
-  #       # t.integer  'census_region_number'
-  #     end
-  #     ActiveRecord::Base.connection.execute 'ALTER TABLE census_division_fours ADD INDEX (census_region_name)'
-  #     CensusDivisionFour.reset_column_information
-  #     missing_columns = %w{ census_region_number }
-  #   
-  #     # sanity check
-  #     missing_columns.each do |column|
-  #       assert_false CensusDivisionFour.column_names.include?(column)
-  #     end
-  #     assert_false ActiveRecord::Base.connection.indexes(CensusDivisionFour.table_name).any? { |index| index.name == 'homefry' }
-  #     
-  #     3.times do
-  #       CensusDivisionFour.run_data_miner!
-  #       missing_columns.each do |column|
-  #         assert CensusDivisionFour.column_names.include?(column)
-  #       end
-  #       assert ActiveRecord::Base.connection.indexes(CensusDivisionFour.table_name).any? { |index| index.name == 'homefry' }
-  #       assert_equal :integer, CensusDivisionFour.columns_hash[CensusDivisionFour.primary_key].type
-  #     end
-  #   end
+  if ENV['WIP']
+    context 'with nullify option' do
+      should 'treat blank fields as null values' do
+        Aircraft.delete_all
+        Aircraft.data_miner_runs.delete_all
+        Aircraft.run_data_miner!
+        assert_greater_than 0, Aircraft.count
+        assert_false Aircraft.where(:brighter_planet_aircraft_class_code => nil).empty?
+      end
+    end
+  end
+
+  if ENV['ALL'] == 'true'
+    should 'directly create a table for the model' do
+      if AutomobileMakeFleetYear.table_exists?
+        ActiveRecord::Base.connection.execute 'DROP TABLE automobile_make_fleet_years;'
+      end
+      AutomobileMakeFleetYear.create_table!
+      assert AutomobileMakeFleetYear.table_exists?
+    end
+  end
+    
+  if ENV['ALL'] == 'true' or ENV['FAST'] == 'true'
+    should 'append to an existing config' do
+      AutomobileFuelType.class_eval do
+        data_miner :append => true do
+          import 'example1', :url => 'http://example1.com' do
+            key 'code'
+            store 'name'
+          end
+        end
+        data_miner :append => true do
+          import 'example2', :url => 'http://example2.com' do
+            key 'code'
+            store 'name'
+          end
+        end
+      end
+      assert_equal 'http://example1.com', AutomobileFuelType.data_miner_config.steps[-2].table.url
+      assert_equal 'http://example2.com', AutomobileFuelType.data_miner_config.steps[-1].table.url
+    end
+    
+    should 'override an existing data_miner configuration' do
+      AutomobileFuelType.class_eval do
+        data_miner do
+          import 'example', :url => 'http://example.com' do
+            key 'code'
+            store 'name'
+          end
+        end
+      end
+      assert_kind_of DataMiner::Import, AutomobileFuelType.data_miner_config.steps.first
+      assert_equal 'http://example.com', AutomobileFuelType.data_miner_config.steps.first.table.url
+    end
+    should "stop and finish if it gets a DataMiner::Finish" do
+      AutomobileMakeFleetYear.delete_all
+      AutomobileMakeFleetYear.data_miner_runs.delete_all
+      $force_finish = true
+      AutomobileMakeFleetYear.run_data_miner!
+      assert_equal 0, AutomobileMakeFleetYear.count
+      assert (AutomobileMakeFleetYear.data_miner_runs.count > 0)
+      assert AutomobileMakeFleetYear.data_miner_runs.all? { |run| run.finished? and not run.skipped and not run.killed? }
+      $force_finish = false
+      AutomobileMakeFleetYear.run_data_miner!
+      assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
+    end
+    
+    should "stop and register skipped if it gets a DataMiner::Skip" do
+      AutomobileMakeFleetYear.delete_all
+      AutomobileMakeFleetYear.data_miner_runs.delete_all
+      $force_skip = true
+      AutomobileMakeFleetYear.run_data_miner!
+      assert_equal 0, AutomobileMakeFleetYear.count
+      assert (AutomobileMakeFleetYear.data_miner_runs.count > 0)
+      assert AutomobileMakeFleetYear.data_miner_runs.all? { |run| run.skipped? and not run.finished? and not run.killed? }
+      $force_skip = false
+      AutomobileMakeFleetYear.run_data_miner!
+      assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
+    end
+    
+    should "eagerly enforce a schema" do
+      ActiveRecord::Base.connection.create_table 'census_division_trois', :force => true, :options => 'ENGINE=InnoDB default charset=utf8' do |t|
+        t.string   'name'
+        t.string   'census_region_name'
+        # t.integer  'census_region_number'
+      end
+      ActiveRecord::Base.connection.execute 'ALTER TABLE census_division_trois ADD INDEX (census_region_name)'
+      CensusDivisionTrois.reset_column_information
+      missing_columns = %w{ census_region_number }
+
+      # sanity check
+      missing_columns.each do |column|
+        assert_false CensusDivisionTrois.column_names.include?(column)
+      end
+      assert_false ActiveRecord::Base.connection.indexes(CensusDivisionTrois.table_name).any? { |index| index.name == 'homefry' }
+      
+      3.times do
+        CensusDivisionTrois.run_data_miner!
+        missing_columns.each do |column|
+          assert CensusDivisionTrois.column_names.include?(column)
+        end
+        assert ActiveRecord::Base.connection.indexes(CensusDivisionTrois.table_name).any? { |index| index.name == 'homefry' }
+        assert_equal :string, CensusDivisionTrois.columns_hash[CensusDivisionTrois.primary_key].type
+      end
+    end
+    
+    should "let schemas work with default id primary keys" do
+      ActiveRecord::Base.connection.create_table 'census_division_fours', :force => true, :options => 'ENGINE=InnoDB default charset=utf8' do |t|
+        t.string   'name'
+        t.string   'census_region_name'
+        # t.integer  'census_region_number'
+      end
+      ActiveRecord::Base.connection.execute 'ALTER TABLE census_division_fours ADD INDEX (census_region_name)'
+      CensusDivisionFour.reset_column_information
+      missing_columns = %w{ census_region_number }
+    
+      # sanity check
+      missing_columns.each do |column|
+        assert_false CensusDivisionFour.column_names.include?(column)
+      end
+      assert_false ActiveRecord::Base.connection.indexes(CensusDivisionFour.table_name).any? { |index| index.name == 'homefry' }
+      
+      3.times do
+        CensusDivisionFour.run_data_miner!
+        missing_columns.each do |column|
+          assert CensusDivisionFour.column_names.include?(column)
+        end
+        assert ActiveRecord::Base.connection.indexes(CensusDivisionFour.table_name).any? { |index| index.name == 'homefry' }
+        assert_equal :integer, CensusDivisionFour.columns_hash[CensusDivisionFour.primary_key].type
+      end
+    end
     
     should "allow specifying dictionaries explicitly" do
       CensusDivisionDeux.run_data_miner!
       assert_equal 'South Region', CensusDivisionDeux.find(5).census_region_name
     end
     
-  #   should "be able to key on things other than the primary key" do
-  #     Aircraft.run_data_miner!
-  #     assert_equal 'SP', Aircraft.find('DHC6').brighter_planet_aircraft_class_code
-  #   end
-  #   
-  #   should "be able to synthesize rows without using a full parser class" do
-  #     AutomobileMakeFleetYear.run_data_miner!
-  #     assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
-  #   end
-  #   
-  #   should "keep a call stack so that you can call run_data_miner! on a child" do
-  #     CrosscallingCensusDivision.run_data_miner!
-  #     assert CrosscallingCensusDivision.exists? :name => 'Mountain Division', :number => 8, :census_region_number => 4, :census_region_name => 'West Region'
-  #     assert CrosscallingCensusRegion.exists? :name => 'West Region', :number => 4
-  #   end
-  #   
-  #   should "keep a call stack so that you can call run_data_miner! on a parent" do
-  #     CrosscallingCensusRegion.run_data_miner!
-  #     assert CrosscallingCensusDivision.exists? :name => 'Mountain Division', :number => 8, :census_region_number => 4, :census_region_name => 'West Region'
-  #     assert CrosscallingCensusRegion.exists? :name => 'West Region', :number => 4
-  #   end
-  #       
-  #   should "import airports" do
-  #     Airport.run_data_miner!
-  #     assert Airport.count > 0
-  #   end
-  #   
-  #   should "tap airports" do
-  #     TappedAirport.run_data_miner!
-  #     assert TappedAirport.count > 0
-  #   end
-  #   
-  #   should "pull in census divisions using a data.brighterplanet.com dictionary" do
-  #     CensusDivision.run_data_miner!
-  #     assert CensusDivision.count > 0
-  #   end
-  #   
-  #   should "have a way to queue up runs that works with delated_job's send_later" do
-  #     assert AutomobileVariant.respond_to?(:run_data_miner!)
-  #   end
-  #   
-  #   should "be idempotent" do
-  #     Country.data_miner_config.run
-  #     a = Country.count
-  #     Country.data_miner_config.run
-  #     b = Country.count
-  #     assert_equal a, b
-  #   
-  #     CensusRegion.data_miner_config.run
-  #     a = CensusRegion.count
-  #     CensusRegion.data_miner_config.run
-  #     b = CensusRegion.count
-  #     assert_equal a, b
-  #   end
-  #     
-  #   should "hash things" do
-  #     AutomobileVariant.data_miner_config.steps[0].run
-  #     assert AutomobileVariant.first.row_hash.present?
-  #   end
-  # 
-  #   should "process a callback block instead of a method" do
-  #     AutomobileVariant.delete_all
-  #     AutomobileVariant.data_miner_config.steps[0].run
-  #     assert !AutomobileVariant.first.fuel_efficiency_city.present?
-  #     AutomobileVariant.data_miner_config.steps.last.run
-  #     assert AutomobileVariant.first.fuel_efficiency_city.present?
-  #   end
-  # 
-  #   should "keep a log when it does a run" do
-  #     approx_started_at = Time.now
-  #     DataMiner.run :resource_names => %w{ Country }
-  #     approx_terminated_at = Time.now
-  #     last_run = DataMiner::Run.first(:conditions => { :resource_name => 'Country' }, :order => 'id DESC')
-  #     assert (last_run.started_at - approx_started_at).abs < 5 # seconds
-  #     assert (last_run.terminated_at - approx_terminated_at).abs < 5 # seconds
-  #   end
-  # 
-  #   should "request a re-import from scratch" do
-  #     c = Country.new
-  #     c.iso_3166 = 'JUNK'
-  #     c.save!
-  #     assert Country.exists?(:iso_3166 => 'JUNK')
-  #     DataMiner.run :resource_names => %w{ Country }, :from_scratch => true
-  #     assert !Country.exists?(:iso_3166 => 'JUNK')
-  #   end
-  #   
-  #   should "know what runs were on a resource" do
-  #     DataMiner.run :resource_names => %w{ Country }
-  #     DataMiner.run :resource_names => %w{ Country }
-  #     assert Country.data_miner_runs.count > 0
-  #   end
-  # end
-  # 
-  # if ENV['ALL'] == 'true' or ENV['SLOW'] == 'true'
-  #   should "allow errata to be specified with a shorthand, assuming the responder is the resource class itself" do
-  #     AircraftDeux.run_data_miner!
-  #     assert AircraftDeux.exists? :icao_code => 'DC91', :bts_aircraft_type_code => '630'
-  #   end
-  #   
-  #   should "mine aircraft" do
-  #     Aircraft.run_data_miner!
-  #     assert Aircraft.exists? :icao_code => 'DC91', :bts_aircraft_type_code => '630'
-  #   end
-  #   
-  #   should "mine automobile variants" do
-  #     AutomobileVariant.run_data_miner!
-  #     assert AutomobileVariant.count('make_name LIKE "%tesla"') > 0
-  #   end
-  #   
-  #   should "mine T100 flight segments" do
-  #     T100FlightSegment.run_data_miner!
-  #     assert T100FlightSegment.count('dest_country_name LIKE "%United States"') > 0
-  #   end
-  #   
-  #   should "mine residence survey responses" do
-  #     ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
-  #     assert ResidentialEnergyConsumptionSurveyResponse.find(6).residence_class.start_with?('Single-family detached house')
-  #   end
-  # end
-  # should "mark the run as skipped if verification fails" do
-  #   AutomobileFuelType.data_miner_config.instance_eval do
-  #     verify "failure" do
-  #       false
-  #     end
-  #   end
-  # 
-  #   DataMiner::Run.delete_all
-  #   assert_raise do
-  #     AutomobileFuelType.run_data_miner! :from_scratch => true
-  #   end
-  # end
+    should "be able to key on things other than the primary key" do
+      Aircraft.run_data_miner!
+      assert_equal 'SP', Aircraft.find('DHC6').brighter_planet_aircraft_class_code
+    end
+    
+    should "be able to synthesize rows without using a full parser class" do
+      AutomobileMakeFleetYear.run_data_miner!
+      assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
+    end
+    
+    should "keep a call stack so that you can call run_data_miner! on a child" do
+      CrosscallingCensusDivision.run_data_miner!
+      assert CrosscallingCensusDivision.exists? :name => 'Mountain Division', :number => 8, :census_region_number => 4, :census_region_name => 'West Region'
+      assert CrosscallingCensusRegion.exists? :name => 'West Region', :number => 4
+    end
+    
+    should "keep a call stack so that you can call run_data_miner! on a parent" do
+      CrosscallingCensusRegion.run_data_miner!
+      assert CrosscallingCensusDivision.exists? :name => 'Mountain Division', :number => 8, :census_region_number => 4, :census_region_name => 'West Region'
+      assert CrosscallingCensusRegion.exists? :name => 'West Region', :number => 4
+    end
+        
+    should "import airports" do
+      Airport.run_data_miner!
+      assert Airport.count > 0
+    end
+    
+    should "tap airports" do
+      TappedAirport.run_data_miner!
+      assert TappedAirport.count > 0
+    end
+    
+    should "pull in census divisions using a data.brighterplanet.com dictionary" do
+      CensusDivision.run_data_miner!
+      assert CensusDivision.count > 0
+    end
+    
+    should "have a way to queue up runs that works with delated_job's send_later" do
+      assert AutomobileVariant.respond_to?(:run_data_miner!)
+    end
+    
+    should "be idempotent" do
+      Country.data_miner_config.run
+      a = Country.count
+      Country.data_miner_config.run
+      b = Country.count
+      assert_equal a, b
+    
+      CensusRegion.data_miner_config.run
+      a = CensusRegion.count
+      CensusRegion.data_miner_config.run
+      b = CensusRegion.count
+      assert_equal a, b
+    end
+      
+    should "hash things" do
+      AutomobileVariant.data_miner_config.steps[0].run
+      assert AutomobileVariant.first.row_hash.present?
+    end
+  
+    should "process a callback block instead of a method" do
+      AutomobileVariant.delete_all
+      AutomobileVariant.data_miner_config.steps[0].run
+      assert !AutomobileVariant.first.fuel_efficiency_city.present?
+      AutomobileVariant.data_miner_config.steps.last.run
+      assert AutomobileVariant.first.fuel_efficiency_city.present?
+    end
+  
+    should "keep a log when it does a run" do
+      approx_started_at = Time.now
+      DataMiner.run :resource_names => %w{ Country }
+      approx_terminated_at = Time.now
+      last_run = DataMiner::Run.first(:conditions => { :resource_name => 'Country' }, :order => 'id DESC')
+      assert (last_run.started_at - approx_started_at).abs < 5 # seconds
+      assert (last_run.terminated_at - approx_terminated_at).abs < 5 # seconds
+    end
+  
+    should "request a re-import from scratch" do
+      c = Country.new
+      c.iso_3166 = 'JUNK'
+      c.save!
+      assert Country.exists?(:iso_3166 => 'JUNK')
+      DataMiner.run :resource_names => %w{ Country }, :from_scratch => true
+      assert !Country.exists?(:iso_3166 => 'JUNK')
+    end
+    
+    should "know what runs were on a resource" do
+      DataMiner.run :resource_names => %w{ Country }
+      DataMiner.run :resource_names => %w{ Country }
+      assert Country.data_miner_runs.count > 0
+    end
+  end
+  
+  if ENV['ALL'] == 'true' or ENV['SLOW'] == 'true'
+    should "allow errata to be specified with a shorthand, assuming the responder is the resource class itself" do
+      AircraftDeux.run_data_miner!
+      assert AircraftDeux.exists? :icao_code => 'DC91', :bts_aircraft_type_code => '630'
+    end
+    
+    should "mine aircraft" do
+      Aircraft.run_data_miner!
+      assert Aircraft.exists? :icao_code => 'DC91', :bts_aircraft_type_code => '630'
+    end
+    
+    should "mine automobile variants" do
+      AutomobileVariant.run_data_miner!
+      assert AutomobileVariant.count('make_name LIKE "%tesla"') > 0
+    end
+    
+    should "mine T100 flight segments" do
+      T100FlightSegment.run_data_miner!
+      assert T100FlightSegment.count('dest_country_name LIKE "%United States"') > 0
+    end
+    
+    should "mine residence survey responses" do
+      ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
+      assert ResidentialEnergyConsumptionSurveyResponse.find(6).residence_class.start_with?('Single-family detached house')
+    end
+  end
+  should "mark the run as skipped if verification fails" do
+    AutomobileFuelType.data_miner_config.instance_eval do
+      verify "failure" do
+        false
+      end
+    end
+
+    DataMiner::Run.delete_all
+    assert_raise do
+      AutomobileFuelType.run_data_miner! :from_scratch => true
+    end
+  end
 end

metadata

@@ -1,13 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: data_miner
 version: !ruby/object:Gem::Version 
-  hash: 3
   prerelease: 
-  segments: 
-  - 1
-  - 1
-  - 8
-  version: 1.1.8
+  version: 1.2.0
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -17,7 +12,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-06 00:00:00 Z
+date: 2011-06-06 00:00:00 -05:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: remote_table
@@ -27,11 +23,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 27
-        segments: 
-        - 1
-        - 2
-        - 2
         version: 1.2.2
   type: :runtime
   version_requirements: *id001
@@ -43,11 +34,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 0
-        - 0
-        - 4
         version: 0.0.4
   type: :runtime
   version_requirements: *id002
@@ -59,11 +45,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 2
-        - 3
-        - 4
         version: 2.3.4
   type: :runtime
   version_requirements: *id003
@@ -75,11 +56,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 2
-        - 3
-        - 4
         version: 2.3.4
   type: :runtime
   version_requirements: *id004
@@ -91,11 +67,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 15
-        segments: 
-        - 1
-        - 4
-        - 4
         version: 1.4.4
   type: :runtime
   version_requirements: *id005
@@ -107,11 +78,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 0
-        - 3
-        - 2
         version: 0.3.2
   type: :runtime
   version_requirements: *id006
@@ -123,11 +89,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 5
-        segments: 
-        - 0
-        - 3
-        - 11
         version: 0.3.11
   type: :runtime
   version_requirements: *id007
@@ -139,83 +100,61 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 21
-        segments: 
-        - 1
-        - 0
-        - 1
         version: 1.0.1
   type: :runtime
   version_requirements: *id008
 - !ruby/object:Gem::Dependency 
-  name: loose_tight_dictionary
+  name: create_table
   prerelease: false
   requirement: &id009 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 21
-        segments: 
-        - 0
-        - 0
-        - 5
-        version: 0.0.5
+        version: 0.0.2
   type: :development
   version_requirements: *id009
 - !ruby/object:Gem::Dependency 
-  name: test-unit
+  name: loose_tight_dictionary
   prerelease: false
   requirement: &id010 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+        version: 0.0.5
   type: :development
   version_requirements: *id010
 - !ruby/object:Gem::Dependency 
-  name: shoulda
+  name: test-unit
   prerelease: false
   requirement: &id011 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id011
 - !ruby/object:Gem::Dependency 
-  name: mysql
+  name: shoulda
   prerelease: false
   requirement: &id012 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id012
 - !ruby/object:Gem::Dependency 
-  name: ruby-debug
+  name: mysql
   prerelease: false
   requirement: &id013 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id013
@@ -245,7 +184,6 @@ files:
 - lib/data_miner/import.rb
 - lib/data_miner/process.rb
 - lib/data_miner/run.rb
-- lib/data_miner/schema.rb
 - lib/data_miner/tap.rb
 - lib/data_miner/verify.rb
 - lib/data_miner/version.rb
@@ -260,6 +198,7 @@ files:
 - test/test_data_miner_process.rb
 - test/test_data_miner_verify.rb
 - test/test_old_syntax.rb
+has_rdoc: true
 homepage: https://github.com/seamusabshere/data_miner
 licenses: []
 
@@ -273,23 +212,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: data_miner
-rubygems_version: 1.7.2
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Mine remote data into your ActiveRecord models.

data/lib/data_miner/schema.rb

@@ -1,241 +0,0 @@
-require 'blockenspiel'
-class DataMiner
-  class Schema
-    include ::Blockenspiel::DSL
-    
-    attr_reader :config
-    attr_reader :create_table_options
-
-    def initialize(config, create_table_options)
-      @config = config
-      @create_table_options = create_table_options.dup
-      @create_table_options.stringify_keys!
-      raise "'id' => true is not allowed in create_table_options." if @create_table_options['id'] === true
-      raise "'primary_key' is not allowed in create_table_options. Use set_primary_key instead." if @create_table_options.has_key?('primary_key')
-      @create_table_options['id'] = false # always
-    end
-    
-    def resource
-      config.resource
-    end
-    
-    # sabshere 1/25/11 what if there were multiple connections
-    # blockenspiel doesn't like to delegate this to #resource
-    def connection
-      ::ActiveRecord::Base.connection
-    end
-    
-    def table_name
-      resource.table_name
-    end
-    
-    def ideal_table
-      @ideal_table ||= ::ActiveRecord::ConnectionAdapters::TableDefinition.new connection
-    end
-    
-    def ideal_indexes
-      @ideal_indexes ||= []
-    end
-    
-    def actual_indexes
-      connection.indexes table_name
-    end
-    
-    def description
-      "Define a table called #{table_name} with primary key #{ideal_primary_key_name}"
-    end
-    
-    def inspect
-      %{#<DataMiner::Schema(#{resource}): #{description}>}
-    end
-    
-    # sabshere 1/25/11 lifted straight from activerecord-3.0.3/lib/active_record/connection_adapters/abstract/schema_definitions.rb
-    %w( string text integer float decimal datetime timestamp time date binary boolean ).each do |column_type|
-      class_eval <<-EOV
-        def #{column_type}(*args)                                               # def string(*args)
-          options = args.extract_options!                                       #   options = args.extract_options!
-          column_names = args                                                   #   column_names = args
-                                                                                #
-          column_names.each { |name| ideal_table.column(name, '#{column_type}', options) }  #   column_names.each { |name| ideal_table.column(name, 'string', options) }
-        end                                                                     # end
-      EOV
-    end
-    def column(*args)
-      ideal_table.column(*args)
-    end
-    
-    MAX_INDEX_NAME_LENGTH = 32
-    def index(columns, options = {})
-      options = options.dup
-      options.stringify_keys!
-      columns = ::Array.wrap columns
-      unless name = options['name']
-        default_name = connection.index_name(table_name, options.symbolize_keys.merge(:column => columns))
-        name = default_name.length < MAX_INDEX_NAME_LENGTH ? default_name : default_name[0..MAX_INDEX_NAME_LENGTH-11] + ::Zlib.crc32(default_name).to_s
-      end
-      index_unique = options.has_key?('unique') ? options['unique'] : true
-      ideal_indexes.push ::ActiveRecord::ConnectionAdapters::IndexDefinition.new(table_name, name, index_unique, columns)
-      nil
-    end
-    
-    def ideal_primary_key_name
-      resource.primary_key.to_s
-    end
-    
-    def actual_primary_key_name
-      connection.primary_key(table_name).to_s
-    end
-        
-    INDEX_PROPERTIES = %w{ name columns }
-    def index_equivalent?(a, b)
-      return false unless a and b
-      INDEX_PROPERTIES.all? do |property|
-        ::DataMiner.logger.debug "...comparing #{a.send(property).inspect}.to_s <-> #{b.send(property).inspect}.to_s"
-        a.send(property).to_s == b.send(property).to_s
-      end
-    end
-
-    # FIXME mysql only (assume integer primary keys)
-    def column_equivalent?(a, b)
-      return false unless a and b
-      a_type = a.type.to_s == 'primary_key' ? 'integer' : a.type.to_s
-      b_type = b.type.to_s == 'primary_key' ? 'integer' : b.type.to_s
-      a_type == b_type and a.name.to_s == b.name.to_s
-    end
-
-    %w{ column index }.each do |i|
-      eval %{
-        def #{i}_needs_to_be_placed?(name)
-          actual = actual_#{i} name
-          return true unless actual
-          ideal = ideal_#{i} name
-          not #{i}_equivalent? actual, ideal
-        end
-    
-        def #{i}_needs_to_be_removed?(name)
-          ideal_#{i}(name).nil?
-        end
-      }
-    end
-    
-    def ideal_column(name)
-      ideal_table[name.to_s]
-    end
-    
-    def actual_column(name)
-      resource.columns_hash[name.to_s]
-    end
-    
-    def ideal_index(name)
-      ideal_indexes.detect { |ideal| ideal.name == name.to_s }
-    end
-    
-    def actual_index(name)
-      actual_indexes.detect { |actual| actual.name == name.to_s }
-    end
-  
-    def place_column(name)
-      remove_column name if actual_column name
-      ideal = ideal_column name
-      ::DataMiner.logger.debug "ADDING COLUMN #{name}"
-      connection.add_column table_name, name, ideal.type.to_sym # symbol type!
-      resource.reset_column_information
-    end
-    
-    def remove_column(name)
-      ::DataMiner.logger.debug "REMOVING COLUMN #{name}"
-      connection.remove_column table_name, name
-      resource.reset_column_information
-    end
-    
-    def place_index(name)
-      remove_index name if actual_index name
-      ideal = ideal_index name
-      ::DataMiner.logger.debug "ADDING INDEX #{name}"
-      connection.add_index table_name, ideal.columns, :name => ideal.name
-      resource.reset_column_information
-    end
-    
-    def remove_index(name)
-      ::DataMiner.logger.debug "REMOVING INDEX #{name}"
-      connection.remove_index table_name, :name => name
-      resource.reset_column_information
-    end
-    
-    def run
-      _create_table
-      _set_primary_key
-      _remove_columns
-      _add_columns
-      _remove_indexes
-      _add_indexes
-      nil
-    end
-        
-    def _create_table
-      if not resource.table_exists?
-        create_table_options = @create_table_options.dup
-        create_table_options.symbolize_keys!
-        ::DataMiner.logger.debug "CREATING TABLE #{table_name} with #{create_table_options.inspect}"
-        connection.create_table table_name, create_table_options do |t|
-          t.integer 'data_miner_placeholder'
-        end
-        resource.reset_column_information
-      end
-    end
-    
-    # FIXME mysql only
-    def _set_primary_key
-      if ideal_primary_key_name == 'id' and not ideal_column('id')
-        ::DataMiner.logger.debug "no special primary key set on #{table_name}, so using 'id'"
-        column 'id', :primary_key # needs to be a sym?
-      end
-      actual = actual_column actual_primary_key_name
-      ideal = ideal_column ideal_primary_key_name
-      if not column_equivalent? actual, ideal
-        ::DataMiner.logger.debug "looks like #{table_name} has a bad (or missing) primary key"
-        if actual
-          ::DataMiner.logger.debug "looks like primary key needs to change from #{actual_primary_key_name} to #{ideal_primary_key_name}, re-creating #{table_name} from scratch"
-          connection.drop_table table_name
-          resource.reset_column_information
-          _create_table
-        end
-        place_column ideal_primary_key_name
-        unless ideal.type.to_s == 'primary_key'
-          ::DataMiner.logger.debug "SETTING #{ideal_primary_key_name} AS PRIMARY KEY"
-          if connection.adapter_name.downcase == 'sqlite'
-            connection.execute "CREATE UNIQUE INDEX IDX_#{table_name}_#{ideal_primary_key_name} ON #{table_name} (#{ideal_primary_key_name} ASC)"
-          else
-            connection.execute "ALTER TABLE `#{table_name}` ADD PRIMARY KEY (`#{ideal_primary_key_name}`)"
-          end
-        end
-      end
-      resource.reset_column_information
-    end
-    
-    def _remove_columns
-      resource.columns_hash.values.each do |actual|
-        remove_column actual.name if column_needs_to_be_removed? actual.name
-      end
-    end
-    
-    def _add_columns
-      ideal_table.columns.each do |ideal|
-        place_column ideal.name if column_needs_to_be_placed? ideal.name
-      end
-    end
-    
-    def _remove_indexes
-      actual_indexes.each do |actual|
-        remove_index actual.name if index_needs_to_be_removed? actual.name
-      end
-    end
-    
-    def _add_indexes
-      ideal_indexes.each do |ideal|
-        next if ideal.name == ideal_primary_key_name # this should already have been taken care of
-        place_index ideal.name if index_needs_to_be_placed? ideal.name
-      end
-    end
-  end
-end