data_miner 1.3.0 → 1.3.3

data/README.rdoc

@@ -8,21 +8,21 @@ Programmatically import useful data into your ActiveRecord models.
 
 You define <tt>data_miner</tt> blocks in your ActiveRecord models. For example, in <tt>app/models/country.rb</tt>:
 
-  class Country < ActiveRecord::Base
-    set_primary_key :iso_3166_code
+    class Country < ActiveRecord::Base
+      set_primary_key :iso_3166_code
   
-    data_miner do    
-      import 'the official ISO country list',
-             :url => 'http://www.iso.org/iso/list-en1-semic-3.txt',
-             :skip => 2,
-             :headers => false,
-             :delimiter => ';',
-             :encoding => 'ISO-8859-1' do
-        key   'iso_3166_code', :field_number => 1
-        store 'name', :field_number => 0
+      data_miner do    
+        import 'the official ISO country list',
+               :url => 'http://www.iso.org/iso/list-en1-semic-3.txt',
+               :skip => 2,
+               :headers => false,
+               :delimiter => ';',
+               :encoding => 'ISO-8859-1' do
+          key   :iso_3166_code, :field_number => 1
+          store :name, :field_number => 0
+        end
       end
     end
-  end
 
 Now you can run:
 
@@ -31,33 +31,28 @@ Now you can run:
 
 == Creating tables from scratch (changed in 1.2)
 
-We recommend using the <tt>force_schema</tt> gem (https://github.com/seamusabshere/force_schema)
+We recommend using the <tt>mini_record-compat</tt> gem (https://github.com/seamusabshere/mini_record)
 
 This replaces the <tt>schema</tt> method that was available before. It didn't make sense for <tt>data_miner</tt> to provide this natively.
 
-class Car < ActiveRecord::Base
-
-  # THE NEW WAY - depends on force_schema gem, which is not required by default
-  # see the process step in the data_miner block where we actually call force_schema!
-  force_schema do
-    string :make
-    string :model
-  end
+    class Car < ActiveRecord::Base
+      # the mini_record way
+      col :make
+      col :model
   
-  data_miner do
-    # DEPRECATED - see above
-    # schema do
-    #   string :make
-    #   string :model
-    # end
+      data_miner do
+        # DEPRECATED - see above
+        # schema do
+        #   string :make
+        #   string :model
+        # end
   
-    process "create the table, adding and removing columns as necessary" do
-      force_schema!
-    end
+        # the mini_record way
+        process :auto_upgrade!
     
-    # [... other data mining steps]
-  end
-end
+        # [... other data mining steps]
+      end
+    end
 
 ==Advanced usage
 
@@ -69,35 +64,33 @@ This is how we linked together (http://data.brighterplanet.com/aircraft) the FAA
     # table without breaking associations.
     set_primary_key :icao_code
   
-    # Use the force_schema gem to define the database schema in-line.
+    # Use the mini_record-compat gem to define the database schema in-line.
     # It will destructively and automatically add/remove columns.
     # This is "OK" because you can always just re-run the import script to get the data back.
     # PS. If you're using DataMapper, you don't need this
-    force_schema do
-      string   'icao_code'
-      string   'manufacturer_name'
-      string   'name'
-      string   'bts_name'
-      string   'bts_aircraft_type_code'
-      string   'brighter_planet_aircraft_class_code'
-      string   'fuel_use_aircraft_name'
-      float    'm3'
-      string   'm3_units'
-      float    'm2'
-      string   'm2_units'
-      float    'm1'
-      string   'm1_units'
-      float    'endpoint_fuel'
-      string   'endpoint_fuel_units'
-      float    'seats'
-      float    'distance'
-      string   'distance_units'
-      float    'load_factor'
-      float    'freight_share'
-      float    'payload'
-      float    'weighting'
-      index    'bts_aircraft_type_code'
-    end
+    col :icao_code
+    col :manufacturer_name
+    col :name
+    col :bts_name
+    col :bts_aircraft_type_code
+    col :brighter_planet_aircraft_class_code
+    col :fuel_use_aircraft_name
+    col :m3, :type => :float
+    col :m3_units
+    col :m2, :type => :float
+    col :m2_units
+    col :m1, :type => :float
+    col :m1_units
+    col :endpoint_fuel, :type => :float
+    col :endpoint_fuel_units
+    col :seats, :type => :float
+    col :distance, :type => :float
+    col :distance_units
+    col :load_factor, :type => :float
+    col :freight_share, :type => :float
+    col :payload, :type => :float
+    col :weighting, :type => :float
+    col :bts_aircraft_type_code, :type => :index
   
     # A dictionary between BTS aircraft type codes and ICAO aircraft
     # codes that uses string similarity instead of exact matching.

data/data_miner.gemspec

@@ -19,15 +19,15 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
   
-  s.add_dependency 'remote_table', '>=1.2.2'
-  s.add_dependency 'escape', '>=0.0.4'
-  s.add_dependency 'activerecord', '>=2.3.4'
-  s.add_dependency 'activesupport', '>=2.3.4'
-  s.add_dependency 'conversions', '>=1.4.4'
-  s.add_dependency 'blockenspiel', '>=0.3.2'
-  s.add_dependency 'taps', '>=0.3.11'
-  s.add_dependency 'errata', '>=1.0.1'
-  s.add_development_dependency 'force_schema', '>=0.0.2'
+  s.add_runtime_dependency 'remote_table', '>=1.2.2'
+  s.add_runtime_dependency 'activerecord', '>=2.3.4'
+  s.add_runtime_dependency 'activesupport', '>=2.3.4'
+  s.add_runtime_dependency 'conversions', '>=1.4.4'
+  s.add_runtime_dependency 'blockenspiel', '>=0.3.2'
+  s.add_runtime_dependency 'errata', '>=1.0.1'
+  s.add_runtime_dependency 'posix-spawn'
+  s.add_runtime_dependency 'taps'
+  s.add_development_dependency 'mini_record-compat'
   s.add_development_dependency 'loose_tight_dictionary', ">=0.0.5"
   s.add_development_dependency 'test-unit'
   s.add_development_dependency 'shoulda'

data/lib/data_miner.rb

@@ -32,24 +32,7 @@ class DataMiner
     delegate :run, :to => :instance
     delegate :resource_names, :to => :instance
   end
-  
-  # TODO this should probably live somewhere else
-  def self.backtick_with_reporting(cmd)
-    cmd = cmd.gsub /[ ]*\n[ ]*/m, ' '
-    output = `#{cmd}`
-    if not $?.success?
-      raise %{
-From the data_miner gem...
-
-Command failed:
-#{cmd}
-
-Output:
-#{output}
-}
-    end
-  end
-  
+    
   # http://avdi.org/devblog/2009/07/14/recursively-symbolize-keys/
   def self.recursively_stringify_keys(hash)
     hash.inject(::Hash.new) do |result, (key, value)|

data/lib/data_miner/attribute.rb

@@ -70,6 +70,7 @@ class DataMiner
       value = do_split(value) if wants_split?
       value.gsub! /[ ]+/, ' '
       value.strip!
+      return nil if value.blank? and wants_nullification?
       value.upcase! if wants_upcase?
       value = do_convert row, value if wants_conversion?
       value = do_sprintf value if wants_sprintf?
@@ -86,16 +87,19 @@ class DataMiner
       return value if value.is_a? ::ActiveRecord::Base # carry through trapdoor
       value = value_in_dictionary value if wants_dictionary?
       value = synthesize.call(row) if wants_synthesize?
-      value = nil if value.blank? and wants_nullification?
       value
     end
-        
+    
     def set_record_from_row(record, row)
       return false if !wants_overwriting? and !record.send(name).nil?
       record.send "#{name}=", value_from_row(row)
-      record.send "#{name}_units=", (to_units || unit_from_source(row)).to_s if wants_units?
+      if wants_units?
+        unit = (to_units || unit_from_source(row)).to_s
+        unit = nil if unit.blank? and wants_nullification?
+        record.send "#{name}_units=", unit
+      end
     end
-
+    
     def unit_from_source(row)
       row[units_field_name || units_field_number].to_s.strip.underscore.to_sym
     end

data/lib/data_miner/config.rb

@@ -52,11 +52,8 @@ class DataMiner
       
       finished = false
       skipped = false
-      if Run.table_exists?
-        run = Run.create! :started_at => ::Time.now, :resource_name => resource.name, :killed => true
-      else
-        run = nil
-        ::DataMiner.logger.info "Not logging individual runs. Please run DataMiner::Run.create_tables if you want to enable this."
+      run = if Run.table_exists?
+        Run.create! :started_at => ::Time.now, :resource_name => resource.name, :killed => true
       end
       resource.delete_all if options['from_scratch']
       begin
@@ -92,7 +89,6 @@ class DataMiner
     def after_invoke
       return unless resource.table_exists?
       make_sure_unit_definitions_make_sense
-      suggest_missing_column_migrations
     end
     
     COMPLETE_UNIT_DEFINITIONS = [
@@ -124,34 +120,5 @@ You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence
         end
       end
     end
-    
-    def suggest_missing_column_migrations
-      missing_columns = []
-      
-      import_steps.each do |step|
-        step.attributes.each do |_, attribute|
-          raise "You can't have an attribute column that ends in _units (reserved): #{resource.table_name}.#{attribute.name}" if attribute.name.end_with? '_units'
-          unless resource.column_names.include? attribute.name
-            missing_columns << attribute.name
-          end
-          if attribute.wants_units? and !resource.column_names.include?(units_column = "#{attribute.name}_units")
-            missing_columns << units_column
-          end
-        end
-      end
-      missing_columns.uniq!
-      if missing_columns.any?
-        ::DataMiner.logger.debug %{
-
-================================
-
-On #{resource}, it looks like you're missing some columns...
-
-#{missing_columns.map { |column_name| "#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' } '#{column_name}'" }.join("\n") }
-
-================================
-        }
-      end
-    end
   end
 end

data/lib/data_miner/import.rb

@@ -80,10 +80,10 @@ class DataMiner
       table.each do |row|
         record = resource.send "find_or_initialize_by_#{@_key}", attributes[@_key].value_from_row(row)
         attributes.each { |_, attr| attr.set_record_from_row record, row }
-        if record.send(primary_key).present?
+        begin
           record.save!
-        else
-          ::DataMiner.logger.debug "Skipping #{row} because there's no primary key"
+        rescue
+          ::DataMiner.logger.warn "[data_miner] Got #{$!.inspect} when trying to save #{row}"
         end
       end
       free

data/lib/data_miner/tap.rb

@@ -1,4 +1,4 @@
-require 'escape'
+require 'posix/spawn'
 class DataMiner
   class Tap
     attr_reader :config
@@ -28,7 +28,7 @@ class DataMiner
           connection.drop_table possible_obstacle
         end
       end
-      ::DataMiner.backtick_with_reporting taps_pull_cmd
+      taps_pull
       if needs_table_rename?
         connection.rename_table source_table_name, resource.table_name
       end
@@ -55,6 +55,8 @@ class DataMiner
     
     def adapter
       case connection.adapter_name
+      when /mysql2/i
+        'mysql2'
       when /mysql/i
         'mysql'
       when /postgres/i
@@ -71,11 +73,13 @@ class DataMiner
     
     DEFAULT_PORTS = {
       'mysql' => 3306,
+      'mysql2' => 3306,
       'postgres' => 5432
     }
     
     DEFAULT_USERNAMES = {
       'mysql' => 'root',
+      'mysql2' => 'root',
       'postgres' => ''
     }
     
@@ -95,16 +99,15 @@ class DataMiner
     
     def db_locator
       case adapter
-      when 'mysql', 'postgres'
-        "#{username}:#{password}@#{host}:#{port}/#{database}"
       when 'sqlite'
         database
+      else
+        "#{username}:#{password}@#{host}:#{port}/#{database}"
       end
     end
     
-    # taps pull mysql://root:password@localhost/taps_test http://foo:bar@data.brighterplanet.com:5000 --tables aircraft
-    def taps_pull_cmd
-      ::Escape.shell_command [
+    def taps_pull
+      args = [
         'taps',
         'pull',
         "#{adapter}://#{db_locator}",
@@ -113,34 +116,10 @@ class DataMiner
         '--tables',
         source_table_name
       ]
-      # "taps pull  #{source} --indexes-first --tables #{source_table_name}"
+      child = ::POSIX::Spawn::Child.new *args
+      unless child.success?
+        raise %{[data_miner gem] Got "#{child.err}" back when tried to run "#{args.join(' ')}"}
+      end
     end
-    
-    # 2.3.5 mysql
-    # * <tt>:host</tt> - Defaults to "localhost".
-    # * <tt>:port</tt> - Defaults to 3306.
-    # * <tt>:socket</tt> - Defaults to "/tmp/mysql.sock".
-    # * <tt>:username</tt> - Defaults to "root"
-    # * <tt>:password</tt> - Defaults to nothing.
-    # * <tt>:database</tt> - The name of the database. No default, must be provided.
-    # * <tt>:encoding</tt> - (Optional) Sets the client encoding by executing "SET NAMES <encoding>" after connection.
-    # * <tt>:reconnect</tt> - Defaults to false (See MySQL documentation: http://dev.mysql.com/doc/refman/5.0/en/auto-reconnect.html).
-    # * <tt>:sslca</tt> - Necessary to use MySQL with an SSL connection.
-    # * <tt>:sslkey</tt> - Necessary to use MySQL with an SSL connection.
-    # * <tt>:sslcert</tt> - Necessary to use MySQL with an SSL connection.
-    # * <tt>:sslcapath</tt> - Necessary to use MySQL with an SSL connection.
-    # * <tt>:sslcipher</tt> - Necessary to use MySQL with an SSL connection.
-    # 2.3.5 mysql
-    # * <tt>:host</tt> - Defaults to "localhost".
-    # * <tt>:port</tt> - Defaults to 5432.
-    # * <tt>:username</tt> - Defaults to nothing.
-    # * <tt>:password</tt> - Defaults to nothing.
-    # * <tt>:database</tt> - The name of the database. No default, must be provided.
-    # * <tt>:schema_search_path</tt> - An optional schema search path for the connection given as a string of comma-separated schema names.  This is backward-compatible with the <tt>:schema_order</tt> option.
-    # * <tt>:encoding</tt> - An optional client encoding that is used in a <tt>SET client_encoding TO <encoding></tt> call on the connection.
-    # * <tt>:min_messages</tt> - An optional client min messages that is used in a <tt>SET client_min_messages TO <min_messages></tt> call on the connection.
-    # * <tt>:allow_concurrency</tt> - If true, use async query methods so Ruby threads don't deadlock; otherwise, use blocking query methods.
-    # 2.3.5 sqlite[3]
-    # * <tt>:database</tt> - Path to the database file.
   end
 end

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '1.3.0'
+  VERSION = '1.3.3'
 end

data/test/helper.rb

@@ -3,7 +3,8 @@ require 'bundler'
 Bundler.setup
 require 'test/unit'
 require 'shoulda'
-require 'force_schema'
+require 'mini_record'
+require 'logger'
 # require 'ruby-debug'
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))

data/test/support/test_database.rb

@@ -181,7 +181,9 @@ module TestDatabase
         t.datetime "created_at"
         t.datetime "updated_at"
         t.float    "emission_factor"
+        t.string   "emission_factor_units"
         t.float    "annual_distance"
+        t.string   "annual_distance_units"
         t.string   "code"
       end
       execute "ALTER TABLE automobile_fuel_types ADD PRIMARY KEY (code);"

data/test/test_data_miner_attribute.rb

@@ -55,4 +55,57 @@ class TestDataMinerAttribute < Test::Unit::TestCase
       end
     end
   end
+  
+  context '#set_record_from_row' do
+    setup do
+      @automobile_fuel_type = AutomobileFuelType.new
+    end
+    context 'nullify is true, wants units' do
+      setup do
+        @attribute = DataMiner::Attribute.new @automobile_fuel_type, 'annual_distance', :nullify => true, :units_field_name => 'annual_distance_units'
+      end
+      should 'set value and units to nil if field is blank' do
+        @attribute.set_record_from_row(@automobile_fuel_type,
+          'name' => 'electricity',
+          'annual_distance' => '',
+          'annual_distance_units' => ''
+        )
+        assert_nil @automobile_fuel_type.annual_distance
+        assert_nil @automobile_fuel_type.annual_distance_units
+      end
+      should 'set value and units if field is not blank' do
+        @attribute.set_record_from_row(@automobile_fuel_type,
+          'name' => 'electricity',
+          'annual_distance' => '100.0',
+          'annual_distance_units' => 'kilometres'
+        )
+        assert_equal 100.0, @automobile_fuel_type.annual_distance
+        assert_equal 'kilometres', @automobile_fuel_type.annual_distance_units
+      end
+    end
+    
+    context 'nullify is false, wants units' do
+      setup do
+        @attribute = DataMiner::Attribute.new @automobile_fuel_type, 'annual_distance', :units_field_name => 'annual_distance_units'
+      end
+      should 'set value and units to blank if field is blank' do
+        @attribute.set_record_from_row(@automobile_fuel_type,
+          'name' => 'electricity',
+          'annual_distance' => '',
+          'annual_distance_units' => ''
+        )
+        assert_equal 0.0, @automobile_fuel_type.annual_distance
+        assert_equal '', @automobile_fuel_type.annual_distance_units
+      end
+      should 'set value and units if field is not blank' do
+        @attribute.set_record_from_row(@automobile_fuel_type,
+          'name' => 'electricity',
+          'annual_distance' => '100.0',
+          'annual_distance_units' => 'kilometres'
+        )
+        assert_equal 100.0, @automobile_fuel_type.annual_distance
+        assert_equal 'kilometres', @automobile_fuel_type.annual_distance_units
+      end
+    end
+  end
 end

data/test/test_old_syntax.rb

@@ -3,24 +3,15 @@ require 'helper'
 
 TestDatabase.load_models
 
-class TappedAirport < ActiveRecord::Base
-  set_primary_key :iata_code
-  
-  data_miner do
-    tap "Brighter Planet's sanitized airports table", "http://carbon:neutral@data.brighterplanet.com:5001", :source_table_name => 'airports'
-    # tap "Brighter Planet's sanitized airports table", "http://carbon:neutral@localhost:5000", :source_table_name => 'airports'
-  end
-end
-
 class CensusRegion < ActiveRecord::Base
   set_primary_key :number
-  
+
   data_miner do
     import :url => 'http://www.census.gov/popest/geographic/codes02.csv', :skip => 9, :select => lambda { |row| row['Region'].to_i > 0 and row['Division'].to_s.strip == 'X'} do
       key 'number', :field_name => 'Region'
       store 'name', :field_name => 'Name'
     end
-    
+
     # pretend this is a different data source
     # fake! just for testing purposes
     import :url => 'http://www.census.gov/popest/geographic/codes02.csv', :skip => 9, :select => lambda { |row| row['Region'].to_i > 0 and row['Division'].to_s.strip == 'X'} do
@@ -33,7 +24,7 @@ end
 # smaller than a region
 class CensusDivision < ActiveRecord::Base
   set_primary_key :number
-  
+
   data_miner do
     import :url => 'http://www.census.gov/popest/geographic/codes02.csv', :skip => 9, :select => lambda { |row| row['Division'].to_s.strip != 'X' and row['FIPS CODE STATE'].to_s.strip == 'X'} do
       key 'number', :field_name => 'Division'
@@ -46,7 +37,7 @@ end
 
 class CensusDivisionDeux < ActiveRecord::Base
   set_primary_key :number
-  
+
   data_miner do
     import :url => 'http://www.census.gov/popest/geographic/codes02.csv', :skip => 9, :select => lambda { |row| row['Division'].to_s.strip != 'X' and row['FIPS CODE STATE'].to_s.strip == 'X'} do
       key 'number', :field_name => 'Division'
@@ -59,9 +50,9 @@ end
 
 class CrosscallingCensusRegion < ActiveRecord::Base
   set_primary_key :number
-  
+
   has_many :crosscalling_census_divisions
-  
+
   data_miner do
     process "derive ourselves from the census divisions table (i.e., cross call census divisions)" do
       CrosscallingCensusDivision.run_data_miner!
@@ -80,9 +71,9 @@ end
 
 class CrosscallingCensusDivision < ActiveRecord::Base
   set_primary_key :number
-  
+
   belongs_to :crosscalling_census_regions, :foreign_key => 'census_region_number'
-  
+
   data_miner do
     import "get a list of census divisions and their regions", :url => 'http://www.census.gov/popest/geographic/codes02.csv', :skip => 9, :select => lambda { |row| row['Division'].to_s.strip != 'X' and row['FIPS CODE STATE'].to_s.strip == 'X'} do
       key 'number', :field_name => 'Division'
@@ -90,7 +81,7 @@ class CrosscallingCensusDivision < ActiveRecord::Base
       store 'census_region_number', :field_name => 'Region'
       store 'census_region_name', :field_name => 'Region', :dictionary => { :input => 'number', :output => 'name', :url => 'http://data.brighterplanet.com/census_regions.csv' }
     end
-    
+
     process "make sure my parent object is set up (i.e., cross-call it)" do
       CrosscallingCensusRegion.run_data_miner!
     end
@@ -99,18 +90,18 @@ end
 
 class ResidentialEnergyConsumptionSurveyResponse < ActiveRecord::Base
   set_primary_key :department_of_energy_identifier
-  
+
   data_miner do
     process 'Define some unit conversions' do
       Conversions.register :kbtus, :joules, 1_000.0 * 1_055.05585
       Conversions.register :square_feet, :square_metres, 0.09290304
     end
-    
+
     # conversions are NOT performed here, since we first have to zero out legitimate skips
     # otherwise you will get values like "999 pounds = 453.138778 kilograms" (where 999 is really a legit skip)
     import 'RECs 2005 (but not converting units to metric just yet)', :url => 'http://www.eia.doe.gov/emeu/recs/recspubuse05/datafiles/RECS05alldata.csv' do
       key 'department_of_energy_identifier', :field_name => 'DOEID'
-      
+
       store 'residence_class', :field_name => 'TYPEHUQ', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/typehuq/typehuq.csv' }
       store 'construction_year', :field_name => 'YEARMADE', :dictionary => { :input => 'Code', :sprintf => '%02d', :output => 'Date in the middle (synthetic)', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/yearmade/yearmade.csv' }
       store 'construction_period', :field_name => 'YEARMADE', :dictionary => { :input => 'Code', :sprintf => '%02d', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/yearmade/yearmade.csv' }
@@ -120,12 +111,12 @@ class ResidentialEnergyConsumptionSurveyResponse < ActiveRecord::Base
       store 'window_ac_use', :field_name => 'USEWWAC', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/usewwac/usewwac.csv' }
       store 'clothes_washer_use', :field_name => 'WASHLOAD', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/washload/washload.csv' }
       store 'clothes_dryer_use', :field_name => 'DRYRUSE', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/dryruse/dryruse.csv' }
-      
+
       store 'census_division_number', :field_name => 'DIVISION'
       store 'census_division_name', :field_name => 'DIVISION', :dictionary => { :input => 'number', :output => 'name', :url => 'http://data.brighterplanet.com/census_divisions.csv' }
       store 'census_region_number', :field_name => 'DIVISION', :dictionary => { :input => 'number', :output => 'census_region_number', :url => 'http://data.brighterplanet.com/census_divisions.csv' }
       store 'census_region_name', :field_name => 'DIVISION', :dictionary => { :input => 'number', :output => 'census_region_name', :url => 'http://data.brighterplanet.com/census_divisions.csv' }
-      
+
       store 'floorspace', :field_name => 'TOTSQFT'
       store 'residents', :field_name => 'NHSLDMEM'
       store 'ownership', :field_name => 'KOWNRENT'
@@ -248,15 +239,15 @@ class ResidentialEnergyConsumptionSurveyResponse < ActiveRecord::Base
         update_all "#{attr_name} = #{attr_name} * #{Conversions::Unit.exchange_rate from_units, to_units}"
       end
     end
-      
+
     process 'Add a new field "rooms" that estimates how many rooms are in the house' do
       update_all 'rooms = total_rooms + bathrooms/2 + halfbaths/4 + heated_garage*(attached_1car_garage + detached_1car_garage + 2*(attached_2car_garage + detached_2car_garage) + 3*(attached_3car_garage + detached_3car_garage))'
     end
-    
+
     process 'Add a new field "lighting_use" that estimates how many hours light bulbs are turned on in the house' do
       update_all 'lighting_use = 2*(lights_on_1_to_4_hours + efficient_lights_on_1_to_4_hours) + 8*(lights_on_4_to_12_hours + efficient_lights_on_4_to_12_hours) + 16*(lights_on_over_12_hours + efficient_lights_on_over_12_hours) + 12*(outdoor_all_night_lights + outdoor_all_night_gas_lights)'
     end
-    
+
     process 'Add a new field "lighting_efficiency" that estimates what percentage of light bulbs in a house are energy-efficient' do
       update_all 'lighting_efficiency = (2*efficient_lights_on_1_to_4_hours + 8*efficient_lights_on_4_to_12_hours + 16*efficient_lights_on_over_12_hours) / lighting_use'
     end
@@ -494,7 +485,7 @@ class T100FlightSegment < ActiveRecord::Base
         store 'data_source', :field_name => 'DATA_SOURCE'
       end
     end
-    
+
     process 'Derive freight share as a fraction of payload' do
       update_all 'freight_share = (freight + mail) / payload', 'payload > 0'
     end
@@ -502,7 +493,7 @@ class T100FlightSegment < ActiveRecord::Base
     process 'Derive load factor, which is passengers divided by the total seats available' do
       update_all 'load_factor = passengers / seats', 'passengers <= seats'
     end
-    
+
     process 'Derive average seats per departure' do
       update_all 'seats_per_departure = seats / departures_performed', 'departures_performed > 0'
     end
@@ -512,38 +503,38 @@ end
 # note that this depends on stuff in Aircraft
 class AircraftDeux < ActiveRecord::Base
   set_primary_key :icao_code
-  
+
   # defined on the class because we defined the errata with a shorthand
   class << self
     def is_not_attributed_to_aerospatiale?(row)
       not row['Manufacturer'] =~ /AEROSPATIALE/i
     end
-    
+
     def is_not_attributed_to_cessna?(row)
       not row['Manufacturer'] =~ /CESSNA/i
     end
-    
+
     def is_not_attributed_to_learjet?(row)
       not row['Manufacturer'] =~ /LEAR/i
     end
-    
+
     def is_not_attributed_to_dehavilland?(row)
       not row['Manufacturer'] =~ /DE ?HAVILLAND/i
     end
-    
+
     def is_not_attributed_to_mcdonnell_douglas?(row)
       not row['Manufacturer'] =~ /MCDONNELL DOUGLAS/i
     end
-    
+
     def is_not_a_dc_plane?(row)
       not row['Model'] =~ /DC/i
     end
-    
+
     def is_a_crj_900?(row)
       row['Designator'].downcase == 'crj9'
     end
   end
-  
+
   data_miner do
     # ('A'..'Z').each do |letter|
     # Note: for the purposes of testing, only importing "D"
@@ -567,28 +558,24 @@ end
 class AutomobileMakeFleetYear < ActiveRecord::Base
   set_primary_key :name
 
-  force_schema do
-    string "name"
-    string   "make_name"
-    string   "fleet"
-    integer   "year"
-    float    "fuel_efficiency"
-    string "fuel_efficiency_units"
-    integer  "volume"
-    string  "make_year_name"
-    datetime "created_at"
-    datetime "updated_at"
-  end
+  col :name
+  col :make_name
+  col :fleet
+  col :year, :type => :integer
+  col :fuel_efficiency, :type => :float
+  col :fuel_efficiency_units
+  col :volume, :type => :integer
+  col :make_year_name
+  col :created_at, :type => :datetime
+  col :updated_at, :type => :datetime
 
   data_miner do
-    process "create table" do
-      force_schema!
-    end
-    
+    process :auto_upgrade!
+
     process "finish if i tell you to" do
       raise DataMiner::Finish if $force_finish
     end
-    
+
     process "skip if i tell you to" do
       raise DataMiner::Skip if $force_skip
     end
@@ -609,36 +596,28 @@ end
 
 class CensusDivisionTrois < ActiveRecord::Base
   set_primary_key :number_code
-  
-  force_schema do
-    string  'number_code'
-    string   'name'
-    string   'census_region_name'
-    integer  'census_region_number'
-    index    'census_region_name', :name => 'homefry'
-    index   ['number_code', 'name', 'census_region_name', 'census_region_number']
-  end
-  
+
+  col :number_code
+  col :name
+  col :census_region_name
+  col :census_region_number, :type => :integer
+  add_index 'census_region_name', :name => 'homefry'
+  add_index ['number_code', 'name', 'census_region_name', 'census_region_number']
+
   data_miner do
-    process "create table" do
-      force_schema!
-    end
+    process :auto_upgrade!
   end
 end
 
 class CensusDivisionFour < ActiveRecord::Base
-  force_schema do
-    string  'number_code'
-    string   'name'
-    string   'census_region_name'
-    integer  'census_region_number'
-    index    'census_region_name', :name => 'homefry'
-  end
+col :number_code
+col :name
+col :census_region_name
+col :census_region_number, :type => :integer
+add_index 'census_region_name', :name => 'homefry'
 
   data_miner do
-    process "create table" do
-      force_schema!
-    end
+    process :auto_upgrade!
   end
 end
 
@@ -661,11 +640,11 @@ class TestOldSyntax < Test::Unit::TestCase
       if AutomobileMakeFleetYear.table_exists?
         ActiveRecord::Base.connection.execute 'DROP TABLE automobile_make_fleet_years;'
       end
-      AutomobileMakeFleetYear.force_schema!
+      AutomobileMakeFleetYear.auto_upgrade!
       assert AutomobileMakeFleetYear.table_exists?
     end
   end
-    
+
   if ENV['ALL'] == 'true' or ENV['FAST'] == 'true'
     should 'append to an existing config' do
       AutomobileFuelType.class_eval do
@@ -685,7 +664,7 @@ class TestOldSyntax < Test::Unit::TestCase
       assert_equal 'http://example1.com', AutomobileFuelType.data_miner_config.steps[-2].table.url
       assert_equal 'http://example2.com', AutomobileFuelType.data_miner_config.steps[-1].table.url
     end
-    
+
     should 'override an existing data_miner configuration' do
       AutomobileFuelType.class_eval do
         data_miner do
@@ -710,7 +689,7 @@ class TestOldSyntax < Test::Unit::TestCase
       AutomobileMakeFleetYear.run_data_miner!
       assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
     end
-    
+
     should "stop and register skipped if it gets a DataMiner::Skip" do
       AutomobileMakeFleetYear.delete_all
       AutomobileMakeFleetYear.data_miner_runs.delete_all
@@ -723,124 +702,67 @@ class TestOldSyntax < Test::Unit::TestCase
       AutomobileMakeFleetYear.run_data_miner!
       assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
     end
-    
-    should "eagerly enforce a schema" do
-      ActiveRecord::Base.connection.create_table 'census_division_trois', :force => true, :options => 'ENGINE=InnoDB default charset=utf8' do |t|
-        t.string   'name'
-        t.string   'census_region_name'
-        # t.integer  'census_region_number'
-      end
-      ActiveRecord::Base.connection.execute 'ALTER TABLE census_division_trois ADD INDEX (census_region_name)'
-      CensusDivisionTrois.reset_column_information
-      missing_columns = %w{ census_region_number }
 
-      # sanity check
-      missing_columns.each do |column|
-        assert_false CensusDivisionTrois.column_names.include?(column)
-      end
-      assert_false ActiveRecord::Base.connection.indexes(CensusDivisionTrois.table_name).any? { |index| index.name == 'homefry' }
-      
-      3.times do
-        CensusDivisionTrois.run_data_miner!
-        missing_columns.each do |column|
-          assert CensusDivisionTrois.column_names.include?(column)
-        end
-        assert ActiveRecord::Base.connection.indexes(CensusDivisionTrois.table_name).any? { |index| index.name == 'homefry' }
-        assert_equal :string, CensusDivisionTrois.columns_hash[CensusDivisionTrois.primary_key].type
-      end
-    end
-    
-    should "let schemas work with default id primary keys" do
-      ActiveRecord::Base.connection.create_table 'census_division_fours', :force => true, :options => 'ENGINE=InnoDB default charset=utf8' do |t|
-        t.string   'name'
-        t.string   'census_region_name'
-        # t.integer  'census_region_number'
-      end
-      ActiveRecord::Base.connection.execute 'ALTER TABLE census_division_fours ADD INDEX (census_region_name)'
-      CensusDivisionFour.reset_column_information
-      missing_columns = %w{ census_region_number }
-    
-      # sanity check
-      missing_columns.each do |column|
-        assert_false CensusDivisionFour.column_names.include?(column)
-      end
-      assert_false ActiveRecord::Base.connection.indexes(CensusDivisionFour.table_name).any? { |index| index.name == 'homefry' }
-      
-      3.times do
-        CensusDivisionFour.run_data_miner!
-        missing_columns.each do |column|
-          assert CensusDivisionFour.column_names.include?(column)
-        end
-        assert ActiveRecord::Base.connection.indexes(CensusDivisionFour.table_name).any? { |index| index.name == 'homefry' }
-        assert_equal :integer, CensusDivisionFour.columns_hash[CensusDivisionFour.primary_key].type
-      end
-    end
-    
     should "allow specifying dictionaries explicitly" do
       CensusDivisionDeux.run_data_miner!
       assert_equal 'South Region', CensusDivisionDeux.find(5).census_region_name
     end
-    
+
     should "be able to key on things other than the primary key" do
       Aircraft.run_data_miner!
       assert_equal 'SP', Aircraft.find('DHC6').brighter_planet_aircraft_class_code
     end
-    
+
     should "be able to synthesize rows without using a full parser class" do
       AutomobileMakeFleetYear.run_data_miner!
       assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
     end
-    
+
     should "keep a call stack so that you can call run_data_miner! on a child" do
       CrosscallingCensusDivision.run_data_miner!
       assert CrosscallingCensusDivision.exists? :name => 'Mountain Division', :number => 8, :census_region_number => 4, :census_region_name => 'West Region'
       assert CrosscallingCensusRegion.exists? :name => 'West Region', :number => 4
     end
-    
+
     should "keep a call stack so that you can call run_data_miner! on a parent" do
       CrosscallingCensusRegion.run_data_miner!
       assert CrosscallingCensusDivision.exists? :name => 'Mountain Division', :number => 8, :census_region_number => 4, :census_region_name => 'West Region'
       assert CrosscallingCensusRegion.exists? :name => 'West Region', :number => 4
     end
-        
+
     should "import airports" do
       Airport.run_data_miner!
       assert Airport.count > 0
     end
-    
-    should "tap airports" do
-      TappedAirport.run_data_miner!
-      assert TappedAirport.count > 0
-    end
-    
+
     should "pull in census divisions using a data.brighterplanet.com dictionary" do
       CensusDivision.run_data_miner!
       assert CensusDivision.count > 0
     end
-    
+
     should "have a way to queue up runs that works with delated_job's send_later" do
       assert AutomobileVariant.respond_to?(:run_data_miner!)
     end
-    
+
     should "be idempotent" do
       Country.data_miner_config.run
       a = Country.count
       Country.data_miner_config.run
       b = Country.count
       assert_equal a, b
-    
+
       CensusRegion.data_miner_config.run
       a = CensusRegion.count
       CensusRegion.data_miner_config.run
       b = CensusRegion.count
       assert_equal a, b
     end
-      
+
     should "hash things" do
       AutomobileVariant.data_miner_config.steps[0].run
       assert AutomobileVariant.first.row_hash.present?
     end
-  
+
     should "process a callback block instead of a method" do
       AutomobileVariant.delete_all
       AutomobileVariant.data_miner_config.steps[0].run
@@ -848,7 +770,7 @@ class TestOldSyntax < Test::Unit::TestCase
       AutomobileVariant.data_miner_config.steps.last.run
       assert AutomobileVariant.first.fuel_efficiency_city.present?
     end
-  
+
     should "keep a log when it does a run" do
       approx_started_at = Time.now
       DataMiner.run :resource_names => %w{ Country }
@@ -857,7 +779,7 @@ class TestOldSyntax < Test::Unit::TestCase
       assert (last_run.started_at - approx_started_at).abs < 5 # seconds
       assert (last_run.terminated_at - approx_terminated_at).abs < 5 # seconds
     end
-  
+
     should "request a re-import from scratch" do
       c = Country.new
       c.iso_3166 = 'JUNK'
@@ -866,35 +788,35 @@ class TestOldSyntax < Test::Unit::TestCase
       DataMiner.run :resource_names => %w{ Country }, :from_scratch => true
       assert !Country.exists?(:iso_3166 => 'JUNK')
     end
-    
+
     should "know what runs were on a resource" do
       DataMiner.run :resource_names => %w{ Country }
       DataMiner.run :resource_names => %w{ Country }
       assert Country.data_miner_runs.count > 0
     end
   end
-  
+
   if ENV['ALL'] == 'true' or ENV['SLOW'] == 'true'
     should "allow errata to be specified with a shorthand, assuming the responder is the resource class itself" do
       AircraftDeux.run_data_miner!
       assert AircraftDeux.exists? :icao_code => 'DC91', :bts_aircraft_type_code => '630'
     end
-    
+
     should "mine aircraft" do
       Aircraft.run_data_miner!
       assert Aircraft.exists? :icao_code => 'DC91', :bts_aircraft_type_code => '630'
     end
-    
+
     should "mine automobile variants" do
       AutomobileVariant.run_data_miner!
       assert AutomobileVariant.count('make_name LIKE "%tesla"') > 0
     end
-    
+
     should "mine T100 flight segments" do
       T100FlightSegment.run_data_miner!
       assert T100FlightSegment.count('dest_country_name LIKE "%United States"') > 0
     end
-    
+
     should "mine residence survey responses" do
       ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
       assert ResidentialEnergyConsumptionSurveyResponse.find(6).residence_class.start_with?('Single-family detached house')

data/test/test_tap.rb

@@ -0,0 +1,21 @@
+$:.push File.dirname(__FILE__)
+require 'helper'
+
+TestDatabase.load_models
+
+class TappedAirport < ActiveRecord::Base
+  set_primary_key :iata_code
+
+  data_miner do
+    tap "Brighter Planet's sanitized airports table", "http://carbon:neutral@data.brighterplanet.com:5001", :source_table_name => 'airports'
+    # tap "Brighter Planet's sanitized airports table", "http://carbon:neutral@localhost:5000", :source_table_name => 'airports'
+  end
+end
+
+
+class TestTap < Test::Unit::TestCase
+  should "tap airports" do
+    TappedAirport.run_data_miner!
+    assert TappedAirport.count > 0
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_miner
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.3.3
   prerelease: 
 platform: ruby
 authors:
@@ -11,11 +11,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-22 00:00:00.000000000Z
+date: 2011-12-09 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: remote_table
-  requirement: &2166686740 !ruby/object:Gem::Requirement
+  requirement: &2163341440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -23,21 +23,10 @@ dependencies:
         version: 1.2.2
   type: :runtime
   prerelease: false
-  version_requirements: *2166686740
-- !ruby/object:Gem::Dependency
-  name: escape
-  requirement: &2166719520 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.0.4
-  type: :runtime
-  prerelease: false
-  version_requirements: *2166719520
+  version_requirements: *2163341440
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &2166719060 !ruby/object:Gem::Requirement
+  requirement: &2163340680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -45,10 +34,10 @@ dependencies:
         version: 2.3.4
   type: :runtime
   prerelease: false
-  version_requirements: *2166719060
+  version_requirements: *2163340680
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2166718600 !ruby/object:Gem::Requirement
+  requirement: &2163340220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -56,10 +45,10 @@ dependencies:
         version: 2.3.4
   type: :runtime
   prerelease: false
-  version_requirements: *2166718600
+  version_requirements: *2163340220
 - !ruby/object:Gem::Dependency
   name: conversions
-  requirement: &2166718140 !ruby/object:Gem::Requirement
+  requirement: &2163339760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -67,10 +56,10 @@ dependencies:
         version: 1.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *2166718140
+  version_requirements: *2163339760
 - !ruby/object:Gem::Dependency
   name: blockenspiel
-  requirement: &2166717680 !ruby/object:Gem::Requirement
+  requirement: &2163339260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -78,43 +67,54 @@ dependencies:
         version: 0.3.2
   type: :runtime
   prerelease: false
-  version_requirements: *2166717680
+  version_requirements: *2163339260
 - !ruby/object:Gem::Dependency
-  name: taps
-  requirement: &2166717220 !ruby/object:Gem::Requirement
+  name: errata
+  requirement: &2163338800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.3.11
+        version: 1.0.1
   type: :runtime
   prerelease: false
-  version_requirements: *2166717220
+  version_requirements: *2163338800
 - !ruby/object:Gem::Dependency
-  name: errata
-  requirement: &2166716760 !ruby/object:Gem::Requirement
+  name: posix-spawn
+  requirement: &2163338420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2166716760
+  version_requirements: *2163338420
 - !ruby/object:Gem::Dependency
-  name: force_schema
-  requirement: &2166716300 !ruby/object:Gem::Requirement
+  name: taps
+  requirement: &2163337960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.0.2
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2163337960
+- !ruby/object:Gem::Dependency
+  name: mini_record-compat
+  requirement: &2163337540 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2166716300
+  version_requirements: *2163337540
 - !ruby/object:Gem::Dependency
   name: loose_tight_dictionary
-  requirement: &2166715840 !ruby/object:Gem::Requirement
+  requirement: &2163337040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -122,10 +122,10 @@ dependencies:
         version: 0.0.5
   type: :development
   prerelease: false
-  version_requirements: *2166715840
+  version_requirements: *2163337040
 - !ruby/object:Gem::Dependency
   name: test-unit
-  requirement: &2166715460 !ruby/object:Gem::Requirement
+  requirement: &2163336620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -133,10 +133,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2166715460
+  version_requirements: *2163336620
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &2166715000 !ruby/object:Gem::Requirement
+  requirement: &2163336100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -144,10 +144,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2166715000
+  version_requirements: *2163336100
 - !ruby/object:Gem::Dependency
   name: mysql
-  requirement: &2166714580 !ruby/object:Gem::Requirement
+  requirement: &2163335620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -155,10 +155,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2166714580
+  version_requirements: *2163335620
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2166714160 !ruby/object:Gem::Requirement
+  requirement: &2163335040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -166,7 +166,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2166714160
+  version_requirements: *2163335040
 description: Mine remote data into your ActiveRecord models. You can also convert
   units.
 email:
@@ -203,6 +203,7 @@ files:
 - test/test_data_miner_attribute.rb
 - test/test_data_miner_process.rb
 - test/test_old_syntax.rb
+- test/test_tap.rb
 homepage: https://github.com/seamusabshere/data_miner
 licenses: []
 post_install_message: 
@@ -223,7 +224,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: data_miner
-rubygems_version: 1.8.6
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Mine remote data into your ActiveRecord models.
@@ -238,3 +239,4 @@ test_files:
 - test/test_data_miner_attribute.rb
 - test/test_data_miner_process.rb
 - test/test_old_syntax.rb
+- test/test_tap.rb