earth 0.5.0 → 0.5.2

data/lib/earth/air/flight_segment/data_miner.rb

@@ -1,7 +1,15 @@
-# need this to run flight_segment.cache_aircraft!
-require 'loose_tight_dictionary/cached_result'
-
 FlightSegment.class_eval do
+  # For import errata
+  class FlightSegment::Guru
+    def in_may_2009?(row)
+      row ['MONTH'] == 5 and row['YEAR'] == 2009
+    end
+    
+    def in_july_2009?(row)
+      row ['MONTH'] == 7 and row['YEAR'] == 2009
+    end
+  end
+  
   URL = 'http://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=293&Has_Group=3&Is_Zipped=0'
   FORM_DATA = %{
     UserTableName=T_100_Segment__All_Carriers&
@@ -162,51 +170,8 @@ FlightSegment.class_eval do
   }.gsub /[\s]+/,''
   
   data_miner do
-    schema Earth.database_options do
-      string  'row_hash'                          # auto-generated primary key
-      string  'origin_airport_iata_code'          # iata code
-      string  'origin_airport_city'               # city
-      string  'origin_country_iso_3166_code'      # iso code
-      string  'destination_airport_iata_code'     # iata code
-      string  'destination_airport_city'          # city
-      string  'destination_country_iso_3166_code' # iso code
-      string  'airline_bts_code'                  # bts code
-      string  'airline_icao_code'                 # icao code
-      string  'airline_name'                      # text description derived from bts or icao code
-      string  'aircraft_bts_code'                 # bts code
-      string  'aircraft_description'              # text description derived from BTS T100 or ICAO TFS
-      integer 'flights'                           # number of flights over month or year
-      integer 'passengers'                        # total passengers on all flights
-      integer 'seats'                             # total seats on all flights
-      float   'seats_per_flight'                  # average seats per flight; make this a float
-      float   'load_factor'                       # passengers / seats
-      float   'freight_share'                     # (freight + mail) / (freight + mail + (passengers * average passenger weight))
-      float   'distance'                          # flight distance
-      string  'distance_units'                    # 'kilometres'
-      float   'payload_capacity'                  # aircraft maximum payload capacity rating; float b/c unit conversion
-      string  'payload_capacity_units'            # 'kilograms'
-      float   'freight'                           # total freight on all flights performed; float b/c unit conversion
-      string  'freight_units'                     # 'kilograms'
-      float   'mail'                              # total mail on all flights performed; float b/c unit conversion
-      string  'mail_units'                        # 'kilograms'
-      integer 'month'                             # month of flight
-      integer 'year'                              # year of flight
-      date    'approximate_date'                  # assumed 14th day of month
-      string  'source'                            # 'BTS T100' or 'ICAO TFS'
-      index   'origin_airport_iata_code'          # index for faster lookup by origin airport
-      index   'origin_airport_city'               # index for faster lookup by origin city
-      index   'destination_airport_iata_code'     # index for faster lookup by destination airport
-      index   'destination_airport_city'          # index for faster lookup by destination city
-      index   'airline_bts_code'                  # index for faster lookup by airline bts code
-      index   'airline_icao_code'                 # index for faster lookup by airline icao code
-      index   'airline_name'                      # index for faster lookup by airline name
-      index   'aircraft_bts_code'                 # index for faster lookup by aircraft bts code
-      index   'aircraft_description'              # index for faster lookup by aircraft
-      index   'year'                              # index for faster lookup by year
-    end
-    
     months = Hash.new
-    (2009..2010).each do |year|
+    (2009..2011).each do |year|
       (1..12).each do |month|
         time = Time.gm year, month
         form_data = FORM_DATA.dup
@@ -223,6 +188,8 @@ FlightSegment.class_eval do
              :form_data => form_data,
              :compression => :zip,
              :glob => '/*.csv',
+             # FIXME TODO 6/6/2011 the errata doesn't work - still doesn't fill in missing airline in May and July 2009
+             :errata => { :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGxpYU1qWFR3d0syTVMyQVVOaDd0V3c&output=csv', :responder => FlightSegment::Guru.new },
              :select => lambda { |record| record['DEPARTURES_PERFORMED'].to_i > 0 } do
         key 'row_hash'
         store 'origin_airport_iata_code',          :field_name => 'ORIGIN'
@@ -244,33 +211,24 @@ FlightSegment.class_eval do
       end
     end
     
-    # verify origin_airport_iata_code is in airports
-    # verify destination_airport_iata_code is in airports
-    # verify origin_country_iso_3166_code is in countries
-    # verify destination_country_iso_3166_code is in countries
-    # verify airline_bts_code appears in airlines
-    # verify aircraft_description is never missing
-    # verify year is never missing
-    
-    process "Ensure Airline is populated" do
+    process "Ensure Airline and BtsAircraft are populated" do
       Airline.run_data_miner!
+      BtsAircraft.run_data_miner!
     end
     
     process "Look up airline name based on BTS code" do
       connection.select_values("SELECT DISTINCT airline_bts_code FROM flight_segments WHERE airline_bts_code IS NOT NULL").each do |bts_code|
-        name = Airline.find_by_bts_code(bts_code).name
-        update_all %{ airline_name = "#{name}" }, %{ airline_bts_code = "#{bts_code}" }
+        if airline = Airline.find_by_bts_code(bts_code)
+          update_all %{ airline_name = "#{airline.name}" }, %{ airline_bts_code = "#{bts_code}" }
+        end
       end
     end
     
-    process "Ensure BtsAircraft is populated" do
-      BtsAircraft.run_data_miner!
-    end
-    
     process "Look up aircraft description based on BTS code" do
       connection.select_values("SELECT DISTINCT aircraft_bts_code FROM flight_segments WHERE aircraft_bts_code IS NOT NULL").each do |bts_code|
-        description = BtsAircraft.find_by_bts_code(bts_code).description.downcase
-        update_all %{ aircraft_description = "#{description}" }, %{ aircraft_bts_code = "#{bts_code}" }
+        if aircraft = BtsAircraft.find_by_bts_code(bts_code)
+          update_all %{ aircraft_description = "#{aircraft.description.downcase}" }, %{ aircraft_bts_code = "#{bts_code}" }
+        end
       end
     end
     
@@ -316,56 +274,16 @@ FlightSegment.class_eval do
       update_all 'approximate_date = DATE(CONCAT_WS("-", year, month, "14"))', 'month IS NOT NULL'
     end
     
-    process "Ensure Aircraft is populated" do
+    process "Data mine Aircraft to cache fuzzy matches" do
       Aircraft.run_data_miner!
     end
     
-    process "Cache fuzzy matches between FlightSegment aircraft_description and Aircraft description" do
-      LooseTightDictionary::CachedResult.setup
-      FlightSegment.find_by_sql("SELECT DISTINCT aircraft_description FROM flight_segments WHERE aircraft_description IS NOT NULL").each do |flight_segment|
-        original_description = flight_segment.aircraft_description
-        
-        # If the flight segment's aircraft_description contains '/' then it describes multiple aircraft.
-        # We need to synthesize descriptions for those aircraft, find all Aircraft that fuzzily match the
-        # synthesized descriptions, and associate those Aircraft with the original aircraft_description.
-        # e.g. boeing 747-100/200
-        if original_description.include?("/")
-          # Pull out the complete first aircraft description
-          # e.g. 'boeing 747-100'
-          first_description = original_description.split('/')[0]
-          
-          # Pull out the root of the description - the text up to and including the last ' ' or '-'
-          # e.g. 'boeing 747-'
-          root_length = first_description.rindex('-')
-          root = first_description.slice(0..root_length)
-          
-          # Pull out the suffixes - the text separated by forward slashes
-          # e.g. ['100', '200']
-          suffixes = original_description.split(root)[1].split('/')
-          
-          # Create an array of synthesized descriptions by appending each suffix to the root
-          # e.g. ['boeing 747-100', 'boeing 747-200']
-          suffixes.map{ |suffix| root + suffix }.each do |synthesized_description|
-            # Look up the Aircraft that match each synthesized description and associate
-            # them with the original flight segment aircraft_description
-            Aircraft.loose_tight_dictionary.find_all(synthesized_description).each do |aircraft|
-              attrs = {
-                :a_class => "Aircraft",
-                :a => aircraft.description,
-                :b_class => "FlightSegment",
-                :b => original_description
-              }
-              unless ::LooseTightDictionary::CachedResult.exists? attrs
-                ::LooseTightDictionary::CachedResult.create! attrs
-              end
-            end
-          end
-        # If the flight segment's aircraft_description doesn't contain '/' we can use
-        # a method provided by loose_tight_dictionary to associate it with Aircraft
-        else
-          flight_segment.cache_aircraft!
-        end
-      end
-    end
+    # verify origin_airport_iata_code is in airports
+    # verify destination_airport_iata_code is in airports
+    # verify origin_country_iso_3166_code is in countries
+    # verify destination_country_iso_3166_code is in countries
+    # verify airline_name is never missing
+    # verify aircraft_description is never missing
+    # verify year is never missing
   end
 end

data/lib/earth/air/flight_segment.rb

@@ -1,16 +1,15 @@
-# need this for association with Aircraft through loose_tight_dictionary_cached_results
-require 'loose_tight_dictionary/cached_result'
-
 class FlightSegment < ActiveRecord::Base
   set_primary_key :row_hash
   
   extend CohortScope
   self.minimum_cohort_size = 1
   
-  # If airport iata code is missing, associate with all airports in a city
-  # We need this to calculate distance when importing ICAO segments - see cm1 flight_segment.rb
-  has_many :origin_city_airports,      :foreign_key => 'city', :primary_key => 'origin_airport_city',      :class_name => 'Airport'
-  has_many :destination_city_airports, :foreign_key => 'city', :primary_key => 'destination_airport_city', :class_name => 'Airport'
+  # Cutting this for now because if iata code is missing we have to look up airports using both city and country; don't know how to do this with ActiveRecord
+  # - Ian 6/12/2011
+  # # If airport iata code is missing, associate with all airports in a city
+  # # We need this to calculate distance when importing ICAO segments - see cm1 flight_segment.rb
+  # has_many :origin_city_airports,      :foreign_key => 'city', :primary_key => 'origin_airport_city',      :class_name => 'Airport'
+  # has_many :destination_city_airports, :foreign_key => 'city', :primary_key => 'destination_airport_city', :class_name => 'Airport'
   
   # Enable flight_segment.aircraft
   cache_loose_tight_dictionary_matches_with :aircraft, :primary_key => :aircraft_description, :foreign_key => :description
@@ -20,11 +19,46 @@ class FlightSegment < ActiveRecord::Base
                 :load_factor      => lambda { weighted_average(:load_factor,      :weighted_by => :passengers) }, # 0.78073233770097  data1 10-12-2010
                 :freight_share    => lambda { weighted_average(:freight_share,    :weighted_by => :passengers) }  # 0.022567224170157 data1 10-12-2010
   
-  data_miner do
-    tap "Brighter Planet's sanitized flight segment data", Earth.taps_server
-    
-    process "Pull dependencies" do
-      run_data_miner_on_belongs_to_associations
-    end
+  create_table do
+    string  'row_hash'                          # auto-generated primary key
+    string  'origin_airport_iata_code'          # iata code
+    string  'origin_airport_city'               # city
+    string  'origin_country_iso_3166_code'      # iso code
+    string  'destination_airport_iata_code'     # iata code
+    string  'destination_airport_city'          # city
+    string  'destination_country_iso_3166_code' # iso code
+    string  'airline_bts_code'                  # bts code
+    string  'airline_icao_code'                 # icao code
+    string  'airline_name'                      # text description derived from bts or icao code
+    string  'aircraft_bts_code'                 # bts code
+    string  'aircraft_description'              # text description derived from BTS T100 or ICAO TFS
+    integer 'flights'                           # number of flights over month or year
+    integer 'passengers'                        # total passengers on all flights
+    integer 'seats'                             # total seats on all flights
+    float   'seats_per_flight'                  # average seats per flight; make this a float
+    float   'load_factor'                       # passengers / seats
+    float   'freight_share'                     # (freight + mail) / (freight + mail + (passengers * average passenger weight))
+    float   'distance'                          # flight distance
+    string  'distance_units'                    # 'kilometres'
+    float   'payload_capacity'                  # aircraft maximum payload capacity rating; float b/c unit conversion
+    string  'payload_capacity_units'            # 'kilograms'
+    float   'freight'                           # total freight on all flights performed; float b/c unit conversion
+    string  'freight_units'                     # 'kilograms'
+    float   'mail'                              # total mail on all flights performed; float b/c unit conversion
+    string  'mail_units'                        # 'kilograms'
+    integer 'month'                             # month of flight
+    integer 'year'                              # year of flight
+    date    'approximate_date'                  # assumed 14th day of month
+    string  'source'                            # 'BTS T100' or 'ICAO TFS'
+    index   'origin_airport_iata_code'          # index for faster lookup by origin airport
+    index   'origin_airport_city'               # index for faster lookup by origin city
+    index   'destination_airport_iata_code'     # index for faster lookup by destination airport
+    index   'destination_airport_city'          # index for faster lookup by destination city
+    index   'airline_bts_code'                  # index for faster lookup by airline bts code
+    index   'airline_icao_code'                 # index for faster lookup by airline icao code
+    index   'airline_name'                      # index for faster lookup by airline name
+    index   'aircraft_bts_code'                 # index for faster lookup by aircraft bts code
+    index   'aircraft_description'              # index for faster lookup by aircraft
+    index   'year'                              # index for faster lookup by year
   end
 end

data/lib/earth/air.rb

@@ -1,10 +1,11 @@
 require 'earth/air/aircraft'
 require 'earth/air/aircraft_class'
 require 'earth/air/aircraft_fuel_use_equation'
-require 'earth/air/aircraft_manufacturer'
 require 'earth/air/airline'
 require 'earth/air/airport'
 require 'earth/air/bts_aircraft'
 require 'earth/air/flight_distance_class'
 require 'earth/air/flight_seat_class'
 require 'earth/air/flight_segment'
+
+require 'earth/locality'

data/lib/earth/automobile/automobile_fuel/data_miner.rb

@@ -1,29 +1,5 @@
 AutomobileFuel.class_eval do
   data_miner do
-    schema Earth.database_options do
-      string 'name'
-      string 'code'
-      string 'base_fuel_name'
-      string 'blend_fuel_name'
-      float  'blend_portion' # the portion of the blend that is the blend fuel
-      string 'distance_key' # used to look up annual distance from AutomobileTypeFuelYear
-      string 'ef_key' # used to look up ch4 n2o and hfc emission factors from AutomobileTypeFuelYear
-      float  'annual_distance'
-      string 'annual_distance_units'
-      float  'co2_emission_factor'
-      string 'co2_emission_factor_units'
-      float  'co2_biogenic_emission_factor'
-      string 'co2_biogenic_emission_factor_units'
-      float  'ch4_emission_factor'
-      string 'ch4_emission_factor_units'
-      float  'n2o_emission_factor'
-      string 'n2o_emission_factor_units'
-      float  'hfc_emission_factor'
-      string 'hfc_emission_factor_units'
-      float  'emission_factor' # DEPRECATED but motorcycle needs this
-      string 'emission_factor_units' # DEPRECATED but motorcycle needs this
-    end
-    
     import "a list of pure automobile fuels",
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdE9xTEdueFM2R0diNTgxUlk1QXFSb2c&gid=0&output=html' do
       key   'name'
@@ -44,7 +20,7 @@ AutomobileFuel.class_eval do
       store 'ef_key'
     end
     
-    process "Ensure necessary datasets are imported" do
+    process "Ensure AutomobileTypeFuelYearAge, AutomobileTypeYear, Fuel, and GreenhouseGas are populated" do
       AutomobileTypeFuelYearAge.run_data_miner!
       AutomobileTypeYear.run_data_miner!
       Fuel.run_data_miner!

data/lib/earth/automobile/automobile_fuel.rb

@@ -97,15 +97,31 @@ class AutomobileFuel < ActiveRecord::Base
                 :n2o_emission_factor_units => lambda { AutomobileFuel.fallback_n2o_emission_factor_units },
                 :hfc_emission_factor => lambda { AutomobileFuel.fallback_hfc_emission_factor },
                 :hfc_emission_factor_units => lambda { AutomobileFuel.fallback_hfc_emission_factor_units }
-  
-  data_miner do
-    tap "Brighter Planet's sanitized automobile fuel data", Earth.taps_server
-    
-    process "pull dependencies" do
-      run_data_miner_on_belongs_to_associations
-    end
+
+  create_table do
+    string 'name'
+    string 'code'
+    string 'base_fuel_name'
+    string 'blend_fuel_name'
+    float  'blend_portion' # the portion of the blend that is the blend fuel
+    string 'distance_key' # used to look up annual distance from AutomobileTypeFuelYear
+    string 'ef_key' # used to look up ch4 n2o and hfc emission factors from AutomobileTypeFuelYear
+    float  'annual_distance'
+    string 'annual_distance_units'
+    float  'co2_emission_factor'
+    string 'co2_emission_factor_units'
+    float  'co2_biogenic_emission_factor'
+    string 'co2_biogenic_emission_factor_units'
+    float  'ch4_emission_factor'
+    string 'ch4_emission_factor_units'
+    float  'n2o_emission_factor'
+    string 'n2o_emission_factor_units'
+    float  'hfc_emission_factor'
+    string 'hfc_emission_factor_units'
+    float  'emission_factor' # DEPRECATED but motorcycle needs this
+    string 'emission_factor_units' # DEPRECATED but motorcycle needs this
   end
-  
+
   CODES = {
     :electricity => 'El',
     :diesel => 'D'

data/lib/earth/automobile/automobile_make/data_miner.rb

@@ -1,17 +1,12 @@
 AutomobileMake.class_eval do
   data_miner do
     process "Start from scratch" do
-      connection.drop_table table_name
+      delete_all
     end
     
-    schema Earth.database_options do
-      string  'name'
-      float   'fuel_efficiency'
-      string  'fuel_efficiency_units'
-    end
-    
-    process "Ensure AutomobileMakeModelYearVariant is populated" do
+    process "Ensure AutomobileMakeModelYearVariant and AutomobileMakeFleetYear are populated" do
       AutomobileMakeModelYearVariant.run_data_miner!
+      AutomobileMakeFleetYear.run_data_miner!
     end
     
     process "Derive manufacturer names from automobile make model year variants" do
@@ -23,10 +18,6 @@ AutomobileMake.class_eval do
       }
     end
     
-    process "Ensure AutomobileMakeFleetYear is populated" do
-      AutomobileMakeFleetYear.run_data_miner!
-    end
-    
     # sabshere 1/31/11 add Avanti, DaimlerChrysler, IHC, Tesla, etc.
     process "Derive extra manufacturer names from CAFE data" do
       INSERT_IGNORE %{INTO automobile_makes(name)
@@ -35,10 +26,6 @@ AutomobileMake.class_eval do
       }
     end
     
-    process "Ensure AutomobileMakeFleetYear is populated" do
-      AutomobileMakeFleetYear.run_data_miner!
-    end
-    
     # FIXME TODO make this a method on AutomobileMake?
     process "Calculate fuel efficiency from automobile make fleet years for makes with CAFE data" do
       make_fleet_years = AutomobileMakeFleetYear.arel_table

data/lib/earth/automobile/automobile_make.rb

@@ -5,8 +5,10 @@ class AutomobileMake < ActiveRecord::Base
   has_many :models,                   :class_name => 'AutomobileMakeModel',            :foreign_key => 'make_name'
   has_many :fleet_years,              :class_name => 'AutomobileMakeFleetYear',        :foreign_key => 'make_name'
   has_many :make_model_year_variants, :class_name => 'AutomobileMakeModelYearVariant', :foreign_key => 'make_name'
-  
-  data_miner do
-    tap "Brighter Planet's make year data", Earth.taps_server
+
+  create_table do
+    string  'name'
+    float   'fuel_efficiency'
+    string  'fuel_efficiency_units'
   end
 end

data/lib/earth/automobile/automobile_make_fleet_year/data_miner.rb

@@ -1,16 +1,5 @@
 AutomobileMakeFleetYear.class_eval do
   data_miner do
-    schema Earth.database_options do
-      string   'name'
-      string   'make_year_name'
-      string   'make_name'
-      string   'fleet'
-      integer  'year'
-      float    'fuel_efficiency'
-      string   'fuel_efficiency_units'
-      integer  'volume'
-    end
-    
     import "annual corporate average fuel economy data for domestic and imported vehicle fleets from the NHTSA",
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEdXWXB6dkVLWkowLXhYSFVUT01sS2c&hl=en&gid=0&output=csv',
            :errata => { 'url' => 'http://static.brighterplanet.com/science/data/transport/automobiles/make_fleet_years/errata.csv' },
@@ -24,34 +13,31 @@ AutomobileMakeFleetYear.class_eval do
       store 'volume'
     end
     
-    verify "Year should be from 1978 to 2009" do
-      AutomobileMakeFleetYear.all.each do |fleet_year|
-        unless fleet_year.year > 1977 and fleet_year.year < 2010
-          raise "Invalid year for AutomobileMakeFleetYear #{fleet_year.name}: #{fleet_year.year} (should be from 1978 to 2009)"
+    verify "Year should be from 1978 to 2010" do
+      connection.select_values("SELECT DISTINCT year FROM automobile_make_fleet_years").each do |year|
+        unless year > 1977 and year < 2011
+          raise "Invalid year in automobile_make_fleet_years: #{year} is not from 1978 to 2010"
         end
       end
     end
     
-    verify "Fuel efficiency should be greater than zero" do
-      AutomobileMakeFleetYear.all.each do |fleet_year|
-        unless fleet_year.fuel_efficiency > 0
-          raise "Invalid fuel efficiency for AutomobileMakeFleetYear #{fleet_year.name}: #{fleet_year.fuel_efficiency} (should be > 0)"
+    verify "Fuel efficiency and volume should be greater than zero" do
+      [:fuel_efficiency, :volume].each do |field|
+        if AutomobileMakeFleetYear.where(field => nil).any?
+          raise "Invalid #{field} in automobile_make_fleet_years: nil is not > 0"
+        else
+          min = AutomobileMakeFleetYear.minimum(field)
+          unless min > 0
+            raise "Invalid #{field} in automobile_make_fleet_years: #{min} is not > 0"
+          end
         end
       end
     end
     
     verify "Fuel efficiency units should be kilometres per litre" do
-      AutomobileMakeFleetYear.all.each do |fleet_year|
-        unless fleet_year.fuel_efficiency_units == "kilometres_per_litre"
-          raise "Invalid fuel efficiency units for AutomobileMakeFleetYear #{fleet_year.name}: #{fleet_year.fuel_efficiency_units} (should be kilometres_per_litre)"
-        end
-      end
-    end
-    
-    verify "Volume should be greater than zero" do
-      AutomobileMakeFleetYear.all.each do |fleet_year|
-        unless fleet_year.volume > 0
-          raise "Invalid volume for AutomobileMakeFleetYear #{fleet_year.name}: #{fleet_year.volume} (should be > 0)"
+      connection.select_values("SELECT DISTINCT fuel_efficiency_units FROM automobile_make_fleet_years").each do |units|
+        unless units == "kilometres_per_litre"
+          raise "Invalid fuel efficiency units in automobile_make_fleet_years: #{units} is not 'kilometres_per_litre'"
         end
       end
     end

data/lib/earth/automobile/automobile_make_fleet_year.rb

@@ -3,12 +3,15 @@ class AutomobileMakeFleetYear < ActiveRecord::Base
   
   belongs_to :make,      :class_name => 'AutomobileMake',     :foreign_key => 'make_name'
   belongs_to :make_year, :class_name => 'AutomobileMakeYear', :foreign_key => 'make_year_name'
-  
-  data_miner do
-    tap "Brighter Planet's sanitized auto make fleet year data", Earth.taps_server
-    
-    process "Pull dependencies" do
-      run_data_miner_on_belongs_to_associations
-    end
+
+  create_table do
+    string   'name'
+    string   'make_year_name'
+    string   'make_name'
+    string   'fleet'
+    integer  'year'
+    float    'fuel_efficiency'
+    string   'fuel_efficiency_units'
+    integer  'volume'
   end
 end

data/lib/earth/automobile/automobile_make_model/data_miner.rb

@@ -1,17 +1,7 @@
 AutomobileMakeModel.class_eval do
   data_miner do
     process "Start from scratch" do
-      connection.drop_table table_name
-    end
-    
-    schema Earth.database_options do
-      string  'name' # make + model
-      string  'make_name'
-      string  'model_name' # model only
-      float   'fuel_efficiency_city'
-      string  'fuel_efficiency_city_units'
-      float   'fuel_efficiency_highway'
-      string  'fuel_efficiency_highway_units'
+      delete_all
     end
     
     process "Ensure AutomobileMakeModelYearVariant is populated" do

data/lib/earth/automobile/automobile_make_model.rb

@@ -3,12 +3,14 @@ class AutomobileMakeModel < ActiveRecord::Base
   
   belongs_to :make,                   :class_name => 'AutomobileMake',                 :foreign_key => 'make_name'
   has_many :make_model_year_variants, :class_name => 'AutomobileMakeModelYearVariant', :foreign_key => 'make_model_name'
-  
-  data_miner do
-    tap "Brighter Planet's auto model data", Earth.taps_server
-    
-    process "Pull dependencies" do
-      run_data_miner_on_belongs_to_associations
-    end
+
+  create_table do
+    string  'name' # make + model
+    string  'make_name'
+    string  'model_name' # model only
+    float   'fuel_efficiency_city'
+    string  'fuel_efficiency_city_units'
+    float   'fuel_efficiency_highway'
+    string  'fuel_efficiency_highway_units'
   end
 end

data/lib/earth/automobile/automobile_make_model_year/data_miner.rb

@@ -1,20 +1,7 @@
 AutomobileMakeModelYear.class_eval do
   data_miner do
     process "Start from scratch" do
-      connection.drop_table table_name
-    end
-    
-    schema Earth.database_options do
-      string   'name' # make + model + year
-      string   'make_name'
-      string   'model_name'
-      string   'make_model_name'
-      integer  'year'
-      string   'make_year_name'
-      float    'fuel_efficiency_city'
-      string   'fuel_efficiency_city_units'
-      float    'fuel_efficiency_highway'
-      string   'fuel_efficiency_highway_units'
+      delete_all
     end
     
     process "Ensure AutomobileMakeModelYearVariant is populated" do
@@ -45,10 +32,10 @@ AutomobileMakeModelYear.class_eval do
       end
     end
     
-    verify "Year should be from 1985 to 2010" do
+    verify "Year should be from 1985 to 2011" do
       AutomobileMakeModelYear.all.each do |model_year|
-        unless model_year.year.to_i > 1984 and model_year.year.to_i < 2011
-          raise "Invalid year for AutomobileMakeModelYear #{model_year.name}: #{model_year.year} (should be from 1985 to 2010)"
+        unless model_year.year.to_i > 1984 and model_year.year.to_i < 2012
+          raise "Invalid year for AutomobileMakeModelYear #{model_year.name}: #{model_year.year} (should be from 1985 to 2011)"
         end
       end
     end

data/lib/earth/automobile/automobile_make_model_year.rb

@@ -4,11 +4,16 @@ class AutomobileMakeModelYear < ActiveRecord::Base
   belongs_to :make_year,              :class_name => 'AutomobileMakeYear',             :foreign_key => 'make_year_name'
   has_many :make_model_year_variants, :class_name => 'AutomobileMakeModelYearVariant', :foreign_key => 'make_model_year_name'
   
-  data_miner do
-    tap "Brighter Planet's model year data", Earth.taps_server
-    
-    process "pull dependencies" do
-      run_data_miner_on_belongs_to_associations
-    end
+  create_table do
+    string   'name' # make + model + year
+    string   'make_name'
+    string   'model_name'
+    string   'make_model_name'
+    integer  'year'
+    string   'make_year_name'
+    float    'fuel_efficiency_city'
+    string   'fuel_efficiency_city_units'
+    float    'fuel_efficiency_highway'
+    string   'fuel_efficiency_highway_units'
   end
 end