earth 0.5.0 → 0.5.2

data/lib/earth/hospitality/lodging_class.rb

@@ -1,7 +1,15 @@
 class LodgingClass < ActiveRecord::Base
   set_primary_key :name
   
-  data_miner do
-    tap "Brighter Planet's lodging class data", Earth.taps_server
+  create_table do
+    string 'name'
+    float  'electricity_intensity'
+    string 'electricity_intensity_units'
+    float  'natural_gas_intensity'
+    string 'natural_gas_intensity_units'
+    float  'fuel_oil_intensity'
+    string 'fuel_oil_intensity_units'
+    float  'district_heat_intensity'
+    string 'district_heat_intensity_units'
   end
 end

data/lib/earth/industry/industry.rb

@@ -1,5 +1,4 @@
 class Industry < ActiveRecord::Base
-  extend Earth::Base
   set_primary_key :naics_code
   
   has_many :merchant_category_industries, :foreign_key => 'naics_code'
@@ -10,15 +9,9 @@ class Industry < ActiveRecord::Base
   has_many :industry_sectors, :foreign_key => 'naics_code'
   has_many :sectors, :through => :industry_sectors
   
-  def self.schema_definition
-    lambda do
-      string 'naics_code'
-      string 'description'
-    end
-  end
-
-  data_miner do
-    Industry.define_schema(self)
+  create_table do
+    string 'naics_code'
+    string 'description'
   end
 
   def trade_industry?

data/lib/earth/industry/industry_product.rb

@@ -1,22 +1,15 @@
 class IndustryProduct < ActiveRecord::Base
-  extend Earth::Base
   set_primary_key :naics_product_code
   
   has_many :product_line_industry_products, :foreign_key => 'naics_product_code'
   
   belongs_to :industry, :foreign_key => 'naics_code'
   
-  def self.schema_definition
-    lambda do
-      string 'naics_product_code'
-      string 'description'
-      float  'value'
-      string 'value_units'
-      string 'naics_code'
-    end
-  end
-
-  data_miner do
-    IndustryProduct.define_schema(self)
+  create_table do
+    string 'naics_product_code'
+    string 'description'
+    float  'value'
+    string 'value_units'
+    string 'naics_code'
   end
 end

data/lib/earth/industry/industry_product_line.rb

@@ -1,20 +1,13 @@
 class IndustryProductLine < ActiveRecord::Base
-  extend Earth::Base
   set_primary_key :row_hash
   
   belongs_to :industry,     :foreign_key => 'naics_code'
   belongs_to :product_line, :foreign_key => 'ps_code'
 
-  def self.schema_definition
-    lambda do
-      string 'row_hash'
-      string 'naics_code'
-      float  'ratio'
-      string 'ps_code'
-    end
-  end
-
-  data_miner do
-    IndustryProductLine.define_schema(self)
+  create_table do
+    string 'row_hash'
+    string 'naics_code'
+    float  'ratio'
+    string 'ps_code'
   end
 end

data/lib/earth/industry/industry_sector.rb

@@ -1,20 +1,13 @@
 class IndustrySector < ActiveRecord::Base
-  extend Earth::Base
   set_primary_key :row_hash
   
   belongs_to :industry,  :foreign_key => 'naics_code'
   belongs_to :sector,    :foreign_key => 'io_code'
 
-  def self.schema_definition
-    lambda do
-      string 'row_hash'
-      string 'naics_code'
-      float  'ratio'
-      string 'io_code'
-    end
-  end
-
-  data_miner do
-    IndustrySector.define_schema(self)
+  create_table do
+    string 'row_hash'
+    string 'naics_code'
+    float  'ratio'
+    string 'io_code'
   end
 end

data/lib/earth/industry/merchant.rb

@@ -1,18 +1,11 @@
 class Merchant < ActiveRecord::Base
-  extend Earth::Base
   set_primary_key :id
   
   belongs_to :merchant_category, :foreign_key => 'mcc'
 
-  def self.schema_definition
-    lambda do
-      string 'id'
-      string 'name'
-      string 'mcc'
-    end
-  end
-
-  data_miner do
-    Merchant.define_schema(self)
+  create_table do
+    string 'id'
+    string 'name'
+    string 'mcc'
   end
 end

data/lib/earth/industry/merchant_category.rb

@@ -1,5 +1,4 @@
 class MerchantCategory < ActiveRecord::Base
-  extend Earth::Base
   set_primary_key :mcc
 
   has_many :merchant_category_industries, :foreign_key => 'mcc'
@@ -9,14 +8,8 @@ class MerchantCategory < ActiveRecord::Base
     description
   end
 
-  def self.schema_definition
-    lambda do
-      string 'mcc'
-      string 'description'
-    end
-  end
-
-  data_miner do
-    MerchantCategory.define_schema(self)
+  create_table do
+    string 'mcc'
+    string 'description'
   end
 end

data/lib/earth/industry/merchant_category_industry.rb

@@ -1,20 +1,13 @@
 class MerchantCategoryIndustry < ActiveRecord::Base
-  extend Earth::Base
   set_primary_key :row_hash
   
   belongs_to :merchant_category, :foreign_key => 'mcc'
   belongs_to :industry,          :foreign_key => 'naics_code'
 
-  def self.schema_definition
-    lambda do
-      string 'row_hash'
-      string 'mcc'
-      float  'ratio'
-      string 'naics_code'
-    end
-  end
-
-  data_miner do
-    MerchantCategoryIndustry.define_schema(self)
+  create_table do
+    string 'row_hash'
+    string 'mcc'
+    float  'ratio'
+    string 'naics_code'
   end
 end

data/lib/earth/industry/product_line.rb

@@ -1,5 +1,4 @@
 class ProductLine < ActiveRecord::Base
-  extend Earth::Base
   set_primary_key :ps_code
   
   has_many :industry_product_lines,         :foreign_key => 'ps_code'
@@ -7,16 +6,10 @@ class ProductLine < ActiveRecord::Base
   has_many :product_line_industry_products, :foreign_key => 'ps_code'
   has_many :industry_products, :through => :product_line_industry_products
   
-  def self.schema_definition
-    lambda do
-      string 'ps_code'
-      string 'description'
-      string 'broadline' # FIXME TODO do we need this?
-      string 'parent'    # FIXME TODO do we need this?
-    end
-  end
-
-  data_miner do
-    ProductLine.define_schema(self)
+  create_table do
+    string 'ps_code'
+    string 'description'
+    string 'broadline' # FIXME TODO do we need this?
+    string 'parent'    # FIXME TODO do we need this?
   end
 end

data/lib/earth/industry/product_line_industry_product.rb

@@ -1,20 +1,13 @@
 class ProductLineIndustryProduct < ActiveRecord::Base
-  extend Earth::Base
   set_primary_key :row_hash
   
   belongs_to :product_line,     :foreign_key => 'ps_code'
   belongs_to :industry_product, :foreign_key => 'naics_product_code'
 
-  def self.schema_definition
-    lambda do
-      string 'row_hash'
-      string 'ps_code'
-      float  'ratio'
-      string 'naics_product_code'
-    end
-  end
-
-  data_miner do
-    ProductLineIndustryProduct.define_schema(self)
+  create_table do
+    string 'row_hash'
+    string 'ps_code'
+    float  'ratio'
+    string 'naics_product_code'
   end
 end

data/lib/earth/industry/sector.rb

@@ -1,19 +1,12 @@
 class Sector < ActiveRecord::Base
-  extend Earth::Base
   set_primary_key :io_code
 
   has_many :industry_sectors, :foreign_key => 'io_code'
   
-  def self.schema_definition
-    lambda do
-      string 'io_code'
-      string 'description'
-      float  'value'
-      string 'value_units'
-    end
-  end
-
-  data_miner do
-    Sector.define_schema(self)
+  create_table do
+    string 'io_code'
+    string 'description'
+    float  'value'
+    string 'value_units'
   end
 end

data/lib/earth/locality/census_division/data_miner.rb

@@ -1,28 +1,5 @@
 CensusDivision.class_eval do
   data_miner do
-    schema Earth.database_options do
-      integer 'number'
-      string  'name'
-      string  'census_region_name'
-      integer 'census_region_number'
-      float   'meeting_building_natural_gas_intensity'
-      string  'meeting_building_natural_gas_intensity_units'
-      float   'meeting_building_fuel_oil_intensity'
-      string  'meeting_building_fuel_oil_intensity_units'
-      float   'meeting_building_electricity_intensity'
-      string  'meeting_building_electricity_intensity_units'
-      float   'meeting_building_district_heat_intensity'
-      string  'meeting_building_district_heat_intensity_units'
-      float   'lodging_building_natural_gas_intensity'
-      string  'lodging_building_natural_gas_intensity_units'
-      float   'lodging_building_fuel_oil_intensity'
-      string  'lodging_building_fuel_oil_intensity_units'
-      float   'lodging_building_electricity_intensity'
-      string  'lodging_building_electricity_intensity_units'
-      float   'lodging_building_district_heat_intensity'
-      string  'lodging_building_district_heat_intensity_units'
-    end
-    
     process 'Define some unit conversions' do
       Conversions.register :hundred_cubic_feet_per_square_foot_hour, :cubic_metres_per_square_metre_hour,   30.48
       Conversions.register :gallons_per_square_foot_hour,            :litres_per_square_metre_hour,         40.7458333

data/lib/earth/locality/census_division.rb

@@ -13,11 +13,26 @@ class CensusDivision < ActiveRecord::Base
                 :meeting_building_electricity_intensity => 0.0084323684 / 1.square_feet.to(:square_metres),
                 :meeting_building_district_heat_intensity => 0.0004776370.kbtus.to(:megajoules) / 1.square_feet.to(:square_metres)
   
-  data_miner do
-    tap "Brighter Planet's sanitized census divisions", Earth.taps_server
-    
-    process "pull dependencies" do
-      run_data_miner_on_belongs_to_associations
-    end
+  create_table do
+    integer 'number'
+    string  'name'
+    string  'census_region_name'
+    integer 'census_region_number'
+    float   'meeting_building_natural_gas_intensity'
+    string  'meeting_building_natural_gas_intensity_units'
+    float   'meeting_building_fuel_oil_intensity'
+    string  'meeting_building_fuel_oil_intensity_units'
+    float   'meeting_building_electricity_intensity'
+    string  'meeting_building_electricity_intensity_units'
+    float   'meeting_building_district_heat_intensity'
+    string  'meeting_building_district_heat_intensity_units'
+    float   'lodging_building_natural_gas_intensity'
+    string  'lodging_building_natural_gas_intensity_units'
+    float   'lodging_building_fuel_oil_intensity'
+    string  'lodging_building_fuel_oil_intensity_units'
+    float   'lodging_building_electricity_intensity'
+    string  'lodging_building_electricity_intensity_units'
+    float   'lodging_building_district_heat_intensity'
+    string  'lodging_building_district_heat_intensity_units'
   end
 end

data/lib/earth/locality/census_region/data_miner.rb

@@ -1,10 +1,5 @@
 CensusRegion.class_eval do
   data_miner do
-    schema Earth.database_options do
-      integer  'number'
-      string   'name'
-    end
-    
     import 'the U.S. Census Geographic Terms and Definitions',
            :url => 'http://www.census.gov/popest/geographic/codes02.csv',
            :skip => 9,

data/lib/earth/locality/census_region.rb

@@ -7,7 +7,8 @@ class CensusRegion < ActiveRecord::Base
   # has_many :zip_codes, :through => :census_divisions
   has_many :residential_energy_consumption_survey_responses, :foreign_key => 'census_region_number'
   
-  data_miner do
-    tap "Brighter Planet's sanitized census regions", Earth.taps_server
+  create_table do
+    integer  'number'
+    string   'name'
   end
 end

data/lib/earth/locality/climate_division/data_miner.rb

@@ -1,12 +1,5 @@
 ClimateDivision.class_eval do
   data_miner do
-    schema Earth.database_options do
-      string   'name'
-      float    'heating_degree_days'
-      float    'cooling_degree_days'
-      string   'state_postal_abbreviation'
-    end
-    
     import "a list of climate divisions and their average heating and cooling degree days",
            :url => 'http://static.brighterplanet.com/science/data/climate/climate_divisions/climate_divisions.csv' do
       key 'name'

data/lib/earth/locality/climate_division.rb

@@ -6,12 +6,10 @@ class ClimateDivision < ActiveRecord::Base
   
   RADIUS = 750
 
-  data_miner do
-    tap "Brighter Planet's sanitized climate divisions", Earth.taps_server
-
-    process "pull dependencies" do
-      run_data_miner_on_belongs_to_associations
-    end
+  create_table do
+    string   'name'
+    float    'heating_degree_days'
+    float    'cooling_degree_days'
+    string   'state_postal_abbreviation'
   end
-
 end

data/lib/earth/locality/country/data_miner.rb

@@ -1,20 +1,5 @@
 Country.class_eval do
   data_miner do
-    schema Earth.database_options do
-      string 'iso_3166_code'
-      string 'name'
-      float  'automobile_urbanity'
-      float  'automobile_fuel_efficiency'
-      string 'automobile_fuel_efficiency_units'
-      float  'automobile_city_speed'
-      string 'automobile_city_speed_units'
-      float  'automobile_highway_speed'
-      string 'automobile_highway_speed_units'
-      float  'automobile_trip_distance'
-      string 'automobile_trip_distance_units'
-      float  'flight_route_inefficiency_factor'
-    end
-    
     import 'the official ISO country list',
            :url => 'http://www.iso.org/iso/list-en1-semic-3.txt',
            :skip => 2,
@@ -40,21 +25,23 @@ Country.class_eval do
       store 'automobile_trip_distance', :units_field_name => 'automobile_trip_distance_units'
     end
     
+    process "Ensure AutomobileTypeFuelYear is populated" do
+      AutomobileTypeFuelYear.run_data_miner!
+    end
+    
     # FIXME TODO eventually need to do this for all countries
-    # process "Derive US average automobile fuel efficiency from AutomobileTypeFuelYear" do
-    #   AutomobileTypeFuelYear.run_data_miner!
-    #   
-    #   scope = AutomobileTypeFuelYear.where(:year => AutomobileTypeFuelYear.maximum(:year))
-    #   fe = scope.sum(:total_travel) / scope.sum(:fuel_consumption)
-    #   units = scope.first.total_travel_units + '_per_' + scope.first.fuel_consumption_units.singularize
-    #   
-    #   connection.execute %{
-    #     UPDATE countries
-    #     SET automobile_fuel_efficiency = #{fe},
-    #         automobile_fuel_efficiency_units = '#{units}'
-    #     WHERE iso_3166_code = 'US'
-    #   }
-    # end
+    process "Derive US average automobile fuel efficiency from AutomobileTypeFuelYear" do
+      scope = AutomobileTypeFuelYear.where(:year => AutomobileTypeFuelYear.maximum(:year))
+      fe = scope.sum(:total_travel) / scope.sum(:fuel_consumption)
+      units = scope.first.total_travel_units + '_per_' + scope.first.fuel_consumption_units.singularize
+      
+      connection.execute %{
+        UPDATE countries
+        SET automobile_fuel_efficiency = #{fe},
+            automobile_fuel_efficiency_units = '#{units}'
+        WHERE iso_3166_code = 'US'
+      }
+    end
     
     process "Convert automobile city speed from miles per hour to kilometres per hour" do
       conversion_factor = 1.miles.to(:kilometres)

data/lib/earth/locality/country.rb

@@ -13,13 +13,24 @@ class Country < ActiveRecord::Base
                 :automobile_trip_distance_units => lambda { Country.united_states.automobile_trip_distance_units }, # for now assume US represents world
                 :flight_route_inefficiency_factor => lambda { Country.maximum(:flight_route_inefficiency_factor) } # default to the largest inefficiency factor
   
-  data_miner do
-    tap "Brighter Planet's sanitized countries list", Earth.taps_server
-  end
-  
   class << self
     def united_states
       find_by_iso_3166_code('US')
     end
   end
+  
+  create_table do
+    string 'iso_3166_code'
+    string 'name'
+    float  'automobile_urbanity'
+    float  'automobile_fuel_efficiency'
+    string 'automobile_fuel_efficiency_units'
+    float  'automobile_city_speed'
+    string 'automobile_city_speed_units'
+    float  'automobile_highway_speed'
+    string 'automobile_highway_speed_units'
+    float  'automobile_trip_distance'
+    string 'automobile_trip_distance_units'
+    float  'flight_route_inefficiency_factor'
+  end
 end

data/lib/earth/locality/data_miner.rb

@@ -8,3 +8,6 @@ require 'earth/locality/petroleum_administration_for_defense_district/data_miner
 require 'earth/locality/state/data_miner'
 require 'earth/locality/urbanity/data_miner'
 require 'earth/locality/zip_code/data_miner'
+
+require 'earth/automobile/data_miner'
+require 'earth/residence/data_miner'

data/lib/earth/locality/egrid_region/data_miner.rb

@@ -1,10 +1,5 @@
 EgridRegion.class_eval do
   data_miner do
-    schema Earth.database_options do
-      string   'name'
-      float    'loss_factor'
-    end
-    
     import "eGRID regions and loss factors derived from eGRID 2007 data",
            :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2007_Version1-1.zip',
            :filename => 'eGRID2007_Version1-1/eGRID2007V1_1_year0504_STIE_USGC.xls',

data/lib/earth/locality/egrid_region.rb

@@ -3,15 +3,12 @@ class EgridRegion < ActiveRecord::Base
   
   has_many :egrid_subregions, :foreign_key => 'egrid_region_name'
   
-  data_miner do
-    tap "Brighter Planet's egrid region data", Earth.taps_server
-    
-    process "pull dependencies" do
-      run_data_miner_on_belongs_to_associations
-    end
-  end
-  
   # FIXME TODO someday should calculate this from eGRID data: (USTNGN05 + USTNFI05 - USTCON05) / USTNGN05
   falls_back_on :name => 'fallback',
                 :loss_factor => 0.061879 # calculated using above formula 2/21/2011
+  
+  create_table do
+    string   'name'
+    float    'loss_factor'
+  end
 end

data/lib/earth/locality/egrid_subregion/data_miner.rb

@@ -1,24 +1,5 @@
 EgridSubregion.class_eval do
   data_miner do
-    schema Earth.database_options do
-      string 'abbreviation'
-      string 'name'
-      string 'nerc_abbreviation'
-      string 'egrid_region_name'
-      float  'net_generation'
-      string 'net_generation_units'
-      float  'electricity_co2_emission_factor'
-      string 'electricity_co2_emission_factor_units'
-      float  'electricity_co2_biogenic_emission_factor'
-      string 'electricity_co2_biogenic_emission_factor_units'
-      float  'electricity_ch4_emission_factor'
-      string 'electricity_ch4_emission_factor_units'
-      float  'electricity_n2o_emission_factor'
-      string 'electricity_n2o_emission_factor_units'
-      float  'electricity_emission_factor'
-      string 'electricity_emission_factor_units'
-    end
-    
     # FIXME TODO for some reason this doesn't work...
     # import "eGRID regions and electricity emission factors derived from eGRID 2007 data",
     #        :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2007_Version1-1.zip',
@@ -56,7 +37,7 @@ EgridSubregion.class_eval do
                     WHERE electricity_co2_emission_factor_units = 'pounds_per_megawatt_hour' }
     end
     
-    process "Insure necessary datasets are imported" do
+    process "Insure GreenhouseGas is populated" do
       GreenhouseGas.run_data_miner!
     end
     

data/lib/earth/locality/egrid_subregion.rb

@@ -23,11 +23,22 @@ class EgridSubregion < ActiveRecord::Base
                 :electricity_emission_factor => lambda { weighted_average(:electricity_emission_factor, :weighted_by => :net_generation) }, # DEPRECATED
                 :electricity_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour' # DEPRECATED
   
-  data_miner do
-    tap "Brighter Planet's egrid subregion data", Earth.taps_server
-    
-    process "pull dependencies" do
-      run_data_miner_on_belongs_to_associations
-    end
+  create_table do
+    string 'abbreviation'
+    string 'name'
+    string 'nerc_abbreviation'
+    string 'egrid_region_name'
+    float  'net_generation'
+    string 'net_generation_units'
+    float  'electricity_co2_emission_factor'
+    string 'electricity_co2_emission_factor_units'
+    float  'electricity_co2_biogenic_emission_factor'
+    string 'electricity_co2_biogenic_emission_factor_units'
+    float  'electricity_ch4_emission_factor'
+    string 'electricity_ch4_emission_factor_units'
+    float  'electricity_n2o_emission_factor'
+    string 'electricity_n2o_emission_factor_units'
+    float  'electricity_emission_factor'
+    string 'electricity_emission_factor_units'
   end
 end

data/lib/earth/locality/petroleum_administration_for_defense_district/data_miner.rb

@@ -1,21 +1,12 @@
 PetroleumAdministrationForDefenseDistrict.class_eval do
   data_miner do
-    schema Earth.database_options do
-      string   'code'
-      string   'district_code'
-      string   'district_name'
-      string   'subdistrict_code'
-      string   'subdistrict_name'
-    end
-    
     import "a list of PADD districts and states derived from the EIA PADD definitions",
            :url => 'http://spreadsheets.google.com/pub?key=t5HM1KbaRngmTUbntg8JwPA&gid=0&output=csv' do
       key   'code',             :field_name => 'Code'
-      store 'district_code',    :field_name => 'PAD district code'
-      store 'subdistrict_code', :field_name => 'PAD subdistrict code'
-      store 'district_name',    :field_name => 'PAD district name'
-      store 'subdistrict_name', :field_name => 'PAD subdistrict name'
+      store 'district_code',    :field_name => 'PAD district code',    :nullify => true
+      store 'subdistrict_code', :field_name => 'PAD subdistrict code', :nullify => true
+      store 'district_name',    :field_name => 'PAD district name',    :nullify => true
+      store 'subdistrict_name', :field_name => 'PAD subdistrict name', :nullify => true
     end
   end
 end
-

data/lib/earth/locality/petroleum_administration_for_defense_district.rb

@@ -2,13 +2,17 @@ class PetroleumAdministrationForDefenseDistrict < ActiveRecord::Base
   set_primary_key :code
   set_table_name :petroleum_districts
   
-  data_miner do
-    tap "Brighter Planet's PADD info", Earth.taps_server
-  end
-  
   def name
     str = "PAD District #{district_code} (#{district_name})"
     str << " Subdistrict #{district_code}#{subdistrict_code} (#{subdistrict_name})" if subdistrict_code
     str
   end
+  
+  create_table do
+    string   'code'
+    string   'district_code'
+    string   'district_name'
+    string   'subdistrict_code'
+    string   'subdistrict_name'
+  end
 end