earth 0.11.0 → 0.11.1

data/bin/earth_tester.rb

@@ -55,6 +55,7 @@ require 'earth'
 domain = ARGV[0]
 
 Earth.init domain, :load_data_miner => true, :apply_schemas => true
+DataMiner::Run.create_tables
 
 ActiveRecord::Base.logger = Logger.new $stderr
 ActiveRecord::Base.logger.level = Logger::INFO

data/earth.gemspec

@@ -5,7 +5,7 @@ require "earth/version"
 Gem::Specification.new do |s|
   s.name        = "earth"
   s.version     = Earth::VERSION
-  s.date = "2011-10-05"
+  s.date = "2011-12-06"
   s.platform    = Gem::Platform::RUBY
   s.authors = ["Seamus Abshere", "Derek Kastner", "Andy Rossmeissl"]
   s.email = %q{andy@rossmeissl.net}

data/lib/earth.rb

@@ -131,7 +131,7 @@ module Earth
   end
   
   TAPS_STEP = 'Tap the Brighter Planet data server'
-  TAPS_SOURCE = 'http://carbon:neutral@data.brighterplanet.com'
+  TAPS_SOURCE = 'http://carbon:neutral@data.brighterplanet.com:5000'
   def _prepend_taps_step_to_data_miner(resource)
     resource_model = resource.constantize
     return if resource_model.data_miner_config.steps.any? { |step| step.description == TAPS_STEP }

data/lib/earth/air/flight_segment/data_miner.rb

@@ -10,13 +10,15 @@ FlightSegment.class_eval do
     end
   end
   
-  URL = 'http://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=293&Has_Group=3&Is_Zipped=0'
+  URL = 'http://www.transtats.bts.gov/DownLoad_Table.asp'
   FORM_DATA = %{
+    Table_ID=293&
+    Has_Group=3&
     UserTableName=T_100_Segment__All_Carriers&
     DBShortName=Air_Carriers&
     RawDataTable=T_T100_SEGMENT_ALL_CARRIER&
-    sqlstr=+SELECT+DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE+FROM++T_T100_SEGMENT_ALL_CARRIER+WHERE+Month+%3D__MONTH_NUMBER__+AND+YEAR%3D__YEAR__&
-    varlist=DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE&
+    sqlstr=+SELECT+DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE+FROM++T_T100_SEGMENT_ALL_CARRIER+WHERE+Month+%3D__MONTH_NUMBER__+AND+YEAR%3D__YEAR__&
+    varlist=DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE&
     grouplist=&
     suml=&
     sumRegion=&
@@ -92,9 +94,6 @@ FlightSegment.class_eval do
     VarName=ORIGIN_CITY_NAME&
     VarDesc=OriginCityName&
     VarType=Char&
-    VarName=ORIGIN_CITY_NUM&
-    VarDesc=OriginCityNum&
-    VarType=Num&
     VarName=ORIGIN_STATE_ABR&
     VarDesc=OriginState&
     VarType=Char&
@@ -119,9 +118,6 @@ FlightSegment.class_eval do
     VarName=DEST_CITY_NAME&
     VarDesc=DestCityName&
     VarType=Char&
-    VarName=DEST_CITY_NUM&
-    VarDesc=DestCityNum&
-    VarType=Num&
     VarName=DEST_STATE_ABR&
     VarDesc=DestState&
     VarType=Char&

data/lib/earth/automobile/automobile_size_class_year.rb

@@ -1,3 +1,4 @@
+# Not yet in use - for if we want to autocaculate size class stuf - November 10, 2011
 class AutomobileSizeClassYear < ActiveRecord::Base
   set_primary_key :name
 

data/lib/earth/automobile/automobile_size_class_year/data_miner.rb

@@ -1,3 +1,4 @@
+# Not yet in use - for if we want to autocaculate size class stuf - November 10, 2011
 AutomobileSizeClassYear.class_eval do
   data_miner do
     import "automobile size class year fuel efficiencies from the 2010 EPA Fuel Economy Trends report",

data/lib/earth/data_miner.rb

@@ -10,3 +10,4 @@ require 'earth/pet/data_miner'
 require 'earth/rail/data_miner'
 require 'earth/residence/data_miner'
 require 'earth/shipping/data_miner'
+require 'earth/industry/data_miner'

data/lib/earth/fuel/fuel_type.rb

@@ -1,3 +1,4 @@
+# DEPRECATED but FuelPurchase still uses this
 class FuelType < ActiveRecord::Base
   set_primary_key :name
   

data/lib/earth/fuel/fuel_type/data_miner.rb

@@ -1,3 +1,4 @@
+# DEPRECATED but FuelPurchase still uses this
 FuelType.class_eval do
   # FIXME TODO phase 2
   # annual emissions factors

data/lib/earth/hospitality.rb

@@ -1 +1,3 @@
+require 'earth/hospitality/census_region_lodging_class'
+require 'earth/hospitality/country_lodging_class'
 require 'earth/hospitality/lodging_class'

data/lib/earth/hospitality/census_region_lodging_class.rb

@@ -0,0 +1,16 @@
+class CensusRegionLodgingClass < ActiveRecord::Base
+  set_primary_key :name
+  
+  col :name
+  col :census_region_number
+  col :lodging_class_name
+  col :electricity_intensity, :type => :float
+  col :electricity_intensity_units
+  col :natural_gas_intensity, :type => :float
+  col :natural_gas_intensity_units
+  col :fuel_oil_intensity, :type => :float
+  col :fuel_oil_intensity_units
+  col :district_heat_intensity, :type => :float
+  col :district_heat_intensity_units
+  col :weighting, :type => :float
+end

data/lib/earth/hospitality/census_region_lodging_class/data_miner.rb

@@ -0,0 +1,40 @@
+CensusRegionLodgingClass.class_eval do
+  data_miner do
+    import "US census region lodging class fuel intensities derived from CBECS 2003",
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGkxNExTajZPSjRWU3REVks5SFJ0cmc&output=csv',
+           :select => lambda { |row| row['census_region_lodging_class'].present? } do
+      key   'name', :field_name => 'census_region_lodging_class'
+      store 'census_region_number', :field_name => 'census_region'
+      store 'lodging_class_name',    :field_name => 'lodging_class'
+      store 'electricity_intensity',   :units_field_name => 'electricity_intensity_units'
+      store 'natural_gas_intensity',   :units_field_name => 'natural_gas_intensity_units'
+      store 'fuel_oil_intensity',      :units_field_name => 'fuel_oil_intensity_units'
+      store 'district_heat_intensity', :units_field_name => 'district_heat_intensity_units'
+      store 'weighting'
+    end
+    
+    process "Convert natural gas intensities to metric units" do
+      conversion_factor = 2.83168466 # Google: 2.83168466 cubic m / 100 cubic ft
+      where(:natural_gas_intensity_units => 'hundred_cubic_feet_per_room_night').update_all(%{
+        natural_gas_intensity = 1.0 * natural_gas_intensity * #{conversion_factor},
+        natural_gas_intensity_units = 'cubic_metres_per_room_night'
+      })
+    end
+    
+    process "Convert fuel oil intensities to metric units" do
+      conversion_factor = 3.78541178 # Google: 3.78541178 l / gal
+      where(:fuel_oil_intensity_units => 'gallons_per_room_night').update_all(%{
+        fuel_oil_intensity = 1.0 * fuel_oil_intensity * #{conversion_factor},
+        fuel_oil_intensity_units = 'litres_per_room_night'
+      })
+    end
+    
+    process "Convert district heat intensities to metric units" do
+      conversion_factor = 1.05505585 # Google: 1.05505585 MJ / 1000 Btu
+      where(:district_heat_intensity_units => 'thousand_btu_per_room_night').update_all(%{
+        district_heat_intensity = 1.0 * district_heat_intensity * #{conversion_factor},
+        district_heat_intensity_units = 'megajoules_per_room_night'
+      })
+    end
+  end
+end

data/lib/earth/hospitality/country_lodging_class.rb

@@ -0,0 +1,16 @@
+class CountryLodgingClass < ActiveRecord::Base
+  set_primary_key :name
+  
+  col :name
+  col :country_iso_3166_code
+  col :lodging_class_name
+  col :electricity_intensity, :type => :float
+  col :electricity_intensity_units
+  col :natural_gas_intensity, :type => :float
+  col :natural_gas_intensity_units
+  col :fuel_oil_intensity, :type => :float
+  col :fuel_oil_intensity_units
+  col :district_heat_intensity, :type => :float
+  col :district_heat_intensity_units
+  col :weighting, :type => :float
+end

data/lib/earth/hospitality/country_lodging_class/data_miner.rb

@@ -0,0 +1,40 @@
+CountryLodgingClass.class_eval do
+  data_miner do
+    import "US lodging classes and their fuel intensities derived from CBECS 2003",
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGkxNExTajZPSjRWU3REVks5SFJ0cmc&output=csv',
+           :select => lambda { |row| row['country_lodging_class'].present? } do
+      key   'name', :field_name => 'country_lodging_class'
+      store 'country_iso_3166_code', :field_name => 'country'
+      store 'lodging_class_name',    :field_name => 'lodging_class'
+      store 'electricity_intensity',   :units_field_name => 'electricity_intensity_units'
+      store 'natural_gas_intensity',   :units_field_name => 'natural_gas_intensity_units'
+      store 'fuel_oil_intensity',      :units_field_name => 'fuel_oil_intensity_units'
+      store 'district_heat_intensity', :units_field_name => 'district_heat_intensity_units'
+      store 'weighting'
+    end
+    
+    process "Convert natural gas intensities to metric units" do
+      conversion_factor = 2.83168466 # Google: 2.83168466 cubic m / 100 cubic ft
+      where(:natural_gas_intensity_units => 'hundred_cubic_feet_per_room_night').update_all(%{
+        natural_gas_intensity = 1.0 * natural_gas_intensity * #{conversion_factor},
+        natural_gas_intensity_units = 'cubic_metres_per_room_night'
+      })
+    end
+    
+    process "Convert fuel oil intensities to metric units" do
+      conversion_factor = 3.78541178 # Google: 3.78541178 l / gal
+      where(:fuel_oil_intensity_units => 'gallons_per_room_night').update_all(%{
+        fuel_oil_intensity = 1.0 * fuel_oil_intensity * #{conversion_factor},
+        fuel_oil_intensity_units = 'litres_per_room_night'
+      })
+    end
+    
+    process "Convert district heat intensities to metric units" do
+      conversion_factor = 1.05505585 # Google: 1.05505585 MJ / 1000 Btu
+      where(:district_heat_intensity_units => 'thousand_btu_per_room_night').update_all(%{
+        district_heat_intensity = 1.0 * district_heat_intensity * #{conversion_factor},
+        district_heat_intensity_units = 'megajoules_per_room_night'
+      })
+    end
+  end
+end

data/lib/earth/hospitality/data_miner.rb

@@ -1 +1,3 @@
+require 'earth/hospitality/census_region_lodging_class/data_miner'
+require 'earth/hospitality/country_lodging_class/data_miner'
 require 'earth/hospitality/lodging_class/data_miner'

data/lib/earth/hospitality/lodging_class.rb

@@ -2,6 +2,8 @@ class LodgingClass < ActiveRecord::Base
   set_primary_key :name
   
   col :name
+  
+  # DEPRECATED - cut once new Lodging has been phased in
   col :electricity_intensity, :type => :float
   col :electricity_intensity_units
   col :natural_gas_intensity, :type => :float
@@ -10,4 +12,4 @@ class LodgingClass < ActiveRecord::Base
   col :fuel_oil_intensity_units
   col :district_heat_intensity, :type => :float
   col :district_heat_intensity_units
-end
+end

data/lib/earth/hospitality/lodging_class/data_miner.rb

@@ -1,11 +1,12 @@
 LodgingClass.class_eval do
   data_miner do
-    import "a list of lodging classes and pre-calculated emission factors",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGZZWmZtWEJlYzhRNXlPdWpBTldlcUE&hl=en&gid=0&output=csv' do
+    # DEPRECATED - once new Lodging is phased in replace this with an import of lodging class names from the CountryLodgingClass google doc
+    import "a list of lodging classes and pre-calculated energy intensities",
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGZZWmZtWEJlYzhRNXlPdWpBTldlcUE&output=csv' do
       key   'name'
-      store 'natural_gas_intensity', :units_field_name => 'natural_gas_intensity_units'
-      store 'fuel_oil_intensity', :units_field_name => 'fuel_oil_intensity_units'
-      store 'electricity_intensity', :units_field_name => 'electricity_intensity_units'
+      store 'electricity_intensity',   :units_field_name => 'electricity_intensity_units'
+      store 'natural_gas_intensity',   :units_field_name => 'natural_gas_intensity_units'
+      store 'fuel_oil_intensity',      :units_field_name => 'fuel_oil_intensity_units'
       store 'district_heat_intensity', :units_field_name => 'district_heat_intensity_units'
     end
     

data/lib/earth/industry/data_miner.rb

@@ -0,0 +1 @@
+require 'earth/industry/industry/data_miner'

data/lib/earth/industry/industry.rb

@@ -10,7 +10,7 @@ class Industry < ActiveRecord::Base
   has_many :sectors, :through => :industry_sectors
   
   col :naics_code
-  col :description, :type => :text
+  col :description
 
   def trade_industry?
     prefix = naics_code.to_s[0,2]

data/lib/earth/industry/industry/data_miner.rb

@@ -0,0 +1,12 @@
+Industry.class_eval do
+  data_miner do
+    import "the U.S. Census 2002 NAICS code list",
+           :url => 'http://www.census.gov/epcd/naics02/naicod02.txt',
+           :skip => 4,
+           :headers => false,
+           :delimiter => '	' do
+      key 'naics_code', :field_number => 0
+      store 'description', :field_number => 1
+    end
+  end
+end

data/lib/earth/industry/merchant_category.rb

@@ -9,5 +9,5 @@ class MerchantCategory < ActiveRecord::Base
   end
 
   col :mcc
-  col :description, :type => :text
+  col :description
 end

data/lib/earth/locality.rb

@@ -2,16 +2,14 @@ require 'earth/locality/census_division'
 require 'earth/locality/census_region'
 require 'earth/locality/climate_division'
 require 'earth/locality/country'
-require 'earth/locality/country_rail_traction'
-require 'earth/locality/country_rail_class'
-require 'earth/locality/country_rail_traction_class'
+require 'earth/locality/egrid_country'
 require 'earth/locality/egrid_region'
 require 'earth/locality/egrid_subregion'
 require 'earth/locality/petroleum_administration_for_defense_district'
 require 'earth/locality/state'
-require 'earth/locality/urbanity'
 require 'earth/locality/zip_code'
 
-require 'earth/automobile'
-require 'earth/residence'
-require 'earth/rail'
+require 'earth/automobile'  # for automobile data on Country
+require 'earth/fuel'        # for GWPs for EgridSubregion
+require 'earth/hospitality' # for lodging data on Country
+require 'earth/rail'        # for rail data on Country

data/lib/earth/locality/census_division/data_miner.rb

@@ -29,13 +29,14 @@ CensusDivision.class_eval do
       store 'meeting_building_district_heat_intensity', :from_units => :thousand_btu_per_square_foot_hour, :to_units => :megajoules_per_square_metre_hour
     end
     
-    import 'lodging building fuel intensities calculated from CBECS 2003',
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHZ1QjdNTGVpbWxmZDVLbjlpTHlFVHc&hl=en&gid=0&output=csv' do
+    import 'lodging building fuel intensities derived from CBECS 2003',
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGkxNExTajZPSjRWU3REVks5SFJ0cmc&output=csv',
+           :select => lambda { |row| row['census_division'].to_i.between?(1, 9) } do
       key   'number', :field_name => 'census_division'
-      store 'lodging_building_natural_gas_intensity', :from_units => :hundred_cubic_feet_per_room_night, :to_units => :cubic_metres_per_room_night
-      store 'lodging_building_fuel_oil_intensity', :from_units => :gallons_per_room_night, :to_units => :litres_per_room_night
-      store 'lodging_building_electricity_intensity', :units_field_name => 'lodging_building_electricity_intensity_units'
-      store 'lodging_building_district_heat_intensity', :from_units => :thousand_btu_per_room_night, :to_units => :megajoules_per_room_night
+      store 'lodging_building_electricity_intensity',   :field_name => 'electricity_intensity',   :units_field_name => 'electricity_intensity_units'
+      store 'lodging_building_natural_gas_intensity',   :field_name => 'natural_gas_intensity',   :from_units => :hundred_cubic_feet_per_room_night, :to_units => :cubic_metres_per_room_night
+      store 'lodging_building_fuel_oil_intensity',      :field_name => 'fuel_oil_intensity',      :from_units => :gallons_per_room_night,            :to_units => :litres_per_room_night
+      store 'lodging_building_district_heat_intensity', :field_name => 'district_heat_intensity', :from_units => :thousand_btu_per_room_night,       :to_units => :megajoules_per_room_night
     end
   end
 end

data/lib/earth/locality/country.rb

@@ -1,7 +1,8 @@
 class Country < ActiveRecord::Base
   set_primary_key :iso_3166_code
   
-  has_many :rail_companies, :foreign_key => 'country_iso_3166_code'
+  has_many :rail_companies,  :foreign_key => 'country_iso_3166_code' # used to calculate rail data
+  has_many :lodging_classes, :foreign_key => 'country_iso_3166_code', :class_name => 'CountryLodgingClass'
   
   falls_back_on :name => 'fallback',
                 :automobile_urbanity => lambda { united_states.automobile_urbanity }, # for now assume US represents world
@@ -13,7 +14,17 @@ class Country < ActiveRecord::Base
                 :automobile_highway_speed_units => lambda { united_states.automobile_highway_speed_units }, # for now assume US represents world
                 :automobile_trip_distance => lambda { united_states.automobile_trip_distance }, # for now assume US represents world
                 :automobile_trip_distance_units => lambda { united_states.automobile_trip_distance_units }, # for now assume US represents world
+                :electricity_emission_factor => lambda { maximum(:electricity_emission_factor) }, # default to the largest electricity emission factor
+                :electricity_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour', # FIXME TODO derive this
                 :flight_route_inefficiency_factor => lambda { maximum(:flight_route_inefficiency_factor) }, # default to the largest inefficiency factor
+                :lodging_natural_gas_intensity => lambda { united_states.lodging_natural_gas_intensity }, # for now assume US represents world
+                :lodging_natural_gas_intensity_units => lambda { united_states.lodging_natural_gas_intensity_units }, # for now assume US represents world
+                :lodging_fuel_oil_intensity => lambda { united_states.lodging_fuel_oil_intensity }, # for now assume US represents world
+                :lodging_fuel_oil_intensity_units => lambda { united_states.lodging_fuel_oil_intensity_units }, # for now assume US represents world
+                :lodging_electricity_intensity => lambda { united_states.lodging_electricity_intensity }, # for now assume US represents world
+                :lodging_electricity_intensity_units => lambda { united_states.lodging_electricity_intensity_units }, # for now assume US represents world
+                :lodging_district_heat_intensity => lambda { united_states.lodging_district_heat_intensity }, # for now assume US represents world
+                :lodging_district_heat_intensity_units => lambda { united_states.lodging_district_heat_intensity_units }, # for now assume US represents world
                 :rail_trip_distance => lambda { weighted_average(:rail_trip_distance, :weighted_by => :rail_passengers) },
                 :rail_trip_distance_units => 'kilometres', # FIXME TODO derive this
                 :rail_speed => lambda { weighted_average(:rail_speed, :weighted_by => :rail_passengers) },
@@ -31,7 +42,9 @@ class Country < ActiveRecord::Base
     end
   end
   
-  col :iso_3166_code
+  col :iso_3166_code                            # alpha-2 2-letter like GB
+  col :iso_3166_numeric_code, :type => :integer # numeric like 826; aka UN M49 code
+  col :iso_3166_alpha_3_code                    # 3-letter like GBR
   col :name
   col :automobile_urbanity, :type => :float # float from 0 to 1
   col :automobile_fuel_efficiency, :type => :float
@@ -42,7 +55,17 @@ class Country < ActiveRecord::Base
   col :automobile_highway_speed_units
   col :automobile_trip_distance, :type => :float
   col :automobile_trip_distance_units
+  col :electricity_emission_factor, :type => :float
+  col :electricity_emission_factor_units
   col :flight_route_inefficiency_factor, :type => :float
+  col :lodging_natural_gas_intensity, :type => :float
+  col :lodging_natural_gas_intensity_units
+  col :lodging_fuel_oil_intensity, :type => :float
+  col :lodging_fuel_oil_intensity_units
+  col :lodging_electricity_intensity, :type => :float
+  col :lodging_electricity_intensity_units
+  col :lodging_district_heat_intensity, :type => :float
+  col :lodging_district_heat_intensity_units
   col :rail_passengers, :type => :float
   col :rail_trip_distance, :type => :float
   col :rail_trip_distance_units

data/lib/earth/locality/country/data_miner.rb

@@ -1,26 +1,25 @@
 Country.class_eval do
   data_miner do
-    import 'the official ISO country list',
-           :url => 'http://www.iso.org/iso/list-en1-semic-3.txt',
-           :skip => 2,
-           :headers => false,
+    # http://www.iso.org/iso/list-en1-semic-3.txt
+    # http://unstats.un.org/unsd/methods/m49/m49alpha.htm
+    import "OpenGeoCode.org's Country Codes to Country Names list",
+           :url => 'http://opengeocode.org/download/countrynames.txt',
+           :format => :delimited,
            :delimiter => ';',
-           :encoding => 'ISO-8859-1' do
-      key   'iso_3166_code', :field_number => 1
-      store 'name', :field_number => 0
-    end
-    
-    import "country-specific flight route inefficiency factors derived from Kettunen et al. (2005)",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEJoRVBZaGhnUmlhX240VXE3X0F3WkE&output=csv' do
-      key   'iso_3166_code'
-      store 'flight_route_inefficiency_factor'
+           :headers => false,
+           :skip => 22 do
+      key 'iso_3166_code', :field_number => 0
+      store 'iso_3166_alpha_3_code', :field_number => 1
+      store 'iso_3166_numeric_code', :field_number => 2
+      store 'name', :field_number => 5 # romanized version
     end
     
+    # AUTOMOBILE
     import "automobile-related data for the US",
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDdZRm1tNjY0c2dYNG00bXJ3TXRqUVE&gid=0&output=csv' do
       key 'iso_3166_code'
       store 'automobile_urbanity'
-      store 'automobile_city_speed', :units_field_name => 'automobile_city_speed_units'
+      store 'automobile_city_speed',    :units_field_name => 'automobile_city_speed_units'
       store 'automobile_highway_speed', :units_field_name => 'automobile_highway_speed_units'
       store 'automobile_trip_distance', :units_field_name => 'automobile_trip_distance_units'
     end
@@ -29,7 +28,6 @@ Country.class_eval do
       AutomobileTypeFuelYear.run_data_miner!
     end
     
-    # FIXME TODO eventually need to do this for all countries
     process "Derive US average automobile fuel efficiency from AutomobileTypeFuelYear" do
       fuel_years = AutomobileTypeFuelYear.where(:year => AutomobileTypeFuelYear.maximum(:year))
       where(:iso_3166_code => 'US').update_all(
@@ -62,8 +60,67 @@ Country.class_eval do
       })
     end
     
-    process "Ensure RailCompany is populated" do
+    # ELECTRICITY
+    import "calculate national electricity emission factors from Brander et al. (2011)",
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDZmWHFjLVdBZGRBdGxVdDdqd1YtYWc&output=csv' do
+      key 'iso_3166_code', :field_name => 'country_iso_3166_code'
+      store 'electricity_emission_factor', :synthesize => lambda { |row|
+        (
+          row['electricity_co2_emission_factor'].to_f +
+          row['electricity_ch4_emission_factor'].to_f +
+          row['electricity_n2o_emission_factor'].to_f
+        ) / (1 - row['loss_factor'].to_f) }, :units_field_name => 'electricity_ch4_emission_factor_units'
+    end
+    
+    process "Ensure EgridSubregion is populated" do
+      EgridSubregion.run_data_miner!
+    end
+    
+    process "Derive average US electricity emission factor from eGRID" do
+      us = united_states
+      subregion = EgridSubregion.find_by_abbreviation 'US'
+      us.electricity_emission_factor =
+        (
+          subregion.electricity_co2_emission_factor +
+          subregion.electricity_ch4_emission_factor +
+          subregion.electricity_n2o_emission_factor
+        ) / (1 - subregion.egrid_region.loss_factor)
+      us.electricity_emission_factor_units = 'kilograms_co2e_per_kilowatt_hour'
+      us.save!
+    end
+    
+    # FLIGHT
+    import "country-specific flight route inefficiency factors derived from Kettunen et al. (2005)",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEJoRVBZaGhnUmlhX240VXE3X0F3WkE&output=csv' do
+      key   'iso_3166_code'
+      store 'flight_route_inefficiency_factor'
+    end
+    
+    # HOSPITALITY
+    process "Ensure CountryLodgingClass is populated" do
+      CountryLodgingClass.run_data_miner!
+    end
+    
+    process "Derive average hotel characteristics from CountryLodgingClass" do
+      find_each do |country|
+        if country.lodging_classes.any?
+          country.lodging_natural_gas_intensity = country.lodging_classes.weighted_average(:natural_gas_intensity)
+          country.lodging_natural_gas_intensity_units = 'cubic_metres_per_room_night' # FIXME TODO derive this
+          country.lodging_fuel_oil_intensity = country.lodging_classes.weighted_average(:fuel_oil_intensity)
+          country.lodging_fuel_oil_intensity_units = 'gallons_per_room_night' # FIXME TODO derive this
+          country.lodging_electricity_intensity = country.lodging_classes.weighted_average(:electricity_intensity)
+          country.lodging_electricity_intensity_units = 'kilowatt_hours_per_room_night' # FIXME TODO derive this
+          country.lodging_district_heat_intensity = country.lodging_classes.weighted_average(:district_heat_intensity)
+          country.lodging_district_heat_intensity_units = 'megajoules_per_room_night' # FIXME TODO derive this
+          country.save!
+        end
+      end
+    end
+    
+    # RAIL
+    process "Ensure RailCompany and RailFuel are populated" do
       RailCompany.run_data_miner!
+      RailFuel.run_data_miner!
     end
     
     process "Calculate rail passengers, trip distance, and speed from RailCompany" do
@@ -87,10 +144,6 @@ Country.class_eval do
       store 'rail_trip_co2_emission_factor',   :units_field_name => 'rail_trip_co2_emission_factor_units' 
     end
     
-    process "Ensure RailFuel is populated" do
-      RailFuel.run_data_miner!
-    end
-    
     process "Unit conversion for European rail diesel intensity" do
       diesel = RailFuel.find_by_name("diesel")
       where(:rail_trip_diesel_intensity_units => 'grams_per_passenger_kilometre').update_all(%{
@@ -107,7 +160,7 @@ Country.class_eval do
     end
     
     process "Derive US rail fuel and emission data from RailCompany" do
-      country = Country.united_states
+      country = united_states
       country.rail_trip_electricity_intensity = country.rail_companies.weighted_average(:electricity_intensity, :weighted_by => :passengers)
       country.rail_trip_electricity_intensity_units = 'kilowatt_hours_per_passenger_kilometre'
       country.rail_trip_diesel_intensity = country.rail_companies.weighted_average(:diesel_intensity, :weighted_by => :passengers)
@@ -116,7 +169,5 @@ Country.class_eval do
       country.rail_trip_co2_emission_factor_units = 'kilograms_per_passenger_kilometre'
       country.save!
     end
-    
-    # FIXME TODO verify this
   end
 end