earth 0.12.3 → 0.12.4

data/lib/earth/hospitality/lodging_class.rb

@@ -1,4 +1,3 @@
-require 'earth/locality'
 class LodgingClass < ActiveRecord::Base
   self.primary_key = "name"
   

data/lib/earth/hospitality/lodging_class/data_miner.rb

@@ -1,14 +1,8 @@
-require 'earth/locality/data_miner'
 LodgingClass.class_eval do
   data_miner do
-    process "Ensure CountryLodgingClass is populated" do
-      CountryLodgingClass.run_data_miner!
-    end
-    
-    process "Derive lodging classes from CountryLodgingClass" do
-      connection.select_values("SELECT DISTINCT lodging_class_name FROM #{CountryLodgingClass.quoted_table_name}").each do |lodging_class_name|
-        find_or_create_by_name lodging_class_name
-      end
+    import "A Brighter Planet-curated list of lodging classes",
+           :url => "file://#{Earth::DATA_DIR}/hospitality/lodging_classes.csv" do
+      key :name
     end
   end
 end

data/lib/earth/industry/merchant_category/data_miner.rb

@@ -0,0 +1,14 @@
+MerchantCategory.data_miner do
+  process "Start from scratch" do
+    delete_all
+  end
+  
+  import "the IRS Merchant Category Code table",
+         :url => 'http://www.irs.gov/irb/2004-31_IRB/ar17.html',
+         :headers => %w{mcc description ignore},
+         :row_css => 'div.informaltable tbody tr',
+         :column_css => 'td' do
+    key 'mcc'
+    store 'description'
+  end
+end

data/lib/earth/locality/country.rb

@@ -1,5 +1,4 @@
 require 'earth/automobile'
-require 'earth/fuel'
 require 'earth/hospitality'
 require 'earth/rail'
 
@@ -7,7 +6,17 @@ class Country < ActiveRecord::Base
   self.primary_key = "iso_3166_code"
   
   has_many :rail_companies,  :foreign_key => 'country_iso_3166_code' # used to calculate rail data
-  has_many :lodging_classes, :foreign_key => 'country_iso_3166_code', :class_name => 'CountryLodgingClass'
+  has_one :electricity_mix, :foreign_key => 'country_iso_3166_code'
+  
+  data_miner do
+    process "Ensure ElectricityMix is imported because it's like a belongs_to association" do
+      ElectricityMix.run_data_miner!
+    end
+  end
+  
+  def self.united_states
+    find_by_iso_3166_code('US')
+  end
   
   falls_back_on :name => 'fallback',
                 :automobile_urbanity => lambda { united_states.automobile_urbanity }, # for now assume US represents world
@@ -19,8 +28,15 @@ class Country < ActiveRecord::Base
                 :automobile_highway_speed_units => lambda { united_states.automobile_highway_speed_units }, # for now assume US represents world
                 :automobile_trip_distance => lambda { united_states.automobile_trip_distance }, # for now assume US represents world
                 :automobile_trip_distance_units => lambda { united_states.automobile_trip_distance_units }, # for now assume US represents world
+                :electricity_mix => lambda { ElectricityMix.fallback },
                 :electricity_emission_factor => 0.626089, # from ecometrica paper - FIXME TODO calculate this
                 :electricity_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour', # FIXME TODO derive this
+                :electricity_co2_emission_factor => 0.623537, # from ecometrica paper - FIXME TODO calculate this
+                :electricity_co2_emission_factor_units => 'kilograms_per_kilowatt_hour', # FIXME TODO derive this
+                :electricity_ch4_emission_factor => 0.000208, # from ecometrica paper - FIXME TODO calculate this
+                :electricity_ch4_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour', # FIXME TODO derive this
+                :electricity_n2o_emission_factor => 0.002344, # from ecometrica paper - FIXME TODO calculate this
+                :electricity_n2o_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour', # FIXME TODO derive this
                 :electricity_loss_factor => 0.096, # from ecometrica paper - FIXME TODO calculate this
                 :flight_route_inefficiency_factor => lambda { maximum(:flight_route_inefficiency_factor) }, # default to the largest inefficiency factor
                 :lodging_occupancy_rate => lambda { united_states.lodging_occupancy_rate }, # for now assume US represents world
@@ -43,10 +59,6 @@ class Country < ActiveRecord::Base
                 :rail_trip_co2_emission_factor => lambda { weighted_average(:rail_trip_co2_emission_factor, :weighted_by => :rail_passengers) },
                 :rail_trip_co2_emission_factor_units => 'kilograms_per_passenger_kilometre' # FIXME TODO derive this
   
-  def self.united_states
-    find_by_iso_3166_code('US')
-  end
-  
   col :iso_3166_code                            # alpha-2 2-letter like GB
   col :iso_3166_numeric_code, :type => :integer # numeric like 826; aka UN M49 code
   col :iso_3166_alpha_3_code                    # 3-letter like GBR
@@ -66,6 +78,12 @@ class Country < ActiveRecord::Base
   col :automobile_trip_distance_units
   col :electricity_emission_factor, :type => :float
   col :electricity_emission_factor_units
+  col :electricity_co2_emission_factor, :type => :float
+  col :electricity_co2_emission_factor_units
+  col :electricity_ch4_emission_factor, :type => :float
+  col :electricity_ch4_emission_factor_units
+  col :electricity_n2o_emission_factor, :type => :float
+  col :electricity_n2o_emission_factor_units
   col :electricity_loss_factor, :type => :float
   col :flight_route_inefficiency_factor, :type => :float
   col :lodging_occupancy_rate, :type => :float
@@ -90,5 +108,13 @@ class Country < ActiveRecord::Base
   col :rail_trip_co2_emission_factor_units
   
   warn_unless_size 249
-  warn_if_nulls_except /heating/, /cooling/
+  warn_if_nulls_except(
+     /heating/,
+     /cooling/,
+     /automobile/,
+     /electricity/,
+     /flight/,
+     /lodging/,
+     /rail/
+  )
 end

data/lib/earth/locality/country/data_miner.rb

@@ -5,6 +5,10 @@ require 'earth/rail/data_miner'
 
 Country.class_eval do
   data_miner do
+    process "Start from scratch" do
+      delete_all
+    end
+    
     # http://www.iso.org/iso/list-en1-semic-3.txt
     # http://unstats.un.org/unsd/methods/m49/m49alpha.htm
     import "OpenGeoCode.org's Country Codes to Country Names list",
@@ -61,36 +65,48 @@ Country.class_eval do
       )
     end
     
+    # DEPRECATED - eventually cut all elec stuff (replaced by ElectricityMix)
     # ELECTRICITY
     process "Ensure GreehouseGas is populated" do
       GreenhouseGas.run_data_miner!
     end
     
-    import "calculate national electricity emission factors from Brander et al. (2011)",
-           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDZmWHFjLVdBZGRBdGxVdDdqd1YtYWc&output=csv' do
+    import "national average electricity emission factors from Brander et al. (2011)",
+           :url => "file://#{Earth::DATA_DIR}/locality/national_electricity_efs.csv" do
       key 'iso_3166_code', :field_name => 'country_iso_3166_code'
-      store 'electricity_emission_factor', :synthesize => proc { |row|
-        (
-          row['electricity_co2_emission_factor'].to_f +
-          (row['electricity_ch4_emission_factor'].to_f * GreenhouseGas[:ch4].global_warming_potential) +
-          (row['electricity_n2o_emission_factor'].to_f * GreenhouseGas[:n2o].global_warming_potential)
-        ) }, :units => 'kilograms_co2e_per_kilowatt_hour'
+      store 'electricity_co2_emission_factor', :field_name => 'co2_emission_factor', :units_field_name => 'co2_emission_factor_units'
+      store 'electricity_ch4_emission_factor', :synthesize => proc { |row| row['ch4_emission_factor'].to_f * GreenhouseGas[:ch4].global_warming_potential }, :units => 'kilograms_co2e_per_kilowatt_hour'
+      store 'electricity_n2o_emission_factor', :synthesize => proc { |row| row['n2o_emission_factor'].to_f * GreenhouseGas[:n2o].global_warming_potential }, :units => 'kilograms_co2e_per_kilowatt_hour'
       store 'electricity_loss_factor', :field_name => 'loss_factor'
     end
     
-    process "Ensure EgridSubregion and EgridRegion are populated" do
+    process "Ensure EgridSubregion is populated" do
       EgridSubregion.run_data_miner!
-      EgridRegion.run_data_miner!
     end
     
-    process "Derive average US electricity emission factor and loss factor from eGRID" do
+    process "Derive average US electricity data from EgridSubregion" do
       united_states.update_attributes!(
-        :electricity_emission_factor =>       EgridSubregion.fallback.electricity_emission_factor,
-        :electricity_emission_factor_units => EgridSubregion.fallback.electricity_emission_factor_units,
-        :electricity_loss_factor =>           EgridRegion.fallback.loss_factor
+        :electricity_co2_emission_factor =>       EgridSubregion.fallback.co2_emission_factor,
+        :electricity_co2_emission_factor_units => EgridSubregion.fallback.co2_emission_factor_units,
+        :electricity_ch4_emission_factor =>       EgridSubregion.fallback.ch4_emission_factor,
+        :electricity_ch4_emission_factor_units => EgridSubregion.fallback.ch4_emission_factor_units,
+        :electricity_n2o_emission_factor =>       EgridSubregion.fallback.n2o_emission_factor,
+        :electricity_n2o_emission_factor_units => EgridSubregion.fallback.n2o_emission_factor_units,
+        :electricity_loss_factor =>               EgridSubregion.fallback.egrid_region.loss_factor
       )
     end
     
+    process "Calculate combined electricity emission factor" do
+      where('electricity_co2_emission_factor IS NOT NULL').update_all(%{
+        electricity_emission_factor = electricity_co2_emission_factor + electricity_ch4_emission_factor + electricity_n2o_emission_factor,
+        electricity_emission_factor_units = 'kilograms_co2e_per_kilowatt_hour'
+      })
+    end
+    
+    process "Ensure ElectricityMix is data mined because it's like a belongs_to association" do
+      ElectricityMix.run_data_miner!
+    end
+    
     # FLIGHT
     import "country-specific flight route inefficiency factors derived from Kettunen et al. (2005)",
            :url => "file://#{Earth::DATA_DIR}/locality/country_flight_data.csv" do
@@ -103,25 +119,23 @@ Country.class_eval do
       united_states.update_attributes! :lodging_occupancy_rate => 0.601 # per http://www.pwc.com/us/en/press-releases/2012/pwc-us-lodging-industry-forecast.jhtml
     end
     
-    process "Ensure CountryLodgingClass is populated" do
-      CountryLodgingClass.run_data_miner!
+    process "Ensure CommercialBuildingEnergyConsumptionSurveyResponse is populated" do
+      CommercialBuildingEnergyConsumptionSurveyResponse.run_data_miner!
     end
     
-    process "Derive average hotel characteristics from CountryLodgingClass" do
-      find_each do |country|
-        if (lodging_classes = country.lodging_classes).any?
-          country.update_attributes!(
-            :lodging_natural_gas_intensity         => lodging_classes.weighted_average(:natural_gas_intensity),
-            :lodging_fuel_oil_intensity            => lodging_classes.weighted_average(:fuel_oil_intensity),
-            :lodging_electricity_intensity         => lodging_classes.weighted_average(:electricity_intensity),
-            :lodging_district_heat_intensity       => lodging_classes.weighted_average(:district_heat_intensity),
-            :lodging_natural_gas_intensity_units   => 'cubic_metres_per_occupied_room_night', # FIXME TODO derive this
-            :lodging_fuel_oil_intensity_units      => 'litres_per_occupied_room_night', # FIXME TODO derive this
-            :lodging_electricity_intensity_units   => 'kilowatt_hours_per_occupied_room_night', # FIXME TODO derive this
-            :lodging_district_heat_intensity_units => 'megajoules_per_occupied_room_night' # FIXME TODO derive this
-          )
-        end
-      end
+    process "Derive US average hotel characteristics from CommercialBuildingEnergyConsumptionSurveyResponse" do
+      lodgings = CommercialBuildingEnergyConsumptionSurveyResponse.lodging_records
+      
+      united_states.update_attributes!(
+        :lodging_natural_gas_intensity         => lodgings.weighted_average(:natural_gas_per_room_night),
+        :lodging_fuel_oil_intensity            => lodgings.weighted_average(:fuel_oil_per_room_night),
+        :lodging_electricity_intensity         => lodgings.weighted_average(:electricity_per_room_night),
+        :lodging_district_heat_intensity       => lodgings.weighted_average(:district_heat_per_room_night),
+        :lodging_natural_gas_intensity_units   => lodgings.first.natural_gas_per_room_night_units,
+        :lodging_fuel_oil_intensity_units      => lodgings.first.fuel_oil_per_room_night_units,
+        :lodging_electricity_intensity_units   => lodgings.first.electricity_per_room_night_units,
+        :lodging_district_heat_intensity_units => lodgings.first.district_heat_per_room_night_units
+      )
     end
     
     # RAIL
@@ -131,7 +145,7 @@ Country.class_eval do
     end
     
     process "Calculate rail passengers, trip distance, and speed from RailCompany" do
-      find_each do |country|
+      safe_find_each do |country|
         if (rail_companies = country.rail_companies).any?
           country.update_attributes!(
             :rail_passengers          => rail_companies.sum(:passengers),

data/lib/earth/locality/egrid_subregion.rb

@@ -3,8 +3,8 @@ require 'earth/locality/egrid_region'
 class EgridSubregion < ActiveRecord::Base
   self.primary_key = "abbreviation"
   
-  has_many :zip_codes, :foreign_key => 'egrid_subregion_abbreviation'
   belongs_to :egrid_region, :foreign_key => 'egrid_region_name'
+  has_one :electricity_mix, :foreign_key => 'egrid_subregion_abbreviation'
   
   falls_back_on :name => 'fallback',
                 :egrid_region => lambda { EgridRegion.fallback },

data/lib/earth/locality/egrid_subregion/data_miner.rb

@@ -14,10 +14,9 @@ EgridSubregion.class_eval do
       store 'name',                            :field_name => 'SRNAME'
       store 'nerc_abbreviation',               :field_name => 'NERC'
       store 'net_generation',                  :field_name => 'SRNGENAN', :units => :megawatt_hours
-      store 'co2_emission_factor', :field_name => 'SRCO2RTA', :units => :pounds_per_megawatt_hour
-      store 'co2_biogenic_emission_factor', :static => '0.0', :units => :kilograms_per_kilowatt_hour
-      store 'ch4_emission_factor', :field_name => 'SRCH4RTA', :units => :pounds_per_gigawatt_hour
-      store 'n2o_emission_factor', :field_name => 'SRN2ORTA', :units => :pounds_per_gigawatt_hour
+      store 'co2_emission_factor', :field_name => 'SRCO2RTA', :from_units => :pounds_per_megawatt_hour, :to_units => :kilograms_per_kilowatt_hour
+      store 'ch4_emission_factor', :field_name => 'SRCH4RTA', :from_units => :pounds_per_gigawatt_hour, :to_units => :kilograms_per_kilowatt_hour
+      store 'n2o_emission_factor', :field_name => 'SRN2ORTA', :from_units => :pounds_per_gigawatt_hour, :to_units => :kilograms_per_kilowatt_hour
     end
     
     import "eGRID subregion to region associations",
@@ -26,41 +25,27 @@ EgridSubregion.class_eval do
       store 'egrid_region_name'
     end
     
-    process "Convert co2 emission factors to metric units" do
-      conversion_factor = 1.pounds.to(:kilograms) / 1_000.0 # kg / lbs * MWh / kWh
-      where(:co2_emission_factor_units => 'pounds_per_megawatt_hour').update_all(%{
-        co2_emission_factor = co2_emission_factor * #{conversion_factor},
-        co2_emission_factor_units = 'kilograms_per_kilowatt_hour'
-      })
-    end
-    
     process "Insure GreenhouseGas is populated" do
       GreenhouseGas.run_data_miner!
     end
     
-    process "Convert ch4 emission factor to metric units and co2e" do
-      conversion_factor = 1.pounds.to(:kilograms) / 1_000_000.0 # kg / lbs * GWh / kWh
-      gwp = GreenhouseGas[:ch4].global_warming_potential
-      where(:ch4_emission_factor_units => 'pounds_per_gigawatt_hour').update_all(%{
-        ch4_emission_factor = ch4_emission_factor * #{conversion_factor} * #{gwp},
-        ch4_emission_factor_units = 'kilograms_co2e_per_kilowatt_hour'
-      })
-    end
-    
-    process "Convert n2o emission factor to metric units and co2e" do
-      conversion_factor = 1.pounds.to(:kilograms) / 1_000_000.0 # kg / lbs * GWh / kWh
-      gwp = GreenhouseGas[:n2o].global_warming_potential
-      where(:n2o_emission_factor_units => 'pounds_per_gigawatt_hour').update_all(%{
-        n2o_emission_factor = n2o_emission_factor * #{conversion_factor} * #{gwp},
-        n2o_emission_factor_units = 'kilograms_co2e_per_kilowatt_hour'
-      })
+    %w{ ch4 n2o }.each do |gas|
+      process "Convert #{gas} emission factors to co2e" do
+        gwp = GreenhouseGas[gas].global_warming_potential
+        
+        where("#{gas}_emission_factor_units = 'kilograms_per_kilowatt_hour'").update_all(%{
+          #{gas}_emission_factor = #{gas}_emission_factor * #{gwp},
+          #{gas}_emission_factor_units = 'kilograms_co2e_per_kilowatt_hour'
+        })
+      end
     end
     
-    process "Calculate combined emission factor" do
-      update_all(%{
+    # DEPRECATED - but don't remove until cut State.electricity_emission_factor
+    process "Calculate combined electricity emission factor" do
+      update_all %{
         electricity_emission_factor = co2_emission_factor + ch4_emission_factor + n2o_emission_factor,
         electricity_emission_factor_units = 'kilograms_co2e_per_kilowatt_hour'
-      })
+      }
     end
   end
 end

data/lib/earth/locality/electricity_mix.rb

@@ -0,0 +1,37 @@
+class ElectricityMix < ActiveRecord::Base
+  self.primary_key = :name
+  
+  def energy_content
+    1.kilowatt_hours.to(:megajoules)
+  end
+  
+  def energy_content_units
+    'megajoules_per_kilowatt_hour'
+  end
+  
+  falls_back_on :name => 'fallback',
+                :co2_emission_factor => 0.623537, # from ecometrica paper FIXME TODO calculate this
+                :co2_emission_factor_units => 'kilograms_per_kilowatt_hour',
+                :ch4_emission_factor => 0.000208, # from ecometrica paper FIXME TODO calculate this
+                :ch4_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour',
+                :n2o_emission_factor => 0.002344, # from ecometrica paper FIXME TODO calculate this
+                :n2o_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour',
+                :loss_factor => 0.096 # from ecometrica paper FIXME TODO calculate this
+  
+  col :name
+  col :egrid_subregion_abbreviation
+  col :state_postal_abbreviation
+  col :country_iso_3166_code
+  col :co2_emission_factor, :type => :float
+  col :co2_emission_factor_units
+  col :co2_biogenic_emission_factor, :type => :float
+  col :co2_biogenic_emission_factor_units
+  col :ch4_emission_factor, :type => :float
+  col :ch4_emission_factor_units
+  col :n2o_emission_factor, :type => :float
+  col :n2o_emission_factor_units
+  col :loss_factor, :type => :float
+  
+  warn_unless_size 187
+  warn_if_any_nulls
+end

data/lib/earth/locality/electricity_mix/data_miner.rb

@@ -0,0 +1,87 @@
+ElectricityMix.class_eval do
+  data_miner do
+    process "Start from scratch" do
+      delete_all
+    end
+    
+    process "Ensure GreenhouseGas is populated" do
+      GreenhouseGas.run_data_miner!
+    end
+    
+    import "national electricity mixes from Brander et al. (2011)",
+           :url => "file://#{Earth::DATA_DIR}/locality/national_electricity_efs.csv" do
+      key :name, :synthesize => lambda { |row| [row['country_iso_3166_code'], 'national electricity'].join(' ') }
+      store :country_iso_3166_code
+      store :co2_emission_factor, :units_field_name => 'co2_emission_factor_units'
+      store :ch4_emission_factor, :synthesize => proc { |row| row['ch4_emission_factor'].to_f * GreenhouseGas[:ch4].global_warming_potential }, :units => 'kilograms_co2e_per_kilowatt_hour'
+      store :n2o_emission_factor, :synthesize => proc { |row| row['n2o_emission_factor'].to_f * GreenhouseGas[:n2o].global_warming_potential }, :units => 'kilograms_co2e_per_kilowatt_hour'
+      store :loss_factor
+    end
+    
+    process "Ensure EgridSubregion is populated" do
+      EgridSubregion.run_data_miner!
+    end
+    
+    process "Derive eGRID subregion electricity mixes" do
+      EgridSubregion.safe_find_each do |subregion|
+        mix = find_or_create_by_name [subregion.abbreviation, 'egrid subregion electricity'].join(' ')
+        mix.update_attributes!(
+          :egrid_subregion_abbreviation => subregion.abbreviation,
+          :co2_emission_factor =>          subregion.co2_emission_factor,
+          :co2_emission_factor_units =>    subregion.co2_emission_factor_units,
+          :ch4_emission_factor =>          subregion.ch4_emission_factor,
+          :ch4_emission_factor_units =>    subregion.ch4_emission_factor_units,
+          :n2o_emission_factor =>          subregion.n2o_emission_factor,
+          :n2o_emission_factor_units =>    subregion.n2o_emission_factor_units,
+          :loss_factor =>                  subregion.egrid_region.loss_factor
+        )
+      end
+    end
+    
+    process "Derive US electricity mix from EgridSubregion" do
+      us = find_by_country_iso_3166_code 'US'
+      us.update_attributes!(
+        :co2_emission_factor =>       EgridSubregion.fallback.co2_emission_factor,
+        :co2_emission_factor_units => EgridSubregion.fallback.co2_emission_factor_units,
+        :ch4_emission_factor =>       EgridSubregion.fallback.ch4_emission_factor,
+        :ch4_emission_factor_units => EgridSubregion.fallback.ch4_emission_factor_units,
+        :n2o_emission_factor =>       EgridSubregion.fallback.n2o_emission_factor,
+        :n2o_emission_factor_units => EgridSubregion.fallback.n2o_emission_factor_units,
+        :loss_factor =>               EgridSubregion.fallback.egrid_region.loss_factor
+      )
+    end
+    
+    process "Ensure State is populated" do
+      State.run_data_miner!
+    end
+    
+    process "Derive state electricity mixes" do
+      State.safe_find_each do |state|
+        mix = find_or_create_by_name_and_state_postal_abbreviation(
+          [state.postal_abbreviation, 'state electricity'].join(' '), state.postal_abbreviation
+        )
+        
+        sub_pops = state.zip_codes.known_subregion.sum(:population, :group => :egrid_subregion)
+        
+        %w{ co2 ch4 n2o }.each do |gas|
+          ef = sub_pops.inject(0) do |memo, (subregion, population)|
+            memo += subregion.send("#{gas}_emission_factor") * population
+            memo
+          end / sub_pops.values.sum
+          
+          mix.update_attributes!(
+            "#{gas}_emission_factor" => ef,
+            "#{gas}_emission_factor_units" => sub_pops.keys.first.send("#{gas}_emission_factor_units")
+          )
+        end
+        
+        lf = sub_pops.inject(0) do |memo, (subregion, population)|
+          memo += subregion.egrid_region.loss_factor * population
+          memo
+        end / sub_pops.values.sum
+        
+        mix.update_attributes! :loss_factor => lf
+      end
+    end
+  end
+end