earth 0.11.0 → 0.11.1

data/lib/earth/locality/data_miner.rb

@@ -2,16 +2,14 @@ require 'earth/locality/census_division/data_miner'
 require 'earth/locality/census_region/data_miner'
 require 'earth/locality/climate_division/data_miner'
 require 'earth/locality/country/data_miner'
-require 'earth/locality/country_rail_traction/data_miner'
-require 'earth/locality/country_rail_class/data_miner'
-require 'earth/locality/country_rail_traction_class/data_miner'
+require 'earth/locality/egrid_country/data_miner'
 require 'earth/locality/egrid_region/data_miner'
 require 'earth/locality/egrid_subregion/data_miner'
 require 'earth/locality/petroleum_administration_for_defense_district/data_miner'
 require 'earth/locality/state/data_miner'
-require 'earth/locality/urbanity/data_miner'
 require 'earth/locality/zip_code/data_miner'
 
-require 'earth/automobile/data_miner'
-require 'earth/residence/data_miner'
-require 'earth/rail/data_miner'
+require 'earth/automobile/data_miner'  # for automobile data on Country
+require 'earth/fuel/data_miner'        # for GWPs for EgridSubregion
+require 'earth/hospitality/data_miner' # for lodging data on Country
+require 'earth/rail/data_miner'        # for rail data on Country

data/lib/earth/locality/egrid_country.rb

@@ -0,0 +1,17 @@
+class EgridCountry < ActiveRecord::Base
+  set_primary_key :name
+  
+  class << self
+    def us
+      find_by_name 'US'
+    end
+  end
+  
+  col :name
+  col :generation, :type => :float
+  col :generation_units
+  col :imports, :type => :float
+  col :imports_units
+  col :consumption, :type => :float
+  col :consumption_units
+end

data/lib/earth/locality/egrid_country/data_miner.rb

@@ -0,0 +1,13 @@
+EgridCountry.class_eval do
+  data_miner do
+    import "the US average grid loss factor derived eGRID 2007 data",
+           :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2010V1_1_STIE_USGC.xls',
+           :sheet => 'USGC',
+           :skip => 5 do
+      key 'name', :static => 'US'
+      store 'generation',  :field_name => 'USTNGN07', :units => :gigawatt_hours
+      store 'imports',     :field_name => 'USTNFI07', :units => :gigawatt_hours
+      store 'consumption', :field_name => 'USTCON07', :units => :gigawatt_hours
+    end
+  end
+end

data/lib/earth/locality/egrid_region.rb

@@ -3,21 +3,9 @@ class EgridRegion < ActiveRecord::Base
   
   has_many :egrid_subregions, :foreign_key => 'egrid_region_name'
   
-  # FIXME TODO someday should calculate this from eGRID data: (USTNGN05 + USTNFI05 - USTCON05) / USTNGN05
   falls_back_on :name => 'fallback',
-                :loss_factor => 0.061879 # calculated using above formula 2/21/2011
+                :loss_factor => lambda { (EgridCountry.us.generation + EgridCountry.us.imports - EgridCountry.us.consumption) / EgridCountry.us.generation }
   
   col :name
   col :loss_factor, :type => :float
-  
-  # verify "Loss factor should be greater than zero and less than one" do
-  #   EgridRegion.all.each do |region|
-  #     unless region.loss_factor > 0 and region.loss_factor < 1
-  #       raise "Invalid loss factor for EgridRegion #{region.name}: #{region.loss_factor} (should be > 0 and < 1)"
-  #     end
-  #   end
-  # end
-  
-  # FIXME TODO verify fallback loss factor
-  
-end
+end

data/lib/earth/locality/egrid_region/data_miner.rb

@@ -1,23 +1,24 @@
 EgridRegion.class_eval do
   data_miner do
-    import "eGRID regions and loss factors derived from eGRID 2007 data",
-           :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2007_Version1-1.zip',
-           :filename => 'eGRID2007_Version1-1/eGRID2007V1_1_year0504_STIE_USGC.xls',
-           :sheet => 'STIE05',
+    import "eGRID 2010 regions and loss factors",
+           :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2010V1_1_STIE_USGC.xls',
+           :sheet => 'STIE07',
            :skip => 4,
-           :select => lambda { |row| row['eGRID2007 2005 file State sequence number'].to_i.between?(1, 51) } do
+           :select => lambda { |row| row['eGRID2010 year 2007 file state sequence number'].to_i.between?(1, 51) } do
       key   'name', :field_name => 'Grid region (E=Eastern grid, W=Western grid, AK=Alaska, HI=Hawaii, TX=Texas)'
-      store 'loss_factor', :field_name => '2005 grid gross loss factor'
+      store 'loss_factor', :field_name => 'Year 2007 grid gross loss factor'
     end
     
-    # resurrected from a7bb363f10d951957dd051ff3cfb81c280f61151
-    import "the US average grid loss factor derived eGRID 2007 data",
-           :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2007_Version1-1.zip',
-           :filename => 'eGRID2007_Version1-1/eGRID2007V1_1_year0504_STIE_USGC.xls',
-           :sheet => 'USGC',
-           :skip => 5 do
-      key 'name', :static => 'US'
-      store 'loss_factor', :synthesize => lambda { |row| (row['USTNGN05'].to_f + row['USTNFI05'].to_f - row['USTCON05'].to_f) / row['USTNGN05'].to_f }
+    # Need this for fallback loss_factor calculation
+    process "Ensure EgridCountry is populated" do
+      EgridCountry.run_data_miner!
+    end
+    
+    # DEPRECATED but don't remove until confirmed that all emitters use EgridRegion.fallback rather than EgridRegion.find_by_abbreviation 'US'
+    process "Calculate national averages" do
+      us_average = find_or_create_by_name 'US'
+      us_average.loss_factor = (EgridCountry.us.generation + EgridCountry.us.imports - EgridCountry.us.consumption) / EgridCountry.us.generation
+      us_average.save!
     end
   end
 end

data/lib/earth/locality/egrid_subregion.rb

@@ -4,14 +4,8 @@ class EgridSubregion < ActiveRecord::Base
   has_many :zip_codes, :foreign_key => 'egrid_subregion_abbreviation'
   belongs_to :egrid_region, :foreign_key => 'egrid_region_name'
   
-  class << self
-    def fallback_egrid_region
-      EgridRegion.fallback
-    end
-  end
-  
   falls_back_on :name => 'fallback',
-                :egrid_region => lambda { EgridSubregion.fallback_egrid_region },
+                :egrid_region => lambda { EgridRegion.fallback },
                 :electricity_co2_emission_factor => lambda { weighted_average(:electricity_co2_emission_factor, :weighted_by => :net_generation) },
                 :electricity_co2_emission_factor_units => 'kilograms_per_kilowatt_hour',
                 :electricity_co2_biogenic_emission_factor => lambda { weighted_average(:electricity_co2_biogenic_emission_factor, :weighted_by => :net_generation) },
@@ -20,8 +14,8 @@ class EgridSubregion < ActiveRecord::Base
                 :electricity_ch4_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour',
                 :electricity_n2o_emission_factor => lambda { weighted_average(:electricity_n2o_emission_factor, :weighted_by => :net_generation) },
                 :electricity_n2o_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour',
-                :electricity_emission_factor => lambda { weighted_average(:electricity_emission_factor, :weighted_by => :net_generation) }, # DEPRECATED
-                :electricity_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour' # DEPRECATED
+                :electricity_emission_factor => lambda { weighted_average(:electricity_emission_factor, :weighted_by => :net_generation) },
+                :electricity_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour'
   
   col :abbreviation
   col :name
@@ -39,59 +33,4 @@ class EgridSubregion < ActiveRecord::Base
   col :electricity_n2o_emission_factor_units
   col :electricity_emission_factor, :type => :float
   col :electricity_emission_factor_units
-  
-  # FIXME TODO verify egrid_region_name is found in EgridRegions
-  # %w{ egrid_region_name }.each do |attribute|
-  #   verify "#{attribute.humanize} should never be missing" do
-  #     EgridSubregion.all.each do |subregion|
-  #       unless subregion.send("#{attribute}").present?
-  #         raise "Missing #{attribute.humanize.downcase} for EgridSubregion #{subregion.name}"
-  #       end
-  #     end
-  #   end
-  # end
-  # 
-  # ["net_generation",
-  #  "electricity_co2_emission_factor",
-  #  "electricity_ch4_emission_factor",
-  #  "electricity_n2o_emission_factor",
-  #  "electricity_emission_factor" ].each do |attribute|
-  #   verify "#{attribute.humanize} should be > 0" do
-  #     EgridSubregion.all.each do |subregion|
-  #       value = subregion.send(:"#{attribute}")
-  #       unless value > 0
-  #         raise "Invalid #{attribute.humanize.downcase} for EgridSubregion #{subregion.name}: #{value} (should be > 0)"
-  #       end
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Electricity co2 biogenic emission factor should be 0" do
-  #   EgridSubregion.all.each do |subregion|
-  #     value = subregion.electricity_co2_biogenic_emission_factor
-  #     unless value == 0
-  #       raise "Invalid electricity co2 biogenic emission factor for EgridSubregion #{subregion.name}: #{value} (should be 0)"
-  #     end
-  #   end
-  # end
-  # 
-  # [["net_generation_units","megawatt_hours"],
-  #  ["electricity_co2_emission_factor_units","kilograms_per_kilowatt_hour"],
-  #  ["electricity_co2_biogenic_emission_factor_units","kilograms_per_kilowatt_hour"],
-  #  ["electricity_ch4_emission_factor_units","kilograms_co2e_per_kilowatt_hour"],
-  #  ["electricity_n2o_emission_factor_units","kilograms_co2e_per_kilowatt_hour"],
-  #  ["electricity_emission_factor_units","kilograms_co2e_per_kilowatt_hour"]].each do |pair|
-  #   attribute = pair[0]
-  #   proper_units = pair[1]
-  #   verify "#{attribute.humanize} should be #{proper_units.humanize.downcase}" do
-  #     EgridSubregion.all.each do |subregion|
-  #       units = subregion.send(:"#{attribute}")
-  #       unless units == proper_units
-  #         raise "Invalid #{attribute.humanize.downcase} for EgridSubregion #{subregion.name}: #{units} (should be #{proper_units})"
-  #       end
-  #     end
-  #   end
-  # end
-  
-  # FIXME TODO verify fallbacks
 end

data/lib/earth/locality/egrid_subregion/data_miner.rb

@@ -1,33 +1,47 @@
 EgridSubregion.class_eval do
   data_miner do
-    # FIXME TODO for some reason this doesn't work...
-    # import "eGRID regions and electricity emission factors derived from eGRID 2007 data",
-    #        :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2007_Version1-1.zip',
-    #        :filename => 'eGRID2007_Version1-1/eGRID2007V1_1_year05_aggregation.xls',
-    #        :sheet => 'SRL05',
-    #        :skip => 3,
-    #        :select => lambda { |row| row['eGRID2007 2005 file eGRID subregion location (operator)-based sequence number'].to_i.between?(1, 26) } do
-    #   key   'abbreviation', :field_name => 'eGRID subregion acronym'
-    #   store 'name', :field_name => 'eGRID subregion name associated with eGRID subregion acronym'
-    #   store 'nerc_abbreviation', :field_name => 'NERC region acronym associated with the eGRID subregion acronym'
-    #   store 'net_generation', :field_name => 'eGRID subregion annual net generation (MWh)', :units => 'megawatt_hours'
-    #   store 'electricity_co2_emission_factor', :field_name => 'eGRID subregion annual CO2 output emission rate (lb/MWh)', :units => 'pounds_per_megawatt_hour'
-    #   store 'electricity_co2_biogenic_emission_factor', :static => '0.0', :units => 'kilograms_per_kilowatt_hour'
-    #   store 'electricity_ch4_emission_factor', :field_name => 'eGRID subregion annual CH4 output emission rate (lb/GWh)', :units => 'pounds_per_gigawatt_hour'
-    #   store 'electricity_n2o_emission_factor', :field_name => 'eGRID subregion annual N2O output emission rate (lb/GWh)', :units => 'pounds_per_gigawatt_hour'
-    # end
+    import "eGRID 2010 subregions and electricity emission factors",
+           :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2010_Version1-1_xls_only.zip',
+           :filename => 'eGRID2010V1_1_year07_AGGREGATION.xls',
+           :sheet => 'SRL07',
+           :skip => 4,
+           :select => lambda { |row| row['SEQSRL07'].to_i.between?(1, 26) } do
+      key   'abbreviation', :field_name => 'SUBRGN'
+      store 'name', :field_name => 'SRNAME'
+      store 'nerc_abbreviation',               :field_name => 'NERC'
+      store 'net_generation',                  :field_name => 'SRNGENAN', :units => :megawatt_hours
+      store 'electricity_co2_emission_factor', :field_name => 'SRCO2RTA', :units => :pounds_per_megawatt_hour
+      store 'electricity_co2_biogenic_emission_factor', :static => '0.0', :units => :kilograms_per_kilowatt_hour
+      store 'electricity_ch4_emission_factor', :field_name => 'SRCH4RTA', :units => :pounds_per_gigawatt_hour
+      store 'electricity_n2o_emission_factor', :field_name => 'SRN2ORTA', :units => :pounds_per_gigawatt_hour
+    end
     
-    import "eGRID subregion data",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGRORTJNSWRMQ1puRVprYlAtZHhDaFE&hl=en&gid=0&output=csv' do
+    # FIXME TODO once 'US' subregion is no longer needed remove it from this source file
+    import "eGRID subregion to region associations",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGRORTJNSWRMQ1puRVprYlAtZHhDaFE&output=csv' do
       key 'abbreviation'
-      store 'name'
-      store 'nerc_abbreviation'
       store 'egrid_region_name'
-      store 'net_generation', :units => 'megawatt_hours'
-      store 'electricity_co2_emission_factor', :field_name => 'electricty_ef_co2', :units => 'pounds_per_megawatt_hour'
-      store 'electricity_co2_biogenic_emission_factor', :static => '0.0', :units => 'kilograms_per_kilowatt_hour'
-      store 'electricity_ch4_emission_factor', :field_name => 'electricity_ef_ch4',  :units => 'pounds_per_gigawatt_hour'
-      store 'electricity_n2o_emission_factor', :field_name => 'electricity_ef_n2o',  :units => 'pounds_per_gigawatt_hour'
+    end
+    
+    # DEPRECATED but don't remove until confirmed that all emitters use EgridSubregion.fallback rather than EgridSubregion.find_by_abbreviation 'US'
+    process "Calculate national averages" do
+      us_average = find_by_abbreviation 'US'
+      subregions = where("abbreviation != 'US'")
+      
+      us_average.name = 'United States'
+      us_average.nerc_abbreviation = 'US'
+      us_average.egrid_region_name = 'US'
+      us_average.net_generation                           = subregions.sum(:net_generation)
+      us_average.electricity_co2_emission_factor          = subregions.weighted_average(:electricity_co2_emission_factor, :weighted_by => :net_generation)
+      us_average.electricity_co2_biogenic_emission_factor = subregions.weighted_average(:electricity_co2_biogenic_emission_factor, :weighted_by => :net_generation)
+      us_average.electricity_ch4_emission_factor          = subregions.weighted_average(:electricity_ch4_emission_factor, :weighted_by => :net_generation)
+      us_average.electricity_n2o_emission_factor          = subregions.weighted_average(:electricity_n2o_emission_factor, :weighted_by => :net_generation)
+      us_average.net_generation_units                           = 'megawatt_hours'
+      us_average.electricity_co2_emission_factor_units          = 'pounds_per_megawatt_hour'
+      us_average.electricity_co2_biogenic_emission_factor_units = 'kilograms_per_kilowatt_hour'
+      us_average.electricity_ch4_emission_factor_units          = 'pounds_per_gigawatt_hour'
+      us_average.electricity_n2o_emission_factor_units          = 'pounds_per_gigawatt_hour'
+      us_average.save!
     end
     
     process "Convert co2 emission factor to metric units" do

data/lib/earth/locality/zip_code/data_miner.rb

@@ -1,3 +1,4 @@
+# FIXME TODO try to clean up this data
 ZipCode.class_eval do
   # sabshere 9/20/10 this isn't called anywhere
   # def set_latitude_and_longitude

data/lib/earth/rail.rb

@@ -1,3 +1,6 @@
+require 'earth/rail/country_rail_traction'
+require 'earth/rail/country_rail_class'
+require 'earth/rail/country_rail_traction_class'
 require 'earth/rail/rail_company'
 require 'earth/rail/rail_company_traction'
 require 'earth/rail/rail_company_traction_class'
@@ -7,3 +10,6 @@ require 'earth/rail/rail_traction'
 require 'earth/rail/national_transit_database_company'
 require 'earth/rail/national_transit_database_mode'
 require 'earth/rail/national_transit_database_record'
+
+require 'earth/fuel'
+require 'earth/locality' # eGRID needed for US electricity emission factor in RailCompany import

data/lib/earth/rail/data_miner.rb

@@ -1,3 +1,6 @@
+require 'earth/rail/country_rail_traction/data_miner'
+require 'earth/rail/country_rail_class/data_miner'
+require 'earth/rail/country_rail_traction_class/data_miner'
 require 'earth/rail/rail_class/data_miner'
 require 'earth/rail/rail_company/data_miner'
 require 'earth/rail/rail_company_traction/data_miner'
@@ -8,3 +11,6 @@ require 'earth/rail/rail_traction/data_miner'
 require 'earth/rail/national_transit_database_company/data_miner'
 require 'earth/rail/national_transit_database_mode/data_miner'
 require 'earth/rail/national_transit_database_record/data_miner'
+
+require 'earth/fuel/data_miner'
+require 'earth/locality/data_miner' # eGRID needed for US electricity emission factor in RailCompany import

data/lib/earth/rail/national_transit_database_record/data_miner.rb

@@ -12,6 +12,7 @@ NationalTransitDatabaseRecord.class_eval do
       store 'passengers'
     end
     
+    # FIXME TODO electricity synthesization should return NULL rather than 0
     import "US National Transit Database fuel consumption data",
            :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDItVVR5NjA2Y3FCVjgza25Ccy0zS2c&output=csv' do
       key 'name', :synthesize => lambda { |row| [row['trs_id'], row['mode_code'], row['service_code']].join(' ') }

data/lib/earth/rail/rail_company/data_miner.rb

@@ -29,7 +29,7 @@ RailCompany.class_eval do
     
     process "Derive US transit rail company data from the National Transit Database" do
       NationalTransitDatabaseCompany.rail_companies.each do |ntd_company|
-        company = RailCompany.find_or_create_by_name(ntd_company.name)
+        company = find_or_create_by_name(ntd_company.name)
         company.country_iso_3166_code    = 'US'
         company.duns_number              = ntd_company.duns_number
         company.passengers               = ntd_company.rail_passengers
@@ -55,7 +55,7 @@ RailCompany.class_eval do
     end
     
     process "Calculate average trip distance" do
-      RailCompany.find_each do |company|
+      find_each do |company|
         if company.passenger_distance.present? and company.passengers.present? and company.passengers > 0
           company.trip_distance = company.passenger_distance / company.passengers
           company.trip_distance_units = company.passenger_distance_units
@@ -65,7 +65,7 @@ RailCompany.class_eval do
     end
     
     process "Calculate average trip speed" do
-      RailCompany.find_each do |company|
+      find_each do |company|
         if company.train_distance.present? and company.train_time.present? and company.train_time > 0
           company.speed = company.train_distance / company.train_time
           company.speed_units = "#{company.train_distance_units}_per_#{company.train_time_units.singularize}"
@@ -74,12 +74,13 @@ RailCompany.class_eval do
       end
     end
     
-    process "Ensure RailFuel is populated" do
+    process "Ensure RailFuel and EgridSubregion are populated" do
       RailFuel.run_data_miner!
+      EgridSubregion.run_data_miner!
     end
     
     process "Calculate co2 emission factor for US rail companies" do
-      RailCompany.where(:country_iso_3166_code => 'US').find_each do |company|
+      where(:country_iso_3166_code => 'US').find_each do |company|
         company.co2_emission_factor = 0
         
         if company.diesel_intensity.present?

data/lib/earth/residence.rb

@@ -6,5 +6,4 @@ require 'earth/residence/residence_class'
 require 'earth/residence/residence_fuel_price'
 require 'earth/residence/residence_fuel_type'
 require 'earth/residence/residential_energy_consumption_survey_response'
-
-require 'earth/locality'
+require 'earth/residence/urbanity'

data/lib/earth/residence/data_miner.rb

@@ -6,5 +6,4 @@ require 'earth/residence/residence_class/data_miner'
 require 'earth/residence/residence_fuel_price/data_miner'
 require 'earth/residence/residence_fuel_type/data_miner'
 require 'earth/residence/residential_energy_consumption_survey_response/data_miner'
-
-require 'earth/locality/data_miner'
+require 'earth/residence/urbanity/data_miner'

data/lib/earth/residence/residence_fuel_type.rb

@@ -12,7 +12,7 @@ class ResidenceFuelType < ActiveRecord::Base
   def price_per_unit(relaxations = [])
     conditions = { :residence_fuel_type_name => self }
     relaxations.push Hash.new
-    relaxations.grab do |relaxation|
+    relaxations.each do |relaxation|
       relaxation_conditions = Hash.new
       if timeframe = relaxation[:timeframe]
         relaxation_conditions[:year] = timeframe.from.year
@@ -22,8 +22,11 @@ class ResidenceFuelType < ActiveRecord::Base
         relaxation_conditions[:locatable_type] = location.class.to_s
         relaxation_conditions[:locatable_id] = location.id
       end
-      ResidenceFuelPrice.average :price, :conditions => conditions.merge(relaxation_conditions)
+      if non_nil_result = ResidenceFuelPrice.average(:price, :conditions => conditions.merge(relaxation_conditions))
+        return non_nil_result
+      end
     end
+    nil
   end
 
   class << self

data/lib/earth/version.rb

@@ -1,3 +1,3 @@
 module Earth
-  VERSION = "0.11.0"
+  VERSION = "0.11.1"
 end

data/spec/earth_spec.rb

@@ -23,7 +23,7 @@ describe Earth do
   describe '.resource_map' do
     it 'should get a list of resource_map' do
       resource_map = Earth.resource_map
-      resource_map.keys.count.should == 88
+      resource_map.keys.count.should == 91
       resource_map['FuelType'].should == 'fuel'
     end
     it 'should exclude data_miner files' do
@@ -33,7 +33,7 @@ describe Earth do
 
   describe '.search' do
     it 'should get a list of all resource names' do
-      Earth.search.length.should == 88
+      Earth.search.length.should == 91
       Earth.search.should include('Aircraft')
       Earth.search.should include('Industry')
     end
@@ -44,12 +44,12 @@ describe Earth do
       Earth.search('computation').length.should == 3
       Earth.search('diet').length.should == 2
       Earth.search('fuel').length.should == 5
-      Earth.search('hospitality').length.should == 1
+      Earth.search('hospitality').length.should == 3
       Earth.search('industry').length.should == 10
-      Earth.search('locality').length.should == 13
+      Earth.search('locality').length.should == 10
       Earth.search('pet').length.should == 4
-      Earth.search('rail').length.should == 9
-      Earth.search('residence').length.should == 8
+      Earth.search('rail').length.should == 12
+      Earth.search('residence').length.should == 9
       Earth.search('shipping').length.should == 3
       Earth.search('fuel').should include('FuelType')
     end