earth 0.4.0 → 0.4.1

data/lib/earth/bus/bus_fuel.rb

@@ -0,0 +1,18 @@
+class BusFuel < ActiveRecord::Base
+  set_primary_key :name
+  
+  belongs_to :fuel, :foreign_key => 'fuel_name'
+  has_many :fuel_year_controls, :class_name => 'BusFuelYearControl', :foreign_key => 'bus_fuel_name'
+  
+  data_miner do
+    tap "Brighter Planet's sanitized bus fuel data", Earth.taps_server
+    
+    process "Pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+  
+  def latest_fuel_year_controls
+    fuel_year_controls.where(:year => fuel_year_controls.maximum('year'))
+  end
+end

data/lib/earth/bus/bus_fuel/data_miner.rb

@@ -0,0 +1,204 @@
+BusFuel.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string 'name'
+      string 'fuel_name'
+      float  'energy_content'
+      string 'energy_content_units'
+      float  'co2_emission_factor'
+      string 'co2_emission_factor_units'
+      float  'co2_biogenic_emission_factor'
+      string 'co2_biogenic_emission_factor_units'
+      float  'ch4_emission_factor'
+      string 'ch4_emission_factor_units'
+      float  'n2o_emission_factor'
+      string 'n2o_emission_factor_units'
+    end
+    
+    process "Ensure necessary datasets are imported" do
+      Fuel.run_data_miner!
+      GreenhouseGas.run_data_miner!
+      BusFuelYearControl.run_data_miner!
+    end
+    
+    import "a list of bus fuels without emission factors",
+           :url => 'https://spreadsheets.google.com/pub?hl=en&hl=en&key=0AoQJbWqPrREqdGNscUdpbkdBUmJkTlFxdEtfSEdJS2c&output=csv' do
+      key 'name'
+      store 'fuel_name'
+    end
+    
+    import "a list of bus fuels and their emission factors",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHAtZ1YyWmxuS1JWUzZYSVhhdUViYmc&output=csv' do
+      key   'name'
+      store 'fuel_name'
+      store 'ch4_emission_factor', :units_field_name => 'ch4_emission_factor_units'
+      store 'n2o_emission_factor', :units_field_name => 'n2o_emission_factor_units'
+    end
+    
+    import "energy contents for compressed/liquified natural gas fuels",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDFfdHMycVB6dTkyUFFNM0x4b19TQVE&output=csv' do
+      key   'name'
+      store 'energy_content', :units_field_name => 'energy_content_units'
+    end
+    
+    process "Convert emission factors to metric units" do
+      conversion_factor = (1 / 1.609344) * (1.0 / 1_000.0 ) # Google: 1 mile / 1.609344 km * 1 kg / 1000 g
+      gwp_ch4 = GreenhouseGas[:ch4].global_warming_potential
+      connection.execute %{
+        UPDATE bus_fuels
+        SET ch4_emission_factor = ch4_emission_factor * #{conversion_factor} * #{gwp_ch4},
+            ch4_emission_factor_units = 'kilograms_co2e_per_kilometre'
+        WHERE ch4_emission_factor_units = 'grams_per_mile'
+      }
+      
+      gwp_n2o = GreenhouseGas[:n2o].global_warming_potential
+      connection.execute %{
+        UPDATE bus_fuels
+        SET n2o_emission_factor = n2o_emission_factor * #{conversion_factor} * #{gwp_n2o},
+            n2o_emission_factor_units = 'kilograms_co2e_per_kilometre'
+        WHERE n2o_emission_factor_units = 'grams_per_mile'
+      }
+    end
+    
+    process "Convert energy contents to metric units" do
+      conversion_factor = (1 / 947.81712) * (1 / 3.78541178) # Google: 1 MJ / 947.81712 btu * 1 gallon / 3.78541178 l
+      connection.execute %{
+        UPDATE bus_fuels
+        SET energy_content = energy_content * #{conversion_factor},
+            energy_content_units = 'megajoules_per_litre'
+        WHERE energy_content_units = 'btu_per_gallon'
+      }
+    end
+    
+    process 'Calculate CO2 and CO2 biogenic emission factors and units' do
+      BusFuel.all.each do |bus_fuel|
+        fuel = bus_fuel.fuel
+        if bus_fuel.energy_content.present?
+          co2_factor = (bus_fuel.energy_content * fuel.carbon_content * fuel.oxidation_factor * (1 - fuel.biogenic_fraction))
+          bus_fuel.co2_emission_factor = co2_factor.grams.to(:kilograms).carbon.to(:co2)
+          
+          co2_prefix = fuel.co2_emission_factor_units.split("_per_")[0]
+          co2_suffix = bus_fuel.energy_content_units.split("_per_")[1]
+          bus_fuel.co2_emission_factor_units = co2_prefix + "_per_" + co2_suffix
+          
+          co2_biogenic_factor = (bus_fuel.energy_content * fuel.carbon_content * fuel.oxidation_factor * fuel.biogenic_fraction)
+          bus_fuel.co2_biogenic_emission_factor = co2_biogenic_factor.grams.to(:kilograms).carbon.to(:co2)
+          
+          co2_biogenic_prefix = fuel.co2_biogenic_emission_factor_units.split("_per_")[0]
+          co2_biogenic_suffix = bus_fuel.energy_content_units.split("_per_")[1]
+          bus_fuel.co2_biogenic_emission_factor_units = co2_biogenic_prefix + "_per_" + co2_biogenic_suffix
+        else
+          bus_fuel.co2_emission_factor = fuel.co2_emission_factor
+          bus_fuel.co2_emission_factor_units = fuel.co2_emission_factor_units
+          
+          bus_fuel.co2_biogenic_emission_factor = fuel.co2_biogenic_emission_factor
+          bus_fuel.co2_biogenic_emission_factor_units = fuel.co2_biogenic_emission_factor_units
+        end
+        
+        bus_fuel.save!
+      end
+    end
+    
+    process 'Calculate CH4 and N2O emission factor and units for BusFuels with BusFuelYearControls' do
+      BusFuel.all.each do |bus_fuel|
+        if bus_fuel.latest_fuel_year_controls.present?
+          ch4_factors_by_year = bus_fuel.latest_fuel_year_controls.map do |fyc|
+             fyc.total_travel_percent * fyc.fuel_control.ch4_emission_factor
+          end
+          bus_fuel.ch4_emission_factor = ch4_factors_by_year.sum * GreenhouseGas[:ch4].global_warming_potential
+          
+          ch4_prefix = bus_fuel.latest_fuel_year_controls.first.fuel_control.ch4_emission_factor_units.split("_per_")[0]
+          ch4_suffix = bus_fuel.latest_fuel_year_controls.first.fuel_control.ch4_emission_factor_units.split("_per_")[1]
+          bus_fuel.ch4_emission_factor_units = ch4_prefix + "_co2e_per_" + ch4_suffix
+          
+          n2o_factors_by_year = bus_fuel.latest_fuel_year_controls.map do |fyc|
+             fyc.total_travel_percent * fyc.fuel_control.n2o_emission_factor
+          end
+          bus_fuel.n2o_emission_factor = n2o_factors_by_year.sum * GreenhouseGas[:n2o].global_warming_potential
+          
+          n2o_prefix = bus_fuel.latest_fuel_year_controls.first.fuel_control.n2o_emission_factor_units.split("_per_")[0]
+          n2o_suffix = bus_fuel.latest_fuel_year_controls.first.fuel_control.n2o_emission_factor_units.split("_per_")[1]
+          bus_fuel.n2o_emission_factor_units = n2o_prefix + "_co2e_per_" + n2o_suffix
+        end
+        
+        bus_fuel.save!
+      end
+    end
+    
+    # FIXME TODO verify fuel_name appears in fuels
+    verify "Fuel name should never be missing" do
+      BusFuel.all.each do |fuel|
+        value = fuel.send(:fuel_name)
+        unless value.present?
+          puts "Missing fuel name for BusFuel #{fuel.name}"
+          fail
+        end
+      end
+    end
+    
+    verify "Energy content should be greater than zero if present" do
+      BusFuel.all.each do |fuel|
+        value = fuel.send(:energy_content)
+        if value.present?
+          unless value > 0
+            puts "Invalid energy content for BusFuel #{fuel.name}: #{value} (should be > 0)"
+            fail
+          end
+        end
+      end
+    end
+    
+    %w{ co2_emission_factor co2_biogenic_emission_factor }.each do |attribute|
+      verify "#{attribute.humanize} should be 0 or more" do
+        BusFuel.all.each do |fuel|
+          value = fuel.send(attribute)
+          unless value >= 0
+            puts "Invalid #{attribute.humanize.downcase} for BusFuel #{fuel.name}: #{value} (should be 0 or more)"
+            fail
+          end
+        end
+      end
+    end
+    
+    %w{ ch4_emission_factor n2o_emission_factor }.each do |attribute|
+      verify "#{attribute.humanize} should be > 0" do
+        BusFuel.all.each do |fuel|
+          value = fuel.send(attribute)
+          unless value > 0
+            puts "Invalid #{attribute.humanize.downcase} for BusFuel #{fuel.name}: #{value} (should be > 0)"
+            fail
+          end
+        end
+      end
+    end
+    
+    verify "Energy content units should be megajoules per litre if present" do
+      BusFuel.all.each do |fuel|
+        units = fuel.send(:energy_content_units)
+        if units.present?
+          unless units == "megajoules_per_litre"
+            puts "Invalid energy content units for BusFuel #{fuel.name}: #{units} (should be megajoules_per_litre)"
+            fail
+          end
+        end
+      end
+    end
+    
+    [["co2_emission_factor_units", "kilograms_per_litre"],
+     ["co2_biogenic_emission_factor_units", "kilograms_per_litre"],
+     ["ch4_emission_factor_units", "kilograms_co2e_per_kilometre"],
+     ["n2o_emission_factor_units", "kilograms_co2e_per_kilometre"]].each do |pair|
+      attribute = pair[0]
+      proper_units = pair[1]
+      verify "#{attribute.humanize} should be #{proper_units.humanize.downcase}" do
+        BusFuel.all.each do |fuel|
+          units = fuel.send(:"#{attribute}")
+          unless units == proper_units
+            puts "Invalid #{attribute.humanize.downcase} for BusFuel #{fuel.name}: #{units} (should be #{proper_units})"
+            fail
+          end
+        end
+      end
+    end
+  end
+end

data/lib/earth/bus/bus_fuel_control.rb

@@ -0,0 +1,7 @@
+class BusFuelControl < ActiveRecord::Base
+  set_primary_key :name
+  
+  data_miner do
+    tap "Brighter Planet's sanitized bus fuel data", Earth.taps_server
+  end
+end

data/lib/earth/bus/bus_fuel_control/data_miner.rb

@@ -0,0 +1,41 @@
+BusFuelControl.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string 'name'
+      string 'bus_fuel_name'
+      string 'control'
+      float  'ch4_emission_factor'
+      string 'ch4_emission_factor_units'
+      float  'n2o_emission_factor'
+      string 'n2o_emission_factor_units'
+    end
+    
+    import "a list of bus fuel controls",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEw1QW80VVJhaXRkUURQSFhHczNyVWc&output=csv' do
+      key   'name'
+      store 'bus_fuel_name'
+      store 'control'
+      store 'ch4_emission_factor', :units_field_name => 'ch4_emission_factor_units'
+      store 'n2o_emission_factor', :units_field_name => 'n2o_emission_factor_units'
+    end
+    
+    process "Convert emission factors to metric units" do
+      conversion_factor = (1 / 1.609344) * (1.0 / 1_000.0 ) # Google: 1 mile / 1.609344 km * 1 kg / 1000 g
+      connection.execute %{
+        UPDATE bus_fuel_controls
+        SET ch4_emission_factor = ch4_emission_factor * #{conversion_factor},
+            ch4_emission_factor_units = 'kilograms_per_kilometre'
+        WHERE ch4_emission_factor_units = 'grams_per_mile'
+      }
+      
+      connection.execute %{
+        UPDATE bus_fuel_controls
+        SET n2o_emission_factor = n2o_emission_factor * #{conversion_factor},
+            n2o_emission_factor_units = 'kilograms_per_kilometre'
+        WHERE n2o_emission_factor_units = 'grams_per_mile'
+      }
+    end
+    
+    # FIXME TODO verify this
+  end
+end

data/lib/earth/bus/bus_fuel_year_control.rb

@@ -0,0 +1,13 @@
+class BusFuelYearControl < ActiveRecord::Base
+  set_primary_key :name
+  
+  belongs_to :fuel_control, :class_name => 'BusFuelControl', :foreign_key => 'bus_fuel_control_name'
+  
+  data_miner do
+    tap "Brighter Planet's sanitized bus fuel data", Earth.taps_server
+    
+    process "Pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+end

data/lib/earth/bus/bus_fuel_year_control/data_miner.rb

@@ -0,0 +1,35 @@
+BusFuelYearControl.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string  'name'
+      string  'bus_fuel_name'
+      integer 'year'
+      string  'control'
+      string  'bus_fuel_control_name'
+      float   'total_travel_percent'
+    end
+    
+    process "Ensure all BusFuelControls are imported" do
+      BusFuelControl.run_data_miner!
+    end
+    
+    import "a list of bus fuel year controls",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGhHQkZPZW4zbXYzZ3NkYThBWnQ2QXc&output=csv' do
+      key   'name'
+      store 'bus_fuel_name'
+      store 'year'
+      store 'control'
+      store 'total_travel_percent'
+    end
+    
+    process "Derive bus fuel control name for association with BusFuelControl" do
+      if ActiveRecord::Base.connection.adapter_name.downcase == 'sqlite'
+        update_all "bus_fuel_control_name = bus_fuel_name || ' ' || control"
+      else
+        update_all "bus_fuel_control_name = CONCAT(bus_fuel_name, ' ', control)"
+      end
+    end
+    
+    # FIXME TODO verify that for any year the percentages sum to 1
+  end
+end

data/lib/earth/bus/data_miner.rb

@@ -1 +1,4 @@
 require 'earth/bus/bus_class/data_miner'
+require 'earth/bus/bus_fuel/data_miner'
+require 'earth/bus/bus_fuel_control/data_miner'
+require 'earth/bus/bus_fuel_year_control/data_miner'

data/lib/earth/fuel/fuel.rb

@@ -43,6 +43,22 @@ class Fuel < ActiveRecord::Base
     end
   end
   
+  def oxidation_factor
+    if oxidation_factor = super
+      oxidation_factor
+    elsif fuel_years.present?
+      latest_year.oxidation_factor
+    end
+  end
+  
+  def biogenic_fraction
+    if biogenic_fraction = super
+      biogenic_fraction
+    elsif fuel_years.present?
+      latest_year.biogenic_fraction
+    end
+  end
+  
   def co2_emission_factor
     if ef = super
       ef

data/lib/earth/fuel/fuel/data_miner.rb

@@ -18,8 +18,13 @@ Fuel.class_eval do
     
     process "Derive fuel names from FuelYear" do
       FuelYear.run_data_miner!
+      if Fuel.connection.adapter_name.downcase == 'sqlite'
+        ignorance = ''
+      else
+        ignorance = 'IGNORE'
+      end
       connection.execute %{
-        INSERT IGNORE INTO fuels(name)
+        INSERT #{ignorance} INTO fuels(name)
         SELECT DISTINCT fuel_years.fuel_name FROM fuel_years
       }
     end

data/lib/earth/locality/egrid_region.rb

@@ -5,11 +5,12 @@ class EgridRegion < ActiveRecord::Base
   
   data_miner do
     tap "Brighter Planet's egrid region data", Earth.taps_server
-
+    
     process "pull dependencies" do
       run_data_miner_on_belongs_to_associations
     end
   end
   
-  falls_back_on :loss_factor => 0.061311 # Ian
+  # FIXME TODO someday should calculate this from eGRID data: (USTNGN05 + USTNFI05 - USTCON05) / USTNGN05
+  falls_back_on :loss_factor => 0.061879 # calculated using above formula 2/21/2011
 end

data/lib/earth/locality/egrid_region/data_miner.rb

@@ -5,31 +5,14 @@ EgridRegion.class_eval do
       float    'loss_factor'
     end
     
-    # NOTE: the following import uses an 18 Mb zip - don't know if two imports will cause it to be downloaded twice...
-    # 
-    # import "eGRID regions and loss factors derived from eGRID 2007 data",
-    #        :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2007_Version1-1.zip',
-    #        :filename => 'eGRID2007_Version1-1/eGRID2007V1_1_year0504_STIE_USGC.xls',
-    #        :sheet => 'STIE05',
-    #        :skip => 4,
-    #        :select => lambda { |row| row['eGRID2007 2005 file State sequence number'].to_i.between?(1, 51) } do
-    #   key   'name', :field_name => 'Grid region (E=Eastern grid, W=Western grid, AK=Alaska, HI=Hawaii, TX=Texas)'
-    #   store 'loss_factor', :field_name => '2005 grid gross loss factor'
-    # end
-    # 
-    # import "the US average grid loss factor derived eGRID 2007 data"
-    #        :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2007_Version1-1.zip',
-    #        :filename => 'eGRID2007_Version1-1/eGRID2007V1_1_year0504_STIE_USGC.xls',
-    #        :sheet => 'USGC',
-    #        :skip => 5 do
-    #   key # the name should be 'US'
-    #   store # store the result of (USTNGN05 + USTNFI05 - USTCON05) / USTNGN05
-    # end
-    
     import "eGRID regions and loss factors derived from eGRID 2007 data",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHoydC1MdDY0YVZkRE5zN0huOUZYbnc&hl=en&single=true&gid=0&output=csv' do
-      key 'name'
-      store 'loss_factor'
+           :url => 'http://www.epa.gov/cleanenergy/documents/egridzips/eGRID2007_Version1-1.zip',
+           :filename => 'eGRID2007_Version1-1/eGRID2007V1_1_year0504_STIE_USGC.xls',
+           :sheet => 'STIE05',
+           :skip => 4,
+           :select => lambda { |row| row['eGRID2007 2005 file State sequence number'].to_i.between?(1, 51) } do
+      key   'name', :field_name => 'Grid region (E=Eastern grid, W=Western grid, AK=Alaska, HI=Hawaii, TX=Texas)'
+      store 'loss_factor', :field_name => '2005 grid gross loss factor'
     end
     
     verify "Loss factor should be greater than zero and less than one" do
@@ -40,12 +23,6 @@ EgridRegion.class_eval do
       end
     end
     
-    # FIXME TODO get this verification to work
-    # verify "Fallback loss factor should be greater than zero and less than one" do
-    #   factor = EgridRegion.fallback.loss_factor
-    #   unless factor > 0 and factor < 1
-    #     raise "Invalid EgridRegion fallback loss factor: #{factor} (should be 0 < loss_factor < 1)"
-    #   end
-    # end
+    # FIXME TODO verify fallback loss factor
   end
 end

data/lib/earth/locality/egrid_subregion.rb

@@ -4,11 +4,27 @@ class EgridSubregion < ActiveRecord::Base
   has_many :zip_codes, :foreign_key => 'egrid_subregion_abbreviation'
   belongs_to :egrid_region, :foreign_key => 'egrid_region_name'
   
-  falls_back_on :electricity_emission_factor => 1.404.pounds.to(:kilograms) # kg CO2 / kWh https://brighterplanet.sifterapp.com/projects/30/issues/437?return_uri=%2Fprojects%2F30%2Fissues%3Fa%3D79%26s%3D1-2
+  class << self
+    def fallback_egrid_region
+      EgridRegion.fallback
+    end
+  end
+  
+  falls_back_on :egrid_region => lambda { EgridSubregion.fallback_egrid_region },
+                :electricity_co2_emission_factor => lambda { weighted_average(:electricity_co2_emission_factor, :weighted_by => :net_generation) },
+                :electricity_co2_emission_factor_units => 'kilograms_per_kilowatt_hour',
+                :electricity_co2_biogenic_emission_factor => lambda { weighted_average(:electricity_co2_biogenic_emission_factor, :weighted_by => :net_generation) },
+                :electricity_co2_biogenic_emission_factor_units => 'kilograms_per_kilowatt_hour',
+                :electricity_ch4_emission_factor => lambda { weighted_average(:electricity_ch4_emission_factor, :weighted_by => :net_generation) },
+                :electricity_ch4_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour',
+                :electricity_n2o_emission_factor => lambda { weighted_average(:electricity_n2o_emission_factor, :weighted_by => :net_generation) },
+                :electricity_n2o_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour',
+                :electricity_emission_factor => lambda { weighted_average(:electricity_emission_factor, :weighted_by => :net_generation) }, # DEPRECATED
+                :electricity_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour' # DEPRECATED
   
   data_miner do
     tap "Brighter Planet's egrid subregion data", Earth.taps_server
-
+    
     process "pull dependencies" do
       run_data_miner_on_belongs_to_associations
     end