earth 0.11.12 → 0.11.13

data/earth.gemspec

@@ -5,7 +5,7 @@ require "earth/version"
 Gem::Specification.new do |s|
   s.name        = "earth"
   s.version     = Earth::VERSION
-  s.date = "2012-02-20"
+  s.date = "2012-03-05"
   s.platform    = Gem::Platform::RUBY
   s.authors = ["Seamus Abshere", "Derek Kastner", "Andy Rossmeissl"]
   s.email = %q{andy@rossmeissl.net}
@@ -29,7 +29,7 @@ Gem::Specification.new do |s|
   
   s.add_runtime_dependency 'activerecord'
   s.add_runtime_dependency 'activesupport'
-  s.add_runtime_dependency 'cohort_scope'
+  s.add_runtime_dependency 'cohort_analysis'
   s.add_runtime_dependency 'conversions'
   s.add_runtime_dependency 'data_miner', '>=1.3'
   s.add_runtime_dependency 'falls_back_on'

data/lib/earth/air/flight_segment.rb

@@ -3,9 +3,6 @@ require 'earth/locality'
 class FlightSegment < ActiveRecord::Base
   self.primary_key = "row_hash"
   
-  extend CohortScope
-  self.minimum_cohort_size = 1
-  
   # Cutting this for now because if iata code is missing we have to look up airports using both city and country; don't know how to do this with ActiveRecord
   # - Ian 6/12/2011
   # # If airport iata code is missing, associate with all airports in a city

data/lib/earth/hospitality/commercial_building_energy_consumption_survey_response/data_miner.rb

@@ -1,7 +1,7 @@
 require 'earth/locality/data_miner'
 CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
   data_miner do
-    import 'building characteristics from the 2003 EIA Commercial Building Energy Consumption Survey',
+    import 'building characteristics from the 2003 EIA CBECS - part 1',
            :url => 'http://www.eia.gov/emeu/cbecs/cbecs2003/public_use_2003/data/FILE01.csv',
            :skip => 1,
            :headers => ["PUBID8", "REGION8", "CENDIV8", "SQFT8", "SQFTC8", "YRCONC8", "PBA8", "ELUSED8", "NGUSED8", "FKUSED8", "PRUSED8", "STUSED8", "HWUSED8", "CLIMATE8", "FREESTN8", "WLCNS8", "RFCNS8", "GLSSPC8", "EQGLSS8", "SUNGLS8", "BLDSHP8", "NFLOOR8", "ELEVTR8", "NELVTR8", "ESCLTR8", "NESLTR8", "YRCON8", "MONCON8", "RENOV8", "RENADD8", "RENRDC8", "RENCOS8", "RENEXT8", "RENINT8", "RENHVC8", "RENLGT8", "RENWIN8", "RENPLB8", "RENINS8", "RENOTH8", "GOVOWN8", "GOVTYP8", "OWNER8", "OWNOCC8", "NOCC8", "NOCCAT8", "MONUSE8", "PORVAC8", "OPEN248", "OPNMF8", "OPNWE8", "WKHRS8", "WKHRSC8", "NWKER8", "NWKERC8", "HT18", "HT28", "COOL8", "WATR8", "COOK8", "MANU8", "GENR8", "ADJWT8", "STRATUM8", "PAIR8"] do
@@ -20,7 +20,7 @@ CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
       store 'weighting',              :field_name => 'ADJWT8'
     end
     
-    import 'building characteristics from the 2003 EIA Commercial Building Energy Consumption Survey',
+    import 'building characteristics from the 2003 EIA CBECS - part 2',
            :url => 'http://www.eia.gov/emeu/cbecs/cbecs2003/public_use_2003/data/FILE02.csv',
            :skip => 1,
            :headers => ["PUBID8","REGION8","CENDIV8","SQFT8","SQFTC8","YRCONC8","PBA8","ELUSED8","NGUSED8","FKUSED8","PRUSED8","STUSED8","HWUSED8","ONEACT8","ACT18","ACT28","ACT38","ACT1PCT8","ACT2PCT8","ACT3PCT8","PBAPLUS8","VACANT8","RWSEAT8","PBSEAT8","EDSEAT8","FDSEAT8","HCBED8","NRSBED8","LODGRM8","FACIL8","FEDFAC8","FACACT8","MANIND8","PLANT8","FACDST8","FACDHW8","FACDCW8","FACELC8","BLDPLT8","ADJWT8","STRATUM8","PAIR8"] do
@@ -35,7 +35,7 @@ CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
       store 'lodging_rooms',         :field_name => 'LODGRM8'
     end
     
-    import 'building characteristics from the 2003 EIA Commercial Building Energy Consumption Survey',
+    import 'building characteristics from the 2003 EIA CBECS - part 3',
            :url => 'http://www.eia.gov/emeu/cbecs/cbecs2003/public_use_2003/data/FILE03.csv',
            :skip => 1,
            :headers => ["PUBID8","REGION8","CENDIV8","SQFT8","SQFTC8","YRCONC8","PBA8","ELUSED8","NGUSED8","FKUSED8","PRUSED8","STUSED8","HWUSED8","HEATP8","HTLS508","FURNAC8","BOILER8","PKGHT8","SLFCON8","HTPMPH8","STHW8","OTHTEQ8","FURNP8","BOILP8","PKGHP8","SLFCNP8","HTPHP8","STHWP8","OTHTP8","MAINHT8","PKGHPS8","SPLHPS8","RMHPS8","AIRHPT8","GRDHPT8","WTRHPT8","NWMNHT8","RDHTNF8","HWRDHT8","COOLP8","PKGCL8","RCAC8","ACWNWL8","HTPMPC8","CHWT8","CHILLR8","EVAPCL8","OTCLEQ8","PKGCP8","RCACP8","ACWNWP8","HTPCP8","CHWTP8","CHILP8","EVAPP8","OTCLP8","MAINCL8","PKGCPS8","SPLCPS8","RMCPS8","AIRCPT8","GRDCPT8","WTRCPT8","NWMNCL8","RDCLNF8","HWRDCL8","VAV8","ECN8","MAINT8","EMCS8","ADJWT8","STRATUM8","PAIR8"] do
@@ -43,7 +43,7 @@ CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
       store 'percent_cooled', :synthesize => Proc.new { |row| row['COOLP8'].to_f / 100.0 }
     end
     
-    import 'electricity use from the 2003 EIA Commercial Building Energy Consumption Survey',
+    import 'electricity use from the 2003 EIA CBECS',
            :url => 'http://www.eia.gov/emeu/cbecs/cbecs2003/public_use_2003/data/FILE15.csv',
            :skip => 1,
            :headers => ["PUBID8", "REGION8", "CENDIV8", "SQFT8", "SQFTC8", "YRCONC8", "PBA8", "ELUSED8", "NGUSED8", "FKUSED8", "PRUSED8", "STUSED8", "HWUSED8", "ADJWT8", "STRATUM8", "PAIR8", "HDD658", "CDD658", "MFUSED8", "MFBTU8", "MFEXP8", "ELCNS8", "ELBTU8", "ELEXP8", "ZELCNS8", "ZELEXP8"] do
@@ -54,7 +54,7 @@ CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
       store 'electricity_energy',  :field_name => 'ELBTU8', :from_units => :kbtus, :to_units => :megajoules
     end
     
-    import 'fuel use characteristics from the 2003 EIA Commercial Building Energy Consumption Survey',
+    import 'fuel use characteristics from the 2003 EIA CBECS',
            :url => 'http://www.eia.gov/emeu/cbecs/cbecs2003/public_use_2003/data/FILE16.csv',
            :skip => 1,
            :headers => ["PUBID8", "REGION8", "CENDIV8", "SQFT8", "SQFTC8", "YRCONC8", "PBA8", "ELUSED8", "NGUSED8", "FKUSED8", "PRUSED8", "STUSED8", "HWUSED8", "ADJWT8", "STRATUM8", "PAIR8", "NGCNS8", "NGBTU8", "NGEXP8", "ZNGCNS8", "ZNGEXP8", "FKCNS8", "FKBTU8", "FKEXP8", "ZFKCNS8", "ZFKEXP8", "DHUSED8", "DHHT18", "DHHT28", "DHCOOL8", "DHWATR8", "DHCOOK8", "DHMANU8", "DHOTH8", "DHCNS8", "DHBTU8", "DHEXP8", "ZDHCNS8", "ZDHEXP8"] do
@@ -78,5 +78,18 @@ CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
         }
       end
     end
+    
+    process "Derive fuel intensities per unit area" do
+      [:natural_gas, :fuel_oil, :electricity].each do |fuel|
+        update_all %{
+          #{fuel}_energy_intensity = #{fuel}_energy / area,
+          #{fuel}_energy_intensity_units = 'megajoules_per_square_metre'
+        }
+        update_all %{
+          district_heat_energy_intensity = district_heat_use / area,
+          district_heat_energy_intensity_units = 'megajoules_per_square_metre'
+        }
+      end
+    end
   end
 end

data/lib/earth/hospitality/commercial_building_energy_consumption_survey_response.rb

@@ -3,9 +3,6 @@ class CommercialBuildingEnergyConsumptionSurveyResponse < ActiveRecord::Base
   self.primary_key = "id"
   self.table_name = :cbecs_responses
   
-  extend CohortScope
-  self.minimum_cohort_size = 8 # CBECS doesn't report averages based on fewer than 20 samples
-  
   def self.lodging_records
     where(:detailed_activity => ['Hotel', 'Motel or inn'], :first_activity => nil)
   end
@@ -59,4 +56,12 @@ class CommercialBuildingEnergyConsumptionSurveyResponse < ActiveRecord::Base
   col :fuel_oil_per_room_night_units                 # for lodging fuzzy weighting
   col :district_heat_per_room_night, :type => :float # for lodging fuzzy weighting
   col :district_heat_per_room_night_units            # for lodging fuzzy weighting
+  col :electricity_energy_intensity, :type => :float
+  col :electricity_energy_intensity_units
+  col :natural_gas_energy_intensity, :type => :float
+  col :natural_gas_energy_intensity_units
+  col :fuel_oil_energy_intensity, :type => :float
+  col :fuel_oil_energy_intensity_units
+  col :district_heat_energy_intensity, :type => :float
+  col :district_heat_energy_intensity_units
 end

data/lib/earth/industry/mecs_energy.rb

@@ -47,16 +47,37 @@ class MecsEnergy < ActiveRecord::Base
     record
   end
   
-  # TODO make this less paranoid
-  # for now only return a ratio if no fuels are nil and at least one fuel is nonzero
   def fuel_ratios
-    if energy.to_f > 0 # return nil if energy is nil or 0
-      ratios = FUELS.inject({}) do |r, fuel|
-        fuel_use = send("#{fuel}")
-        r[fuel] = fuel_use.present? ? fuel_use / energy : nil
-        r
+    # Don't return a ratio if reported total energy was withheld
+    if energy.to_f > 0
+      # Calculate the sum of all fuels and note if any were withheld
+      withheld = 0
+      fuels_sum = MecsEnergy::FUELS.inject(0) do |sum, fuel|
+        (value = send("#{fuel}")).nil? ? withheld = 1 : sum += value
+        sum
+      end
+      
+      # If energy > sum of all fuels and some fuels were withheld, calculate fuel ratios as fraction of energy
+      # and attribute the disparity between energy and sum of all fuels to the dirtiest fuel that was withheld
+      if energy > fuels_sum and withheld == 1
+        ratios = MecsEnergy::FUELS.inject({}) do |r, fuel|
+          fuel_use = send("#{fuel}")
+          r[fuel] = fuel_use.present? ? fuel_use / energy : nil
+          r
+        end
+        
+        dirtiest_withheld = ([:coal, :other_fuel, :coke_and_breeze, :residual_fuel_oil, :distillate_fuel_oil, :lpg_and_ngl, :natural_gas] & ratios.select{|k,v| v.nil?}.keys).first
+        ratios[dirtiest_withheld] = (energy - fuels_sum) / energy
+        ratios.delete_if{ |fuel, ratio| ratio.to_f == 0.0 }
+      # Otherwise calculate ratios as fraction of sum of all fuels, skipping any fuels that were withheld
+      else
+        ratios = MecsEnergy::FUELS.inject({}) do |r, fuel|
+          fuel_use = send("#{fuel}")
+          r[fuel] = fuel_use / fuels_sum if fuel_use.to_f > 0
+          r
+        end
+        ratios.keys.any? ? ratios : nil
       end
-      ratios unless ratios.values.include? nil or ratios.values.all?(&:zero?) # return nil if any fuel uses were missing or all fuel uses were zero
     end
   end
 end

data/lib/earth/locality/country/data_miner.rb

@@ -103,23 +103,20 @@ Country.class_eval do
           row['electricity_co2_emission_factor'].to_f +
           (row['electricity_ch4_emission_factor'].to_f * GreenhouseGas[:ch4].global_warming_potential) +
           (row['electricity_n2o_emission_factor'].to_f * GreenhouseGas[:n2o].global_warming_potential)
-        ) / (1 - row['loss_factor'].to_f) }, :units => 'kilograms_co2e_per_kilowatt_hour'
+        ) }, :units => 'kilograms_co2e_per_kilowatt_hour'
+      store 'electricity_loss_factor', :field_name => 'loss_factor'
     end
     
-    process "Ensure EgridSubregion is populated" do
+    process "Ensure EgridSubregion and EgridRegion are populated" do
       EgridSubregion.run_data_miner!
+      EgridRegion.run_data_miner!
     end
     
-    process "Derive average US electricity emission factor from eGRID" do
+    process "Derive average US electricity emission factor and loss factor from eGRID" do
       us = united_states
-      subregion = EgridSubregion.fallback
-      us.electricity_emission_factor =
-        (
-          subregion.electricity_co2_emission_factor +
-          subregion.electricity_ch4_emission_factor +
-          subregion.electricity_n2o_emission_factor
-        ) / (1 - subregion.egrid_region.loss_factor)
-      us.electricity_emission_factor_units = 'kilograms_co2e_per_kilowatt_hour'
+      us.electricity_emission_factor = EgridSubregion.fallback.electricity_emission_factor
+      us.electricity_emission_factor_units = EgridSubregion.fallback.electricity_emission_factor_units
+      us.electricity_loss_factor = EgridRegion.fallback.loss_factor
       us.save!
     end
     

data/lib/earth/locality/country.rb

@@ -18,8 +18,9 @@ class Country < ActiveRecord::Base
                 :automobile_highway_speed_units => lambda { united_states.automobile_highway_speed_units }, # for now assume US represents world
                 :automobile_trip_distance => lambda { united_states.automobile_trip_distance }, # for now assume US represents world
                 :automobile_trip_distance_units => lambda { united_states.automobile_trip_distance_units }, # for now assume US represents world
-                :electricity_emission_factor => 0.69252, # from ecometrica paper - FIXME TODO calculate this
+                :electricity_emission_factor => 0.626089, # from ecometrica paper - FIXME TODO calculate this
                 :electricity_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour', # FIXME TODO derive this
+                :electricity_loss_factor => 0.096, # from ecometrica paper - FIXME TODO calculate this
                 :flight_route_inefficiency_factor => lambda { maximum(:flight_route_inefficiency_factor) }, # default to the largest inefficiency factor
                 :lodging_occupancy_rate => lambda { united_states.lodging_occupancy_rate }, # for now assume US represents world
                 :lodging_natural_gas_intensity => lambda { united_states.lodging_natural_gas_intensity }, # for now assume US represents world
@@ -68,6 +69,7 @@ class Country < ActiveRecord::Base
   col :automobile_trip_distance_units
   col :electricity_emission_factor, :type => :float
   col :electricity_emission_factor_units
+  col :electricity_loss_factor, :type => :float
   col :flight_route_inefficiency_factor, :type => :float
   col :lodging_occupancy_rate, :type => :float
   col :lodging_natural_gas_intensity, :type => :float

data/lib/earth/locality/state/data_miner.rb

@@ -26,5 +26,35 @@ State.class_eval do
       key   'postal_abbreviation', :field_name => 'State'
       store 'petroleum_administration_for_defense_district_code', :field_name => 'Code'
     end
+    
+    process 'ensure ZipCode, EgridSubregion, and EgridRegion are populated' do
+      ZipCode.run_data_miner!
+      EgridSubregion.run_data_miner!
+      EgridRegion.run_data_miner!
+    end
+    
+    process 'derive average electricity emission factor and loss factor from zip code and eGRID data' do
+      update_all %{
+        electricity_emission_factor = (
+          SELECT SUM(zip_codes.population * egrid_subregions.electricity_emission_factor) / SUM(zip_codes.population)
+          FROM zip_codes
+          INNER JOIN egrid_subregions ON egrid_subregions.abbreviation = zip_codes.egrid_subregion_abbreviation
+          WHERE zip_codes.state_postal_abbreviation = states.postal_abbreviation
+        ),
+        electricity_emission_factor_units = 'kilograms_co2e_per_kilowatt_hour',
+        electricity_loss_factor = (
+          SELECT SUM(zip_codes.population * egrid_regions.loss_factor) / SUM(zip_codes.population)
+          FROM zip_codes
+          INNER JOIN egrid_subregions ON egrid_subregions.abbreviation = zip_codes.egrid_subregion_abbreviation
+          INNER JOIN egrid_regions ON egrid_regions.name = egrid_subregions.egrid_region_name
+          WHERE zip_codes.state_postal_abbreviation = states.postal_abbreviation
+        )
+      }
+    end
+    
+    # TODO import this from US census? would be slightly different: 0.7% for Alaska, 0.2% for New Mexico, etc.
+    process 'derive population from zip code data' do
+      update_all "population = (SELECT SUM(population) FROM zip_codes WHERE zip_codes.state_postal_abbreviation = states.postal_abbreviation)"
+    end
   end
 end

data/lib/earth/locality/state.rb

@@ -4,8 +4,8 @@ class State < ActiveRecord::Base
   has_many :climate_divisions, :foreign_key => 'state_postal_abbreviation'
   has_many :zip_codes, :foreign_key => 'state_postal_abbreviation'
   belongs_to :census_division, :foreign_key => 'census_division_number'
-  belongs_to :petroleum_administration_for_defense_district, :foreign_key => 'petroleum_administration_for_defense_district_code'
   has_one :census_region, :through => :census_division
+  belongs_to :petroleum_administration_for_defense_district, :foreign_key => 'petroleum_administration_for_defense_district_code'
   
   def country
     Country.united_states
@@ -16,4 +16,8 @@ class State < ActiveRecord::Base
   col :name
   col :census_division_number, :type => :integer
   col :petroleum_administration_for_defense_district_code
+  col :population, :type => :integer
+  col :electricity_emission_factor, :type => :float
+  col :electricity_emission_factor_units
+  col :electricity_loss_factor, :type => :float
 end

data/lib/earth/locality/zip_code/data_miner.rb

@@ -32,6 +32,20 @@ ZipCode.class_eval do
       store 'climate_division_name'
     end
     
-    # FIXME TODO verify this
+    # TODO: download file directly from US census
+    # NOTE: ZCTAs are not zip codes but are based on the most common zip code in the area they cover - see http://www.census.gov/geo/ZCTA/zcta.html
+    import 'US Census 2010 zip code tabulation area populations',
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdHJsVjlKNXNLNEQyWENjZ1owd2hFS3c&output=csv',
+           :skip => 1 do
+      key 'name', :field_name => 'Id2', :sprintf => '%05d'
+      store 'population', :field_name => 'Total'
+    end
+    
+    import 'misc zip code data not included in other sources',
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdHFYaUE1cEdHTzZCcTFQOEZOTGVUemc&output=csv' do
+      key 'name', :sprintf => '%05d'
+      store 'state_postal_abbreviation'
+      store 'egrid_subregion_abbreviation'
+    end
   end
 end

data/lib/earth/locality/zip_code.rb

@@ -26,4 +26,5 @@ class ZipCode < ActiveRecord::Base
   col :longitude
   col :egrid_subregion_abbreviation
   col :climate_division_name
+  col :population, :type => :integer
 end

data/lib/earth/residence/clothes_machine_use/data_miner.rb

@@ -1,3 +1,4 @@
+require 'cohort_analysis'
 ClothesMachineUse.class_eval do
   data_miner do
     process "Ensure ResidentialEnergyConsumptionSurveyResponse is populated" do
@@ -15,7 +16,7 @@ ClothesMachineUse.class_eval do
     # sabshere 5/20/10 weird that this uses cohort
     process "precalculate annual energy use" do
       find_each do |record|
-        record.annual_energy_from_electricity_for_clothes_driers = ResidentialEnergyConsumptionSurveyResponse.big_cohort(:clothes_machine_use_id => record.name).weighted_average :annual_energy_from_electricity_for_clothes_driers
+        record.annual_energy_from_electricity_for_clothes_driers = ResidentialEnergyConsumptionSurveyResponse.cohort(:clothes_machine_use_id => record.name).weighted_average :annual_energy_from_electricity_for_clothes_driers
         record.annual_energy_from_electricity_for_clothes_driers_units = 'joules'
         record.save!
       end

data/lib/earth/residence/dishwasher_use/data_miner.rb

@@ -1,3 +1,4 @@
+require 'cohort_analysis'
 DishwasherUse.class_eval do
   data_miner do
     process "Ensure ResidentialEnergyConsumptionSurveyResponse is populated" do
@@ -15,7 +16,7 @@ DishwasherUse.class_eval do
     # sabshere 5/25/10 weird that this uses cohort
     process "precalculate annual energy" do
       find_each do |record|
-        record.annual_energy_from_electricity_for_dishwashers = ResidentialEnergyConsumptionSurveyResponse.big_cohort(:dishwasher_use_id => record.name).weighted_average :annual_energy_from_electricity_for_dishwashers
+        record.annual_energy_from_electricity_for_dishwashers = ResidentialEnergyConsumptionSurveyResponse.cohort(:dishwasher_use_id => record.name).weighted_average :annual_energy_from_electricity_for_dishwashers
         record.annual_energy_from_electricity_for_dishwashers_units = 'joules'
         record.save!
       end

data/lib/earth/residence/residence_fuel_price.rb

@@ -2,9 +2,6 @@ require 'earth/locality'
 class ResidenceFuelPrice < ActiveRecord::Base
   self.primary_key = "row_hash"
   
-  extend CohortScope
-  self.minimum_cohort_size = 5 # ? FIXME
-  
   belongs_to :fuel, :class_name => 'ResidenceFuelType', :foreign_key => 'residence_fuel_type_name'
   belongs_to :locatable, :polymorphic => true
   

data/lib/earth/residence/residential_energy_consumption_survey_response.rb

@@ -12,10 +12,6 @@ class ResidentialEnergyConsumptionSurveyResponse < ActiveRecord::Base
   belongs_to :air_conditioner_use
   belongs_to :clothes_machine_use
   
-  extend CohortScope
-  self.minimum_cohort_size = 5
-  SUBCOHORT_THRESHOLD = 5 # per Matt
-  
   INPUT_CHARACTERISTICS = [
     :ownership,
     :construction_year,

data/lib/earth/version.rb

@@ -1,3 +1,3 @@
 module Earth
-  VERSION = "0.11.12"
+  VERSION = "0.11.13"
 end

data/lib/earth.rb

@@ -1,6 +1,5 @@
 require 'active_support/core_ext'
 require 'active_record'
-require 'cohort_scope'
 require 'earth/conversions_ext'
 require 'earth/inflectors'
 require 'data_miner'

data/spec/earth/industry/mecs_energy_spec.rb

@@ -75,17 +75,137 @@ describe MecsEnergy do
   end
   
   describe '#fuel_ratios' do
-    it 'returns a list of fuel ratios for a given NAICS' do
-      energy = MecsEnergy.new :energy => 100, :electricity => 20,
-        :residual_fuel_oil => 1, :distillate_fuel_oil => 2,
-        :natural_gas => 30, :lpg_and_ngl => 22, :coal => 22,
-        :coke_and_breeze => 1, :other_fuel => 1
+    it 'calculates based on sum of all fuels when no fuels are missing' do
+      # Energy < sum all fuels
+      energy = MecsEnergy.new(
+        :energy => 50,
+        :electricity => 25,
+        :residual_fuel_oil => 10,
+        :distillate_fuel_oil => 10,
+        :natural_gas => 25,
+        :lpg_and_ngl => 5,
+        :coal => 5,
+        :coke_and_breeze => 15,
+        :other_fuel => 5
+      )
       energy.fuel_ratios.should == {
-        :electricity => 0.20,
-        :residual_fuel_oil => 0.01, :distillate_fuel_oil => 0.02,
-        :natural_gas => 0.3, :lpg_and_ngl => 0.22, :coal => 0.22,
-        :coke_and_breeze => 0.01, :other_fuel => 0.01
+        :electricity => 0.25,
+        :residual_fuel_oil => 0.1,
+        :distillate_fuel_oil => 0.1,
+        :natural_gas => 0.25,
+        :lpg_and_ngl => 0.05,
+        :coal => 0.05,
+        :coke_and_breeze => 0.15,
+        :other_fuel => 0.05
       }
+      
+      # Energy = sum all fuels
+      energy = MecsEnergy.new(
+        :energy => 100,
+        :electricity => 25,
+        :residual_fuel_oil => 10,
+        :distillate_fuel_oil => 10,
+        :natural_gas => 25,
+        :lpg_and_ngl => 5,
+        :coal => 5,
+        :coke_and_breeze => 15,
+        :other_fuel => 5
+      )
+      energy.fuel_ratios.should == {
+        :electricity => 0.25,
+        :residual_fuel_oil => 0.1,
+        :distillate_fuel_oil => 0.1,
+        :natural_gas => 0.25,
+        :lpg_and_ngl => 0.05,
+        :coal => 0.05,
+        :coke_and_breeze => 0.15,
+        :other_fuel => 0.05
+      }
+      
+      # Energy > sum of all fuels
+      energy = MecsEnergy.new(
+        :energy => 200,
+        :electricity => 25,
+        :residual_fuel_oil => 75,
+        :distillate_fuel_oil => 0,
+        :natural_gas => 0,
+        :lpg_and_ngl => 0,
+        :coal => 0,
+        :coke_and_breeze => 0,
+        :other_fuel => 0
+      )
+      energy.fuel_ratios.should == {
+        :electricity => 0.25,
+        :residual_fuel_oil => 0.75
+      }
+    end
+    
+    it 'ignores missing fuels when energy <= sum of all fuels' do
+      # Energy < sum of all fuels
+      energy = MecsEnergy.new(
+        :energy => 50,
+        :electricity => 25,
+        :residual_fuel_oil => 75
+      )
+      energy.fuel_ratios.should == {
+        :electricity => 0.25,
+        :residual_fuel_oil => 0.75
+      }
+      
+      # Energy = sum of all fuels
+      energy = MecsEnergy.new(
+        :energy => 100,
+        :electricity => 25,
+        :residual_fuel_oil => 75
+      )
+      energy.fuel_ratios.should == {
+        :electricity => 0.25,
+        :residual_fuel_oil => 0.75
+      }
+    end
+    
+    it 'adjusts for missing fuels when energy > sum of all fuels' do
+      energy = MecsEnergy.new(
+        :energy => 200,
+        :electricity => 25,
+        :residual_fuel_oil => 75
+      )
+      energy.fuel_ratios.should == {
+        :electricity => 0.125,
+        :residual_fuel_oil => 0.375,
+        :coal => 0.5
+      }
+    end
+    
+    it 'returns nil if energy is missing or zero' do
+      energy = MecsEnergy.new(
+        :energy => nil,
+        :electricity => 25,
+        :residual_fuel_oil => 75
+      )
+      energy.fuel_ratios.should be_nil
+      
+      energy = MecsEnergy.new(
+        :energy => 0,
+        :electricity => 25,
+        :residual_fuel_oil => 75
+      )
+      energy.fuel_ratios.should be_nil
+    end
+    
+    it 'returns nil if all fuels are zero' do
+      energy = MecsEnergy.new(
+        :energy => 1,
+        :electricity => 0,
+        :residual_fuel_oil => 0,
+        :distillate_fuel_oil => 0,
+        :natural_gas => 0,
+        :lpg_and_ngl => 0,
+        :coal => 0,
+        :coke_and_breeze => 0,
+        :other_fuel => 0
+      )
+      energy.fuel_ratios.should be_nil
     end
   end
 end

data/spec/earth/locality/country_spec.rb

@@ -1,13 +1,51 @@
 require 'spec_helper'
-require 'earth/locality'
-require 'earth/locality/data_miner'
+require 'earth/locality/country'
 
 describe Country do
+  before :all do
+    Country.auto_upgrade!
+  end
+  
   describe 'import', :slow => true do
+    before do
+      require 'earth/locality/country/data_miner'
+    end
+    
     it 'should import data' do
-      Earth.init 'locality', :load_data_miner => true, :apply_schemas => true
       Country.run_data_miner!
       Country.all.count.should == 249
     end
   end
+  
+  describe 'verify imported data', :sanity => true do
+    it 'has valid electricity emission factor and electricity loss factor for most countries' do
+      Country.where('electricity_emission_factor IS NOT NULL').count.should == 136
+      Country.where(:electricity_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour').count.should == 136
+      Country.where('electricity_loss_factor IS NOT NULL').count.should == 136
+      
+      Country.minimum(:electricity_emission_factor).should >= 0.0
+      Country.minimum(:electricity_loss_factor).should >= 0.0
+      Country.maximum(:electricity_loss_factor).should < 0.3
+      
+      Country.find('US').electricity_emission_factor.should be_within(0.00001).of(0.58946)
+      Country.find('US').electricity_emission_factor_units.should == 'kilograms_co2e_per_kilowatt_hour'
+      Country.find('US').electricity_loss_factor.should be_within(0.001).of(0.062)
+      
+      Country.find('GB').electricity_emission_factor.should be_within(0.00001).of(0.51020)
+      Country.find('GB').electricity_emission_factor_units.should == 'kilograms_co2e_per_kilowatt_hour'
+      Country.find('GB').electricity_loss_factor.should be_within(0.001).of(0.073)
+    end
+    
+    it 'has fallback electricity emission factor and electricity loss factor' do
+      Country.fallback.electricity_emission_factor.should be_within(0.00001).of(0.62609)
+      Country.fallback.electricity_emission_factor_units.should == 'kilograms_co2e_per_kilowatt_hour'
+      Country.fallback.electricity_loss_factor.should be_within(0.001).of(0.096)
+    end
+  end
+  
+  describe '.united_states' do
+    it 'should return the United States' do
+      Country.united_states.should == Country.find('US')
+    end
+  end
 end

data/spec/earth/locality/state_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+require 'earth/locality/state'
+
+describe State do
+  before :all do
+    State.auto_upgrade!
+  end
+  
+  describe 'when importing data', :slow => true do
+    before do
+      require 'earth/locality/state/data_miner'
+    end
+    
+    it 'imports data' do
+      State.run_data_miner!
+      State.count.should == 51 # includes DC but not any territories
+    end
+  end
+  
+  describe 'verify imported data', :sanity => true do
+    it 'should have a population' do
+      State.find('VT').population.should == 625741
+      State.find('CA').population.should == 37249542
+      State.find('MT').population.should == 990213
+      State.find('NM').population.should == 2056349
+    end
+    
+    it 'should have an average electricity emission factor' do
+      State.find('VT').electricity_emission_factor.should be_within(0.00001).of(0.37848)
+      State.find('VT').electricity_emission_factor_units.should == "kilograms_co2e_per_kilowatt_hour"
+      State.find('CA').electricity_emission_factor.should be_within(0.00001).of(0.31315)
+      State.find('CA').electricity_emission_factor_units.should == "kilograms_co2e_per_kilowatt_hour"
+      State.find('MT').electricity_emission_factor.should be_within(0.00001).of(0.41092)
+      State.find('MT').electricity_emission_factor_units.should == "kilograms_co2e_per_kilowatt_hour"
+      State.find('NM').electricity_emission_factor.should be_within(0.00001).of(0.57194)
+      State.find('NM').electricity_emission_factor_units.should == "kilograms_co2e_per_kilowatt_hour"
+    end
+    
+    it 'should have an average electricity loss factor' do
+      State.find('VT').electricity_loss_factor.should be_within(0.00001).of(0.0647079)
+      State.find('CA').electricity_loss_factor.should be_within(0.00001).of(0.0483723)
+      State.find('MT').electricity_loss_factor.should be_within(0.00001).of(0.0491516)
+      State.find('NM').electricity_loss_factor.should be_within(0.00001).of(0.0499872)
+    end
+  end
+  
+  describe '#country' do
+    before do
+      require 'earth/locality/country'
+    end
+    
+    it 'should the United States' do
+      State.first.country.should == Country.united_states
+      State.last.country.should == Country.united_states
+    end
+  end
+end

data/spec/earth/locality/zip_code_spec.rb

@@ -0,0 +1,59 @@
+require 'spec_helper'
+require 'earth/locality/zip_code'
+
+describe ZipCode do
+  before :all do
+    ZipCode.auto_upgrade!
+  end
+  
+  describe 'when importing data', :slow => true do
+    before do
+      require 'earth/locality/zip_code/data_miner'
+    end
+    
+    it 'imports data' do
+      ZipCode.run_data_miner!
+      ZipCode.count.should == 43770
+    end
+  end
+  
+  describe 'verify imported data', :sanity => true do
+    it 'has a state for most zip codes' do
+      ZipCode.where('state_postal_abbreviation IS NOT NULL').count.should == 43734
+    end
+    
+    it 'has a lat and lng for most zip codes' do
+      ZipCode.where('latitude IS NOT NULL AND longitude IS NOT NULL').count.should == 43191
+    end
+    
+    it 'has an eGRID subregion for most zip codes' do
+      ZipCode.where('egrid_subregion_abbreviation IS NOT NULL').count.should == 41297
+    end
+    
+    it 'has a climate division for most zip codes' do
+      ZipCode.where('climate_division_name IS NOT NULL').count.should == 41358
+    end
+    
+    it 'has a population for all 33120 Census 2010 ZCTAs' do
+      ZipCode.where('population IS NOT NULL').count.should == 33120
+    end
+  end
+  
+  describe '#country' do
+    before do
+      require 'earth/locality/country'
+    end
+    
+    it 'should return the US' do
+      ZipCode.first.country.should == Country.united_states
+      ZipCode.last.country.should == Country.united_states
+    end
+  end
+  
+  describe '#latitude_longitude' do
+    it 'should return the lat and lng as an array of strings' do
+      ZipCode.first.latitude_longitude.should == [nil, nil]
+      ZipCode.last.latitude_longitude.should == ['55.875767', '-131.46633']
+    end
+  end
+end

metadata

@@ -2,7 +2,7 @@
 name: earth
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.11.12
+  version: 0.11.13
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -12,7 +12,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-02-20 00:00:00 Z
+date: 2012-03-05 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: activerecord
@@ -37,7 +37,7 @@ dependencies:
   prerelease: false
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: cohort_scope
+  name: cohort_analysis
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -623,6 +623,8 @@ files:
 - spec/earth/industry/naics_2007_spec.rb
 - spec/earth/industry/sic_1987_spec.rb
 - spec/earth/locality/country_spec.rb
+- spec/earth/locality/state_spec.rb
+- spec/earth/locality/zip_code_spec.rb
 - spec/earth/pet/species_spec.rb
 - spec/earth_spec.rb
 - spec/spec_helper.rb
@@ -685,7 +687,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -2879856841116714656
+      hash: -2326149133598914243
       segments: 
       - 0
       version: "0"
@@ -694,7 +696,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -2879856841116714656
+      hash: -2326149133598914243
       segments: 
       - 0
       version: "0"
@@ -805,6 +807,8 @@ test_files:
 - spec/earth/industry/naics_2007_spec.rb
 - spec/earth/industry/sic_1987_spec.rb
 - spec/earth/locality/country_spec.rb
+- spec/earth/locality/state_spec.rb
+- spec/earth/locality/zip_code_spec.rb
 - spec/earth/pet/species_spec.rb
 - spec/earth_spec.rb
 - spec/spec_helper.rb