earth 0.12.3 → 0.12.4

data/lib/earth/locality/state.rb

@@ -1,12 +1,20 @@
 class State < ActiveRecord::Base
   self.primary_key = "postal_abbreviation"
   
+  has_many :zip_codes,         :foreign_key => 'state_postal_abbreviation'
   has_many :climate_divisions, :foreign_key => 'state_postal_abbreviation'
-  has_many :zip_codes, :foreign_key => 'state_postal_abbreviation'
   belongs_to :census_division, :foreign_key => 'census_division_number'
   has_one :census_region, :through => :census_division
+  has_one :electricity_mix, :foreign_key => 'state_postal_abbreviation'
   belongs_to :petroleum_administration_for_defense_district, :foreign_key => 'petroleum_administration_for_defense_district_code'
   
+  data_miner do
+    process "Ensure Country and ElectricityMix are imported because they're like belongs_to associations" do
+      Country.run_data_miner!
+      ElectricityMix.run_data_miner!
+    end
+  end
+  
   def country
     Country.united_states
   end
@@ -20,11 +28,11 @@ class State < ActiveRecord::Base
   col :electricity_emission_factor, :type => :float
   col :electricity_emission_factor_units
   col :electricity_loss_factor, :type => :float
-
+  
   warn_if_nulls_except(
     :census_division_number,
     :petroleum_administration_for_defense_district_code
   )
-
-  warn_unless_size 57
+  
+  warn_unless_size 51
 end

data/lib/earth/locality/state/data_miner.rb

@@ -1,5 +1,9 @@
 State.class_eval do
   data_miner do
+    process "Start from scratch" do
+      delete_all
+    end
+    
     # state names, FIPS codes, and postal abbreviations
     import 'the U.S. Census State ANSI Code file',
            :url => 'http://www.census.gov/geo/www/ansi/state.txt',
@@ -27,34 +31,45 @@ State.class_eval do
       store 'petroleum_administration_for_defense_district_code', :field_name => 'Code'
     end
     
-    process 'ensure ZipCode, EgridSubregion, and EgridRegion are populated' do
+    process 'ensure ZipCode and EgridSubregion are populated' do
       ZipCode.run_data_miner!
       EgridSubregion.run_data_miner!
-      EgridRegion.run_data_miner!
     end
     
+    # DEPRECATED - when cut this can also cut electricity_emission_factor from EgridSubrgion
     process 'derive average electricity emission factor and loss factor from zip code and eGRID data' do
-      update_all %{
-        electricity_emission_factor = (
-          SELECT SUM(zip_codes.population * egrid_subregions.electricity_emission_factor) / SUM(zip_codes.population)
-          FROM zip_codes
-          INNER JOIN egrid_subregions ON egrid_subregions.abbreviation = zip_codes.egrid_subregion_abbreviation
-          WHERE zip_codes.state_postal_abbreviation = states.postal_abbreviation
-        ),
-        electricity_emission_factor_units = 'kilograms_co2e_per_kilowatt_hour',
-        electricity_loss_factor = (
-          SELECT SUM(zip_codes.population * egrid_regions.loss_factor) / SUM(zip_codes.population)
-          FROM zip_codes
-          INNER JOIN egrid_subregions ON egrid_subregions.abbreviation = zip_codes.egrid_subregion_abbreviation
-          INNER JOIN egrid_regions ON egrid_regions.name = egrid_subregions.egrid_region_name
-          WHERE zip_codes.state_postal_abbreviation = states.postal_abbreviation
+      safe_find_each do |state|
+        sub_pops = state.zip_codes.known_subregion.sum(:population, :group => :egrid_subregion)
+        
+        ef = sub_pops.inject(0) do |memo, (subregion, population)|
+          memo += subregion.electricity_emission_factor * population
+          memo
+        end / sub_pops.values.sum
+        
+        state.update_attributes!(
+          :electricity_emission_factor => ef,
+          :electricity_emission_factor_units => 'kilograms_co2e_per_kilowatt_hour'
         )
-      }
+        
+        loss_factor = sub_pops.inject(0) do |memo, (subregion, population)|
+          memo += subregion.egrid_region.loss_factor * population
+          memo
+        end / sub_pops.values.sum
+        
+        state.update_attributes! :electricity_loss_factor => loss_factor
+      end
     end
     
     # TODO import this from US census? would be slightly different: 0.7% for Alaska, 0.2% for New Mexico, etc.
     process 'derive population from zip code data' do
-      update_all "population = (SELECT SUM(population) FROM zip_codes WHERE zip_codes.state_postal_abbreviation = states.postal_abbreviation)"
+      safe_find_each do |state|
+        state.update_attributes! :population => state.zip_codes.sum(:population)
+      end
+    end
+    
+    process "Data mine Country and ElectricityMix because they're like belongs_to associations" do
+      Country.run_data_miner!
+      ElectricityMix.run_data_miner!
     end
   end
 end

data/lib/earth/locality/zip_code.rb

@@ -3,12 +3,20 @@ require ::File.join(Earth::VENDOR_DIR, 'geokit-rails', 'lib', 'geokit-rails') #
 class ZipCode < ActiveRecord::Base
   self.primary_key = "name"
   
-  belongs_to :egrid_subregion,  :foreign_key => 'egrid_subregion_abbreviation'
   belongs_to :climate_division, :foreign_key => 'climate_division_name'
   belongs_to :state,            :foreign_key => 'state_postal_abbreviation'
+  belongs_to :egrid_subregion,  :foreign_key => 'egrid_subregion_abbreviation'
   has_many :electric_markets,   :foreign_key => 'zip_code_name'
   has_many :electric_utilities, :through => :electric_markets
   
+  scope :known_subregion, where('egrid_subregion_abbreviation IS NOT NULL')
+  
+  data_miner do
+    process "Ensure Country is imported because it's like a belongs_to association" do
+      Country.run_data_miner!
+    end
+  end
+  
   def country
     Country.united_states
   end

data/lib/earth/locality/zip_code/data_miner.rb

@@ -48,6 +48,10 @@ ZipCode.class_eval do
       }
     end
     
+    process "Data mine Country because it's like a belongs_to association" do
+      Country.run_data_miner!
+    end
+    
     # FIXME TODO figure out how to speed this up then re-enable it
     # process "Look up missing latitude and longitude" do
     #   where("latitude IS NULL OR longitude IS NULL").each do |zip|

data/lib/earth/rail/country_rail_traction/data_miner.rb

@@ -17,7 +17,7 @@ CountryRailTraction.class_eval do
     end
     
     process "Standardize diesel intensity units" do
-      CountryRailTraction.where(:diesel_intensity_units => 'grams_per_passenger_kilometre').find_each do |record|
+      CountryRailTraction.where(:diesel_intensity_units => 'grams_per_passenger_kilometre').safe_find_each do |record|
         diesel = RailFuel.find_by_name("diesel")
         record.diesel_intensity = record.diesel_intensity.grams.to(:kilograms) / diesel.density
         record.diesel_intensity_units = [diesel.density_units.split("_per_")[1].pluralize, record.diesel_intensity_units.split("_per_")[1]].join('_per_')
@@ -26,7 +26,7 @@ CountryRailTraction.class_eval do
     end
     
     process "Standardize co2 emission factor units" do
-      CountryRailTraction.where(:co2_emission_factor_units => 'grams_per_passenger_kilometre').find_each do |record|
+      CountryRailTraction.where(:co2_emission_factor_units => 'grams_per_passenger_kilometre').safe_find_each do |record|
         record.co2_emission_factor = record.co2_emission_factor.grams.to(:kilograms)
         record.co2_emission_factor_units = ['kilograms', record.co2_emission_factor_units.split("_per_")[1]].join('_per_')
         record.save!

data/lib/earth/rail/country_rail_traction_class/data_miner.rb

@@ -18,7 +18,7 @@ CountryRailTractionClass.class_eval do
     end
     
     process "Standardize diesel intensity units" do
-      CountryRailTractionClass.where(:diesel_intensity_units => 'grams_per_passenger_kilometre').find_each do |record|
+      CountryRailTractionClass.where(:diesel_intensity_units => 'grams_per_passenger_kilometre').safe_find_each do |record|
         diesel = RailFuel.find_by_name("diesel")
         record.diesel_intensity = record.diesel_intensity.grams.to(:kilograms) / diesel.density
         record.diesel_intensity_units = [diesel.density_units.split("_per_")[1].pluralize, record.diesel_intensity_units.split("_per_")[1]].join('_per_')
@@ -27,7 +27,7 @@ CountryRailTractionClass.class_eval do
     end
     
     process "Standardize co2 emission factor units" do
-      CountryRailTractionClass.where(:co2_emission_factor_units => 'grams_per_passenger_kilometre').find_each do |record|
+      CountryRailTractionClass.where(:co2_emission_factor_units => 'grams_per_passenger_kilometre').safe_find_each do |record|
         record.co2_emission_factor = record.co2_emission_factor.grams.to(:kilograms)
         record.co2_emission_factor_units = ['kilograms', record.co2_emission_factor_units.split("_per_")[1]].join('_per_')
         record.save!

data/lib/earth/rail/national_transit_database_record.rb

@@ -41,10 +41,14 @@ class NationalTransitDatabaseRecord < ActiveRecord::Base
   col :other_fuel_description
 
   warn_if_nulls_except(
+    :passenger_distance,
+    :passenger_distance_units,
     :electricity,
     :electricity_units,
     :diesel,
     :diesel_units,
+    :gasoline,
+    :gasoline_units,
     :lpg,
     :lpg_units,
     :lng,

data/lib/earth/rail/rail_company/data_miner.rb

@@ -57,7 +57,7 @@ RailCompany.class_eval do
     end
     
     process "Calculate average trip distance" do
-      find_each do |company|
+      safe_find_each do |company|
         if company.passenger_distance.present? and company.passengers.present? and company.passengers > 0
           company.trip_distance = company.passenger_distance / company.passengers
           company.trip_distance_units = company.passenger_distance_units
@@ -67,7 +67,7 @@ RailCompany.class_eval do
     end
     
     process "Calculate average trip speed" do
-      find_each do |company|
+      safe_find_each do |company|
         if company.train_distance.present? and company.train_time.present? and company.train_time > 0
           company.speed = company.train_distance / company.train_time
           company.speed_units = "#{company.train_distance_units}_per_#{company.train_time_units.singularize}"
@@ -82,7 +82,7 @@ RailCompany.class_eval do
     end
     
     process "Calculate co2 emission factor for US rail companies" do
-      where(:country_iso_3166_code => 'US').find_each do |company|
+      where(:country_iso_3166_code => 'US').safe_find_each do |company|
         company.co2_emission_factor = 0
         
         if company.diesel_intensity.present?

data/lib/earth/rail/rail_fuel/data_miner.rb

@@ -15,7 +15,7 @@ RailFuel.class_eval do
     end
     
     process "Convert ch4 and n2o emission factor units to kg co2e / l" do
-      RailFuel.find_each do |fuel|
+      RailFuel.safe_find_each do |fuel|
         if fuel.ch4_emission_factor_units == "grams_per_kilogram" and fuel.density_units == "kilograms_per_litre"
           fuel.ch4_emission_factor = GreenhouseGas["ch4"].global_warming_potential * fuel.density * fuel.ch4_emission_factor / 1000.0
           fuel.ch4_emission_factor_units = "kilograms_co2e_per_litre"

data/lib/earth/residence/clothes_machine_use/data_miner.rb

@@ -15,7 +15,7 @@ ClothesMachineUse.class_eval do
     
     # sabshere 5/20/10 weird that this uses cohort
     process "precalculate annual energy use" do
-      find_each do |record|
+      safe_find_each do |record|
         record.annual_energy_from_electricity_for_clothes_driers = ResidentialEnergyConsumptionSurveyResponse.cohort(:clothes_machine_use_id => record.name).weighted_average :annual_energy_from_electricity_for_clothes_driers
         record.annual_energy_from_electricity_for_clothes_driers_units = 'joules'
         record.save!

data/lib/earth/residence/dishwasher_use/data_miner.rb

@@ -15,7 +15,7 @@ DishwasherUse.class_eval do
     
     # sabshere 5/25/10 weird that this uses cohort
     process "precalculate annual energy" do
-      find_each do |record|
+      safe_find_each do |record|
         record.annual_energy_from_electricity_for_dishwashers = ResidentialEnergyConsumptionSurveyResponse.cohort(:dishwasher_use_id => record.name).weighted_average :annual_energy_from_electricity_for_dishwashers
         record.annual_energy_from_electricity_for_dishwashers_units = 'joules'
         record.save!

data/lib/earth/residence/residence_fuel_price.rb

@@ -17,5 +17,5 @@ class ResidenceFuelPrice < ActiveRecord::Base
   add_index [:price, :residence_fuel_type_name, :month, :year, :locatable_type, :locatable_id]
   add_index [:price, :residence_fuel_type_name]
 
-  warn_unless_size 13005
+  warn_unless_size 13639
 end

data/lib/earth/version.rb

@@ -1,3 +1,3 @@
 module Earth
-  VERSION = "0.12.3"
+  VERSION = "0.12.4"
 end

data/spec/earth/air/airline_spec.rb

@@ -14,7 +14,7 @@ describe Airline do
   
   describe "verify imported data", :sanity => true do
     it "should have all the data" do
-      Airline.count.should == 1522
+      Airline.count.should == 1523
     end
   end
 end

data/spec/earth/air/airport_spec.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 require 'spec_helper'
 require 'earth/air/airport'
 
@@ -17,12 +19,17 @@ describe Airport do
       Airport.count.should == 5324
     end
     
+    it "should use utf-8 encoding" do
+      Airport.find('PDA').city.should == 'Puerto Inírida'
+    end
+    
     it "should have name" do
       Airport.where(:name => nil).count.should == 0
     end
     
     it "should have city" do
       Airport.where(:city => nil).count.should == 0
+      Airport.where(:city => '').count.should == 0
     end
     
     it "should have country code" do

data/spec/earth/air/flight_segment_spec.rb

@@ -12,41 +12,55 @@ describe FlightSegment do
     end
   end
   
-  describe "verify imported data", :sanity => true do
+  describe "verify imported data", :sanity => true, :flight_segment => true do
     it "should have all the data" do
-      FlightSegment.where(:year => 2009).count.should == 403_959
-      FlightSegment.where(:year => 2010).count.should == 421_725
-      FlightSegment.where(:year => 2011).count.should > 357_687
+      FlightSegment.where(:year => 2009).count.should == 403_980
+      FlightSegment.where(:year => 2010).count.should == 421_884
+      FlightSegment.where(:year => 2011).count.should == 428_550
     end
     
     it "should have year from 2009 to present" do
-      FlightSegment.where("year IS NULL OR year < 2009 OR year > #{::Time.now.year}").count.should == 0
+      FlightSegment.where("NOT(year >= 2009 AND year <= #{::Time.now.year})").count.should == 0
     end
     
     it "should have data through 7 months ago" do
-      latest = Date.today << 7
+      if (today = Date.today).day < 23
+        latest = today << 7
+      else
+        latest = today << 6
+      end
       FlightSegment.maximum(:year).should == latest.year
       FlightSegment.where(:year => latest.year).maximum(:month).should == latest.month
     end
     
     it "should have origin airport in airports" do
-      # FIXME TODO
+      FlightSegment.connection.select_values("SELECT DISTINCT origin_airport_iata_code FROM flight_segments").each do |code|
+        Airport.exists?(:iata_code => code).should == true
+      end
     end
     
     it "should have destination airport in airports" do
-      # FIXME TODO
+      FlightSegment.connection.select_values("SELECT DISTINCT destination_airport_iata_code FROM flight_segments").each do |code|
+        Airport.exists?(:iata_code => code).should == true
+      end
     end
     
     it "should have origin country iso code in countries" do
-      # FIXME TODO
+      FlightSegment.connection.select_values("SELECT DISTINCT origin_country_iso_3166_code FROM flight_segments").each do |code|
+        Country.exists?(:iso_3166_code => code).should == true
+      end
     end
     
     it "should have destination country iso code in countries" do
-      # FIXME TODO
+      FlightSegment.connection.select_values("SELECT DISTINCT destination_country_iso_3166_code FROM flight_segments").each do |code|
+        Country.exists?(:iso_3166_code => code).should == true
+      end
     end
     
-    it "should have airline name" do
-      FlightSegment.where(:airline_name => nil).count.should == 0
+    it "should have airline name in airlines" do
+      FlightSegment.connection.select_values("SELECT DISTINCT airline_name FROM flight_segments").each do |name|
+        Airline.exists?(:name => name).should == true
+      end
     end
     
     it "should have aircraft description" do

data/spec/earth/automobile/automobile_type_fuel_spec.rb

@@ -35,7 +35,7 @@ describe AutomobileTypeFuel do
   
   describe '#latest_activity_year_type_fuel' do
     it 'should always be a match from 2009' do
-      AutomobileTypeFuel.find_each do |atf|
+      AutomobileTypeFuel.safe_find_each do |atf|
         atf.latest_activity_year_type_fuel.type_name.should == atf.type_name
         atf.latest_activity_year_type_fuel.fuel_family.should == atf.fuel_family
         atf.latest_activity_year_type_fuel.activity_year.should == 2009

data/spec/earth/automobile/automobile_type_fuel_year_spec.rb

@@ -29,11 +29,11 @@ describe AutomobileTypeFuelYear do
     it { ATFY.where("n2o_emission_factor > 0").count.should == ATFY.count }
     
     # spot checks
-    it { test_atfy.annual_distance.should == be_within(0.01).of(19598.59) }
+    it { test_atfy.annual_distance.should be_within(0.05).of(19598.6) }
     it { test_atfy.annual_distance_units.should == 'kilometres' }
-    it { test_atfy.ch4_emission_factor.should == be_within(1e-8).of(2.286e-4) }
+    it { test_atfy.ch4_emission_factor.should be_within(5e-7).of(0.000229) }
     it { test_atfy.ch4_emission_factor_units.should == 'kilograms_co2e_per_kilometre' }
-    it { test_atfy.n2o_emission_factor.should == be_within(1e-7).of(1.4688e-3) }
+    it { test_atfy.n2o_emission_factor.should be_within(5e-6).of(0.00147) }
     it { test_atfy.n2o_emission_factor_units.should == 'kilograms_co2e_per_kilometre' }
   end
   

data/spec/earth/fuel/fuel_spec.rb

@@ -20,7 +20,7 @@ describe Fuel do
     
     it 'should have a record for district heat' do
       district_heat = Fuel.find 'District Heat'
-      district_heat.co2_emission_factor.should be_within(0.00001).of(0.0766528)
+      district_heat.co2_emission_factor.should be_within(5e-6).of(0.07598)
     end
     
     # TODO is there a way to check whether the value in the fuels table is NULL?

data/spec/earth/hospitality/commercial_building_energy_consumption_survey_response_spec.rb

@@ -2,42 +2,55 @@ require 'spec_helper'
 require 'earth/hospitality/commercial_building_energy_consumption_survey_response'
 
 describe CommercialBuildingEnergyConsumptionSurveyResponse do
+  let(:cbecs) { CommercialBuildingEnergyConsumptionSurveyResponse }
+  
   describe "when importing data", :data_miner => true do
     before do
       Earth.init :hospitality, :load_data_miner => true, :skip_parent_associations => :true
     end
     
     it "imports all naics codes" do
-      CommercialBuildingEnergyConsumptionSurveyResponse.run_data_miner!
+      cbecs.run_data_miner!
     end
   end
   
   describe "verify imported data", :sanity => true do
-    it { CommercialBuildingEnergyConsumptionSurveyResponse.count.should == 5215 }
+    let(:total) { cbecs.count }
+    let(:first_lodging) { cbecs.lodging_records.first }
     
-    it { CommercialBuildingEnergyConsumptionSurveyResponse.where(:electricity_use      => nil).count.should == 0 }
-    it { CommercialBuildingEnergyConsumptionSurveyResponse.where(:electricity_energy   => nil).count.should == 0 }
-    it { CommercialBuildingEnergyConsumptionSurveyResponse.where(:natural_gas_use      => nil).count.should == 0 }
-    it { CommercialBuildingEnergyConsumptionSurveyResponse.where(:natural_gas_energy   => nil).count.should == 0 }
-    it { CommercialBuildingEnergyConsumptionSurveyResponse.where(:fuel_oil_use         => nil).count.should == 0 }
-    it { CommercialBuildingEnergyConsumptionSurveyResponse.where(:fuel_oil_energy      => nil).count.should == 0 }
-    it { CommercialBuildingEnergyConsumptionSurveyResponse.where(:district_heat_use    => nil).count.should == 0 }
-    it { CommercialBuildingEnergyConsumptionSurveyResponse.where(:district_heat_energy => nil).count.should == 0 }
+    it { total.should == 5215 }
+    it { cbecs.where("electricity_use >= 0").count.should == total }
+    it { cbecs.where("electricity_energy >= 0").count.should == total }
+    it { cbecs.where("natural_gas_use >= 0").count.should == total }
+    it { cbecs.where("natural_gas_energy >= 0").count.should == total }
+    it { cbecs.where("fuel_oil_use >= 0").count.should == total }
+    it { cbecs.where("fuel_oil_energy >= 0").count.should == total }
+    it { cbecs.where("district_heat_use >= 0").count.should == total }
+    it { cbecs.where("district_heat_energy >= 0").count.should == total }
     
+    # spot check
     it "should have room nights and fuel intensities per room night for lodging_records" do
-      spot_check = CommercialBuildingEnergyConsumptionSurveyResponse.lodging_records.first
-      spot_check.room_nights.should == 6205
-      spot_check.natural_gas_per_room_night.should be_within(0.00001).of(1.62234)
-      spot_check.natural_gas_per_room_night_units.should == 'cubic_metres_per_room_night'
+      first_lodging.room_nights.should == 6205
+      
+      first_lodging.electricity_energy.should be_within(0.5).of(221809)
+      first_lodging.electricity_per_room_night.should be_within(5e-4).of(37.711)
+      first_lodging.electricity_per_room_night_units.should == 'megajoules_per_room_night'
+      
+      first_lodging.natural_gas_energy.should be_within(0.5).of(386700)
+      first_lodging.natural_gas_per_room_night.should be_within(5e-4).of(79.343)
+      first_lodging.natural_gas_per_room_night_units.should == 'megajoules_per_room_night'
+      
+      first_lodging.fuel_oil_energy.should be_within(5e-4).of(0)
+      first_lodging.fuel_oil_per_room_night.should be_within(5e-4).of(12.97)
+      first_lodging.fuel_oil_per_room_night_units.should == 'megajoules_per_room_night'
     end
   end
   
   describe ".lodging_records" do
-    it "should return only records representing Hotels, Motels, and Inns with no other activity" do
-      lodging_records = CommercialBuildingEnergyConsumptionSurveyResponse.lodging_records
-      lodging_records.map(&:detailed_activity).uniq.sort.should == ['Hotel', 'Motel or inn']
-      lodging_records.map(&:first_activity).uniq.should == [nil]
-      lodging_records.count.should == 192
+    it "should return records representing Hotels, Motels, and Inns with no other activity" do
+      cbecs.lodging_records.map(&:detailed_activity).uniq.sort.should == ['Hotel', 'Motel or inn']
+      cbecs.lodging_records.map(&:first_activity).uniq.should == [nil]
+      cbecs.lodging_records.count.should == 192
     end
   end
 end