earth 0.11.20 → 0.12.0

data/lib/earth/automobile/automobile_type_fuel_year_control.rb

@@ -1,34 +1,36 @@
-require 'earth/fuel'
 class AutomobileTypeFuelYearControl < ActiveRecord::Base
   self.primary_key = "name"
   
+  # Needs to be a belongs_to so that it gets imported with taps for AutomobileTypeFuelYear ch4 and n2o ef calculation
+  belongs_to :type_fuel_control, :foreign_key => :type_fuel_control_name, :class_name => 'AutomobileTypeFuelControl'
+  
+  # Used by AutomobileTypeFuelYear
+  def self.find_all_by_type_name_and_fuel_common_name_and_closest_year(type_name, fuel_common_name, year)
+    if year > maximum(:year)
+      where(:type_name => type_name, :fuel_common_name => fuel_common_name, :year => maximum(:year))
+    else
+      where(:type_name => type_name, :fuel_common_name => fuel_common_name, :year => [year, minimum(:year)].max)
+    end
+  end
+  
+  %w{ ch4_emission_factor n2o_emission_factor }.each do |method|
+    define_method method do
+      type_fuel_control.send(method)
+    end
+    
+    units_method = method + '_units'
+    define_method units_method do
+      type_fuel_control.send(units_method)
+    end
+  end
+  
   col :name
   col :type_name
   col :fuel_common_name
   col :year, :type => :integer
   col :control_name
+  col :type_fuel_control_name
   col :total_travel_percent, :type => :float
   
-  # %w{ type_name fuel_common_name control_name type_fuel_control_name type_fuel_year_name }.each do |attribute|
-  #   verify "#{attribute.humanize} should never be missing" do
-  #     AutomobileTypeFuelYearControl.all.each do |record|
-  #       value = record.send(:"#{attribute}")
-  #       unless value.present?
-  #         raise "Missing #{attribute.humanize.downcase} for AutomobileTypeFuelYearControl '#{record.name}'"
-  #       end
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Year should be from 1990 to 2008" do
-  #   AutomobileTypeFuelYearControl.all.each do |record|
-  #     year = record.send(:year)
-  #     unless year > 1989 and year < 2009
-  #       raise "Invalid year for AutomobileTypeFuelYearControl '#{record.name}': #{year} (should be from 1990 to 2008)"
-  #     end
-  #   end
-  # end
-  
-  # FIXME TODO verify "Total travel percent for each type fuel year should sum to one"
-  
+  warn_unless_size 142
 end

data/lib/earth/automobile/automobile_type_fuel_year_control/data_miner.rb

@@ -1,13 +1,17 @@
-require 'earth/fuel/data_miner'
 AutomobileTypeFuelYearControl.class_eval do
   data_miner do
+    process "Start from scratch" do
+      delete_all
+    end
+    
     import "automobile type fuel year control data derived from the 2010 EPA GHG Inventory",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGpQV2xMdlZkV1JzVlVTeU5ZalF6elE&hl=en&gid=0&output=csv' do
+           :url => "file://#{Earth::DATA_DIR}/automobile/annual_emission_controls.csv" do
       key   'name'
       store 'type_name'
       store 'fuel_common_name'
       store 'year'
       store 'control_name'
+      store 'type_fuel_control_name', :synthesize => proc { |row| [row['type_name'], row['fuel_common_name'], row['control_name']].join(' ') }
       store 'total_travel_percent'
     end
   end

data/lib/earth/automobile/automobile_year.rb

@@ -1,6 +1,43 @@
-require 'earth/fuel'
 class AutomobileYear < ActiveRecord::Base
   self.primary_key = "year"
   
+  # Estimate of % of vehicles currently in use that were manufactured in each year
+  # Derived from 2012 EPA GHG inventory appendix tables A-91, A-92, and A-95
+  # Used by AutomobileMakeModelYear and AutomobileMakeYear to get weighting so other classes can derive weighted fuel efficiency
+  def self.weighting(year)
+    {
+      2012 => 0.0607,
+      2011 => 0.0491,
+      2010 => 0.0560,
+      2009 => 0.0669,
+      2008 => 0.0677,
+      2007 => 0.0668,
+      2006 => 0.0641,
+      2005 => 0.0620,
+      2004 => 0.0624,
+      2003 => 0.0625,
+      2002 => 0.0610,
+      2001 => 0.0546,
+      2000 => 0.0467,
+      1999 => 0.0410,
+      1998 => 0.0347,
+      1997 => 0.0289,
+      1996 => 0.0244,
+      1995 => 0.0189,
+      1994 => 0.0139,
+      1993 => 0.0112,
+      1992 => 0.0094,
+      1991 => 0.0084,
+      1990 => 0.0066,
+      1989 => 0.0056,
+      1988 => 0.0047,
+      1987 => 0.0037,
+      1986 => 0.0025,
+      1985 => 0.0054
+    }[year]
+  end
+  
   col :year, :type => :integer
+
+  warn_unless_size 28
 end

data/lib/earth/automobile/automobile_year/data_miner.rb

@@ -1,11 +1,10 @@
-require 'earth/fuel/data_miner'
 AutomobileYear.class_eval do
   data_miner do
     process "Ensure AutomobileMakeModelYearVariant is populated" do
       AutomobileMakeModelYearVariant.run_data_miner!
     end
     
-    process "Derive year names from automobile make model year variants" do
+    process "Derive year names from AutomobileMakeModelYearVariant" do
       ::Earth::Utils.insert_ignore(
         :src => AutomobileMakeModelYearVariant,
         :dest => AutomobileYear,

data/lib/earth/automobile/dependencies.txt

@@ -1,45 +1,31 @@
 This file shows which additional classes will be data_mined when you run data_miner on an automobile class. To get a complete set of automobile data you only need to data_mine the classes that are not indented.
 
+MakeModel
+	MakeModelYear
+		MakeModelYearVariant
+		AutomobileFuel (via belongs_to)
+			TypeFuel
+				TypeFuelYear
+					TypeFuelYearControl
+						TypeFuelControl (via belongs_to)
+				ActivityYearTypeFuel
+			Fuel
+				FuelYear
+			GreenhouseGas
+
 Make
-  MakeYearFleet
-  MakeModelYearVariant
+	MakeYear
+		MakeModelYearVariant
+		MakeYearFleet
 
 Model
-  MakeYearFleet
-  MakeModelYearVariant
+	MakeModelYearVariant
 
 Year
-  MakeYearFleet
-  MakeModelYearVariant
-
-MakeModel
-  MakeModelYearVariant
-
-MakeYear
-  MakeYearFleet
-  MakeModelYearVariant
-
-MakeModelYear
-  MakeModelYearVariant
-  AutomobileFuel
-    GreenhouseGas
-    Fuel
-      FuelYear
-    TypeYear
-      TypeFuelYear
-        TypeFuelControl
-        TypeFuelYearControl
-    TypeFuelYearAge
-      TypeFuelYear
-        TypeFuelControl
-        TypeFuelYearControl
+	MakeModelYearVariant
 
 SizeClass
-  TypeYear
-    TypeFuelYear
-      TypeFuelControl
-      TypeFuelYearControl
-  TypeFuelYearAge
-    TypeFuelYear
-      TypeFuelControl
-      TypeFuelYearControl
+
+ActivityYear
+	ActivityYearType
+		ActivityYearTypeFuel

data/lib/earth/bus/bus_class.rb

@@ -51,4 +51,6 @@ class BusClass < ActiveRecord::Base
   col :electricity_intensity_units
   col :air_conditioning_emission_factor, :type => :float
   col :air_conditioning_emission_factor_units
+
+  warn_unless_size 2
 end

data/lib/earth/bus/bus_fuel.rb

@@ -91,4 +91,6 @@ class BusFuel < ActiveRecord::Base
   #     end
   #   end
   # end
+
+  warn_unless_size 7
 end

data/lib/earth/bus/bus_fuel_control.rb

@@ -8,4 +8,6 @@ class BusFuelControl < ActiveRecord::Base
   col :ch4_emission_factor_units
   col :n2o_emission_factor, :type => :float
   col :n2o_emission_factor_units
+
+  warn_unless_size 9
 end

data/lib/earth/bus/bus_fuel_year_control.rb

@@ -10,4 +10,6 @@ class BusFuelYearControl < ActiveRecord::Base
   col :control
   col :bus_fuel_control_name
   col :total_travel_percent, :type => :float
+
+  warn_unless_size 67
 end

data/lib/earth/computation/computation_carrier.rb

@@ -15,4 +15,6 @@ class ComputationCarrier < ActiveRecord::Base
   #     end
   #   end
   # end
+
+  warn_unless_size 1
 end

data/lib/earth/computation/computation_carrier_instance_class.rb

@@ -32,4 +32,6 @@ class ComputationCarrierInstanceClass < ActiveRecord::Base
   #   end
   # end
   
+
+  warn_unless_size 8
 end

data/lib/earth/computation/computation_carrier_region.rb

@@ -8,4 +8,6 @@ class ComputationCarrierRegion < ActiveRecord::Base
   col :computation_carrier_name
   col :region
   col :egrid_subregion_abbreviation
+
+  warn_unless_size 4
 end

data/lib/earth/conversions_ext.rb

@@ -3,6 +3,7 @@ require 'conversions'
 # Distance: base unit = kilometre
 Conversions.register :kilometres, :miles,          0.621371192
 Conversions.register :kilometres, :nautical_miles, 0.539956803
+Conversions.register :kilometres, :billion_miles,  1.kilometres.to(:miles) / 1_000_000_000 # for AutomobileTypeFuelYear
 
 # Area
 Conversions.register :square_metres, :square_feet,         10.7639104
@@ -68,6 +69,11 @@ Conversions.register :teragrams_per_quadrillion_btu,        :grams_per_megajoule
 # Conversions.register :epa_miles_per_gallon_gasoline_equivalent, :cubic_metres_per_hundred_kilometres,   (1 / (1.cubic_metres_compressed_natural_gas.to(:epa_gallon_gasoline_equivalents) / 100.kilometres.to(:miles)))
 # Conversions.register :epa_miles_per_gallon_gasoline_equivalent, :kilograms_per_hundred_kilometres,      (1 / (1.kilograms_hydrogen.to(:epa_gallon_gasoline_equivalents) / 100.kilometres.to(:miles)))
 
+# Odd units for EPA automobile data
+Conversions.register :grams_per_mile,  :kilograms_per_kilometre, 1.grams.to(:kilograms) / 1.miles.to(:kilometres)
+Conversions.register :teragrams_co2e,  :kilograms_co2e,          1_000_000_000.0
+Conversions.register :million_gallons, :litres,                  1_000_000.gallons.to(:litres)
+
 # Odd units for pet - FIXME use megajoules rather than joules
 Conversions.register :kilocalories,           :joules,              4_184.0
 Conversions.register :kilocalories_per_pound, :joules_per_kilogram, (1.kilocalories.to(:joules) / 1.pounds.to(:kilograms))

data/lib/earth/diet/diet_class.rb

@@ -20,4 +20,6 @@ class DietClass < ActiveRecord::Base
   col :fruit_share, :type => :float
   col :vegetables_share, :type => :float
   col :oils_and_sugars_share, :type => :float
+
+  warn_unless_size 3
 end

data/lib/earth/diet/food_group.rb

@@ -17,4 +17,6 @@ class FoodGroup < ActiveRecord::Base
   col :energy, :type => :float
   col :energy_units
   col :suggested_imperial_measurement # ?
+
+  warn_unless_size 10
 end

data/lib/earth/fuel/fuel.rb

@@ -109,4 +109,6 @@ class Fuel < ActiveRecord::Base
       latest_fuel_year.co2_biogenic_emission_factor_units
     end
   end
+
+  warn_unless_size 21
 end

data/lib/earth/fuel/fuel/data_miner.rb

@@ -1,5 +1,9 @@
 Fuel.class_eval do
   data_miner do
+    process "Start from scratch" do
+      delete_all
+    end
+    
     process "Ensure FuelYear is populated" do
       FuelYear.run_data_miner!
     end
@@ -34,6 +38,17 @@ Fuel.class_eval do
       store 'biogenic_fraction'
     end
     
+    import "hydrogen",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGJmYkdtajZyV3Byb0lrd21xLVhXUGc&output=csv',
+           :select => proc {|row| row['energy_content_units'] == 'megajoules_per_kilogram'} do
+      key 'name'
+      store 'physical_units', :static => 'kilogram'
+      store 'energy_content', :units_field_name => 'energy_content_units'
+      store 'carbon_content', :units_field_name => 'carbon_content_units'
+      store 'oxidation_factor'
+      store 'biogenic_fraction'
+    end
+    
     import "densities for aircraft fuels",
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHBjTVE4NmRlc05iUHVZR1E3eEJwOGc&hl=en&gid=0&output=csv' do
       key 'name'

data/lib/earth/fuel/fuel_price.rb

@@ -6,4 +6,6 @@ class FuelPrice < ActiveRecord::Base
   col :name
   col :price, :type => :float
   col :price_units
+
+  warn_unless_size 34
 end

data/lib/earth/fuel/fuel_type.rb

@@ -24,4 +24,6 @@ class FuelType < ActiveRecord::Base
   # col :energy_content_units
   # col :carbon_content, :type => :float
   # col :carbon_content_units
+
+  warn_unless_size 36
 end

data/lib/earth/fuel/fuel_year.rb

@@ -59,4 +59,6 @@ class FuelYear < ActiveRecord::Base
   # end
   # 
   # FIXME TODO verify units
+
+  warn_unless_size 171
 end

data/lib/earth/fuel/greenhouse_gas.rb

@@ -52,4 +52,6 @@ class GreenhouseGas < ActiveRecord::Base
   #   end
   # end
   
+
+  warn_unless_size 4
 end

data/lib/earth/hospitality/commercial_building_energy_consumption_survey_response/data_miner.rb

@@ -51,7 +51,7 @@ CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
       store 'heating_degree_days', :field_name => 'HDD658', :from_units => :degrees_fahrenheit, :to_units => :degrees_celsius
       store 'cooling_degree_days', :field_name => 'CDD658', :from_units => :degrees_fahrenheit, :to_units => :degrees_celsius
       store 'electricity_use',     :synthesize => proc { |row| row['ELCNS8'].to_i }, :units => :kilowatt_hours
-      store 'electricity_energy',  :field_name => 'ELBTU8', :from_units => :kbtus, :to_units => :megajoules
+      store 'electricity_energy',  :synthesize => proc { |row| row['ELBTU8'].to_i.kbtus.to(:megajoules) }, :units => :megajoules
     end
     
     import 'fuel use characteristics from the 2003 EIA CBECS',
@@ -59,12 +59,12 @@ CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
            :skip => 1,
            :headers => ["PUBID8", "REGION8", "CENDIV8", "SQFT8", "SQFTC8", "YRCONC8", "PBA8", "ELUSED8", "NGUSED8", "FKUSED8", "PRUSED8", "STUSED8", "HWUSED8", "ADJWT8", "STRATUM8", "PAIR8", "NGCNS8", "NGBTU8", "NGEXP8", "ZNGCNS8", "ZNGEXP8", "FKCNS8", "FKBTU8", "FKEXP8", "ZFKCNS8", "ZFKEXP8", "DHUSED8", "DHHT18", "DHHT28", "DHCOOL8", "DHWATR8", "DHCOOK8", "DHMANU8", "DHOTH8", "DHCNS8", "DHBTU8", "DHEXP8", "ZDHCNS8", "ZDHEXP8"] do
       key 'id', :field_name => 'PUBID8'
-      store 'natural_gas_use',      :field_name => 'NGCNS8', :from_units => :hundred_cubic_feet, :to_units => :cubic_metres
-      store 'natural_gas_energy',   :field_name => 'NGBTU8', :from_units => :kbtus,              :to_units => :megajoules
-      store 'fuel_oil_use',         :field_name => 'FKCNS8', :from_units => :gallons,            :to_units => :litres
-      store 'fuel_oil_energy',      :field_name => 'FKBTU8', :from_units => :kbtus,              :to_units => :megajoules
-      store 'district_heat_use',    :field_name => 'DHBTU8', :from_units => :kbtus,              :to_units => :megajoules
-      store 'district_heat_energy', :field_name => 'DHBTU8', :from_units => :kbtus,              :to_units => :megajoules
+      store 'natural_gas_use',      :synthesize => proc { |row| row['NGCNS8'].to_i.hundred_cubic_feet.to(:cubic_metres) }, :units => :cubic_metres
+      store 'fuel_oil_use',         :synthesize => proc { |row| row['FKCNS8'].to_i.gallons.to(:litres) }, :units => :litres
+      store 'district_heat_use',    :synthesize => proc { |row| row['DHBTU8'].to_i.kbtus.to(:megajoules) }, :units => :megajoules
+      store 'natural_gas_energy',   :synthesize => proc { |row| row['NGBTU8'].to_i.kbtus.to(:megajoules) }, :units => :megajoules
+      store 'fuel_oil_energy',      :synthesize => proc { |row| row['FKBTU8'].to_i.kbtus.to(:megajoules) }, :units => :megajoules
+      store 'district_heat_energy', :synthesize => proc { |row| row['DHBTU8'].to_i.kbtus.to(:megajoules) }, :units => :megajoules
     end
     
     process "Derive room nights for lodging records" do

data/lib/earth/hospitality/lodging_class.rb

@@ -3,4 +3,6 @@ class LodgingClass < ActiveRecord::Base
   self.primary_key = "name"
   
   col :name
+
+  warn_unless_size 3
 end

data/lib/earth/industry/industry.rb

@@ -25,4 +25,6 @@ class Industry < ActiveRecord::Base
     prefix = naics_code.to_s[0,2]
     %w{42 44 45}.include?(prefix)
   end
+
+  warn_unless_size 2341
 end

data/lib/earth/industry/mecs_energy/data_miner.rb

@@ -28,7 +28,7 @@ MecsEnergy.class_eval do
   data_miner do
     MecsEnergy::CENSUS_REGIONS.each do |region, data|
       import("MECS table 3.2 #{region}",
-        :url => "http://205.254.135.24/emeu/mecs/mecs2006/excel/Table3_2.xls",
+        :url => "http://www.eia.gov/emeu/mecs/mecs2006/excel/Table3_2.xls",
         :crop => data[:crop],
         :headers => ["NAICS Code", "Subsector and Industry", "Total", "BLANK", "Net Electricity", "BLANK", "Residual Fuel Oil", "Distillate Fuel Oil", "Natural Gas", "BLANK", "LPG and NGL", "BLANK", "Coal", "Coke and Breeze", "Other"]) do
         key :name, :synthesize => proc { |row| "#{Industry.format_naics_code(row['NAICS Code'])}-#{data[:code]}" }

data/lib/earth/industry/mecs_ratio/data_miner.rb

@@ -28,7 +28,7 @@ MecsRatio.class_eval do
   data_miner do
     MecsRatio::CENSUS_REGIONS.each do |region, data|
       import( "MECS table 6.1 #{region}",
-        :url => "http://205.254.135.24/emeu/mecs/mecs2006/excel/Table6_1.xls",
+        :url => "http://www.eia.gov/emeu/mecs/mecs2006/excel/Table6_1.xls",
         :crop => data[:crop],
         :headers => ["NAICS Code", "Subsector and Industry", "Consumption per Employee", "Consumption per Dollar of Value Added", "Consumption per Dollar of Value of Shipments"]) do
         key :name, :synthesize => proc { |row| "#{Industry.format_naics_code(row["NAICS Code"])}-#{data[:code]}" }

data/lib/earth/locality/census_division.rb

@@ -26,4 +26,6 @@ class CensusDivision < ActiveRecord::Base
   col :meeting_building_electricity_intensity_units
   col :meeting_building_district_heat_intensity, :type => :float
   col :meeting_building_district_heat_intensity_units
+
+  warn_unless_size 9
 end