earth 0.11.20 → 0.12.0

data/lib/earth/automobile/automobile_fuel/data_miner.rb

@@ -1,100 +1,94 @@
 require 'earth/fuel/data_miner'
+
 AutomobileFuel.class_eval do
   data_miner do
-    import "a list of pure automobile fuels",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdE9xTEdueFM2R0diNTgxUlk1QXFSb2c&gid=0&output=csv' do
-      key   'name'
-      store 'code'
-      store 'base_fuel_name'
-      store 'distance_key'
-      store 'ef_key'
+    process "Start from scratch" do
+      delete_all
     end
     
-    import "a list of blended automobile fuels",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEswNGIxM0U4U0N1UUppdWw2ejJEX0E&gid=0&output=csv' do
+    import "a Brighter Planet-curated list of automobile fuels",
+           :url => "file://#{Earth::DATA_DIR}/automobile/auto_fuel_data.csv" do
       key   'name'
       store 'code'
-      store 'base_fuel_name'
-      store 'blend_fuel_name'
-      store 'blend_portion'
+      store 'common_name'
       store 'distance_key'
-      store 'ef_key'
+      store 'base_fuel_name', :nullify => true
+      store 'blend_fuel_name', :nullify => true
+      store 'blend_portion', :nullify => true
     end
     
-    process "Ensure AutomobileTypeFuelYearAge, AutomobileTypeYear, Fuel, and GreenhouseGas are populated" do
-      AutomobileTypeFuelYearAge.run_data_miner!
-      AutomobileTypeYear.run_data_miner!
+    process "Ensure Fuel is populated" do
       Fuel.run_data_miner!
-      GreenhouseGas.run_data_miner!
     end
     
-    process "Derive annual distance" do
-      find_each do |record|
-        scope = record.type_fuel_year_ages.where(:year => record.type_fuel_year_ages.maximum('year'))
-        record.annual_distance = scope.weighted_average(:annual_distance, :weighted_by => :vehicles)
-        record.annual_distance_units = scope.first.annual_distance_units
-        record.save!
+    process "Derive energy content and co2 emission factors from Fuel" do
+      elec = AutomobileFuel.find 'electricity'
+      elec.energy_content = 1.kilowatt_hours.to(:megajoules)
+      elec.energy_content_units = 'megajoules_per_kilowatt_hour'
+      elec.save!
+      
+      where("name != 'electricity'").each do |auto_fuel|
+        %w{ energy_content co2_emission_factor co2_biogenic_emission_factor }.each do |method|
+          value = if auto_fuel.blend_portion
+            (auto_fuel.blend_portion * auto_fuel.blend_fuel.send(method)) + ((1 - auto_fuel.blend_portion) * auto_fuel.base_fuel.send(method))
+          else
+            auto_fuel.base_fuel.send method
+          end
+          auto_fuel.send("#{method}=", value)
+          auto_fuel.send("#{method}_units=", auto_fuel.base_fuel.send("#{method}_units"))
+        end
+        auto_fuel.save!
       end
     end
     
-    process "Derive energy content" do
-      find_each do |record|
-        if record.blend_fuel.present?
-          record.energy_content = (record.base_fuel.energy_content * (1 - record.blend_portion)) + (record.blend_fuel.energy_content * record.blend_portion)
-        else
-          record.energy_content = record.base_fuel.energy_content
-        end
-        record.energy_content_units = record.base_fuel.energy_content_units
-        record.save!
-      end
+    import "Alternative fuel ch4 and n2o emission factors derived from the EPA GHG inventory",
+           :url => "file://#{Earth::DATA_DIR}/automobile/auto_fuel_efs.csv" do
+      key 'name'
+      store 'ch4_emission_factor', :from_units => :grams_per_mile, :to_units => :kilograms_per_kilometre
+      store 'n2o_emission_factor', :from_units => :grams_per_mile, :to_units => :kilograms_per_kilometre
+    end
+    
+    process "Ensure AutomobileTypeFuel is populated" do
+      AutomobileTypeFuel.run_data_miner!
     end
     
-    process "Derive co2 emission factor and co2 biogenic emission factors" do
+    process "Derive annual distance, emission factors, and total consumption from AutomobileTypeFuel" do
       find_each do |record|
-        if record.blend_fuel.present?
-          record.co2_emission_factor = (record.base_fuel.co2_emission_factor.to_f * (1 - record.blend_portion)) + (record.blend_fuel.co2_emission_factor.to_f * record.blend_portion)
-          record.co2_biogenic_emission_factor = (record.base_fuel.co2_biogenic_emission_factor.to_f * (1 - record.blend_portion)) + (record.blend_fuel.co2_biogenic_emission_factor.to_f * record.blend_portion)
-        else
-          record.co2_emission_factor = record.base_fuel.co2_emission_factor
-          record.co2_biogenic_emission_factor = record.base_fuel.co2_biogenic_emission_factor
+        if (type_fuels = record.type_fuels).any?
+          %w{ annual_distance ch4_emission_factor n2o_emission_factor }.each do |item|
+            record.send("#{item}=", type_fuels.weighted_average(item, :weighted_by => :vehicles))
+            record.send("#{item}_units=", type_fuels.first.send("#{item}_units"))
+          end
+          unless record.name =~ / gasoline/
+            record.total_consumption = record.type_fuels.sum(:fuel_consumption)
+            record.total_consumption_units = record.type_fuels.first.fuel_consumption_units
+          end
+          record.save!
         end
-        record.co2_emission_factor_units = record.base_fuel.co2_emission_factor_units
-        record.co2_biogenic_emission_factor_units = record.base_fuel.co2_biogenic_emission_factor_units
-        record.save!
       end
     end
     
-    process "Derive ch4, n2o, and hfc emission factors" do
-      find_each do |record|
-        scope = record.type_fuel_years.where(:year => record.type_fuel_years.maximum('year'))
-        
-        record.ch4_emission_factor = scope.weighted_average(:ch4_emission_factor, :weighted_by => :total_travel).to_f * GreenhouseGas[:ch4].global_warming_potential
-        ch4_prefix = scope.first.ch4_emission_factor_units.split("_per_")[0]
-        ch4_suffix = scope.first.ch4_emission_factor_units.split("_per_")[1]
-        record.ch4_emission_factor_units = ch4_prefix + "_co2e_per_" + ch4_suffix
-        
-        record.n2o_emission_factor = scope.weighted_average(:n2o_emission_factor, :weighted_by => :total_travel).to_f * GreenhouseGas[:n2o].global_warming_potential
-        n2o_prefix = scope.first.n2o_emission_factor_units.split("_per_")[0]
-        n2o_suffix = scope.first.n2o_emission_factor_units.split("_per_")[1]
-        record.n2o_emission_factor_units = n2o_prefix + "_co2e_per_" + n2o_suffix
-        
-        record.hfc_emission_factor = scope.map do |tfy|
-          tfy.total_travel.to_f * tfy.type_year.hfc_emission_factor
-        end.sum / scope.sum('total_travel')
-        record.hfc_emission_factor_units = scope.first.type_year.hfc_emission_factor_units
-        
-        record.save!
+    process "Derive annual distance for alternative fuels" do
+      %w{ diesel gasoline }.each do |key|
+        reference_fuel = find(key)
+        where(:annual_distance => nil, :distance_key => key).update_all %{
+          annual_distance = #{reference_fuel.annual_distance},
+          annual_distance_units = '#{reference_fuel.annual_distance_units}'
+        }
       end
     end
     
-    # FIXME TODO DEPRECATED motorcycle needs this
-    process "Derive emission factor" do
-      update_all(%{
-        emission_factor = 1.0 * co2_emission_factor + ch4_emission_factor + n2o_emission_factor + hfc_emission_factor,
-        emission_factor_units = 'kilograms_co2e_per_litre'
-      })
+    process "Ensure GreenhouseGas is populated" do
+      GreenhouseGas.run_data_miner!
     end
     
-    # FIXME TODO verify code somehow
+    process "Convert emission factors to co2e" do
+      %w{ ch4 n2o }.each do |gas|
+        where("#{gas}_emission_factor_units = 'kilograms_per_kilometre'").update_all %{
+          #{gas}_emission_factor = #{gas}_emission_factor * #{GreenhouseGas[gas].global_warming_potential},
+          #{gas}_emission_factor_units = 'kilograms_co2e_per_kilometre'
+        }
+      end
+    end
   end
 end

data/lib/earth/automobile/automobile_make.rb

@@ -1,8 +1,12 @@
-require 'earth/fuel'
 class AutomobileMake < ActiveRecord::Base
   self.primary_key = "name"
   
+  # for calculating fuel efficiency
+  has_many :make_years, :foreign_key => :make_name, :primary_key => :name, :class_name => 'AutomobileMakeYear'
+  
   col :name
   col :fuel_efficiency, :type => :float
   col :fuel_efficiency_units
+  
+  warn_unless_size 81
 end

data/lib/earth/automobile/automobile_make/data_miner.rb

@@ -1,54 +1,27 @@
-require 'earth/fuel/data_miner'
 AutomobileMake.class_eval do
   data_miner do
     process "Start from scratch" do
       delete_all
     end
     
-    process "Ensure AutomobileMakeModelYearVariant and AutomobileMakeYearFleet are populated" do
-      AutomobileMakeModelYearVariant.run_data_miner!
-      AutomobileMakeYearFleet.run_data_miner!
+    process "Ensure AutomobileMakeYear is populated" do
+      AutomobileMakeYear.run_data_miner!
     end
     
-    process "Derive manufacturer names from automobile make model year variants" do
+    process "Derive manufacturer names from AutomobileMakeYear" do
       ::Earth::Utils.insert_ignore(
-        :src => AutomobileMakeModelYearVariant,
+        :src => AutomobileMakeYear,
         :dest => AutomobileMake,
         :cols => { :make_name => :name }
       )
     end
     
-    # sabshere 1/31/11 add Avanti, DaimlerChrysler, IHC, Tesla, etc.
-    process "Derive extra manufacturer names from CAFE data" do
-      ::Earth::Utils.insert_ignore(
-        :src => AutomobileMakeYearFleet,
-        :dest => AutomobileMake,
-        :cols => { :make_name => :name }
-      )
-    end
-    
-    # FIXME TODO derive units here
-    process "Calculate fuel efficiency from CAFE data" do
-      makes = arel_table
-      year_fleets = AutomobileMakeYearFleet.arel_table
-      conditional_relation = makes[:name].eq(year_fleets[:make_name])
-      relation = AutomobileMakeYearFleet.weighted_average_relation(:fuel_efficiency, :weighted_by => :volume).where(conditional_relation)
-      update_all(%{
-        fuel_efficiency = (#{relation.to_sql}),
-        fuel_efficiency_units = 'kilometres_per_litre'
-      })
-    end
-    
-    # FIXME TODO derive units here
-    process "Calculate any missing fuel effeciencies from automobile make model year variants" do
-      makes = arel_table
-      variants = AutomobileMakeModelYearVariant.arel_table
-      conditional_relation = variants[:make_name].eq(makes[:name])
-      relation = variants.project(variants[:fuel_efficiency].average).where(conditional_relation)
-      where(:fuel_efficiency => nil).update_all(%{
-        fuel_efficiency = (#{relation.to_sql}),
-        fuel_efficiency_units = 'kilometres_per_litre'
-      })
+    process "Derive fuel efficiency from AutomobileMakeYear" do
+      find_each do |make|
+        make.fuel_efficiency = make.make_years.weighted_average(:fuel_efficiency)
+        make.fuel_efficiency_units = make.make_years.first.fuel_efficiency_units
+        make.save!
+      end
     end
   end
 end

data/lib/earth/automobile/automobile_make_model.rb

@@ -1,12 +1,31 @@
-require 'earth/fuel'
 class AutomobileMakeModel < ActiveRecord::Base
   self.primary_key = "name"
   
-  col :name # make + model
+  # Used by Automobile and AutomobileTrip to look up auto fuel and alt auto fuel
+  belongs_to :automobile_fuel, :foreign_key => :fuel_code, :primary_key => :code
+  belongs_to :alt_automobile_fuel, :foreign_key => :alt_fuel_code, :primary_key => :code, :class_name => 'AutomobileFuel'
+  
+  # for deriving fuel codes and type name
+  def model_years
+    AutomobileMakeModelYear.where(:make_name => make_name, :model_name => model_name)
+  end
+  
+  col :name
   col :make_name
-  col :model_name # model only
+  col :model_name
+  col :fuel_code
   col :fuel_efficiency_city, :type => :float
   col :fuel_efficiency_city_units
   col :fuel_efficiency_highway, :type => :float
   col :fuel_efficiency_highway_units
+  col :alt_fuel_code
+  col :alt_fuel_efficiency_city, :type => :float
+  col :alt_fuel_efficiency_city_units
+  col :alt_fuel_efficiency_highway, :type => :float
+  col :alt_fuel_efficiency_highway_units
+  col :type_name
+  
+  warn_unless_size 2353
+  warn_if_nulls_except :alt_fuel_code
+  warn_if_nulls /alt_fuel_efficiency/, :conditions => 'alt_fuel_code IS NOT NULL'
 end

data/lib/earth/automobile/automobile_make_model/data_miner.rb

@@ -5,13 +5,13 @@ AutomobileMakeModel.class_eval do
       delete_all
     end
     
-    process "Ensure AutomobileMakeModelYearVariant is populated" do
-      AutomobileMakeModelYearVariant.run_data_miner!
+    process "Ensure AutomobileMakeModelYear is populated" do
+      AutomobileMakeModelYear.run_data_miner!
     end
     
-    process "Derive model names from automobile make model year variants" do
+    process "Derive model names from AutomobileMakeModelYear" do
       ::Earth::Utils.insert_ignore(
-        :src => AutomobileMakeModelYearVariant,
+        :src => AutomobileMakeModelYear,
         :dest => AutomobileMakeModel,
         :cols => {
           [:make_name, :model_name] => :name,
@@ -21,18 +21,44 @@ AutomobileMakeModel.class_eval do
       )
     end
     
-    # FIXME TODO not weighted until we get weightings on auto variants
-    process "Derive average fuel economy from automobile make model year variants" do
+    process "Derive fuel codes and type name from AutomobileMakeModelYear" do
+      find_each do |amm|
+        codes = amm.model_years.map(&:fuel_code).uniq
+        if codes.count == 1
+          amm.fuel_code = codes.first
+        elsif codes.all?{ |code| ['R', 'P', 'G'].include? code }
+          amm.fuel_code = 'G'
+        end
+        
+        alt_codes = amm.model_years.map(&:alt_fuel_code).uniq
+        amm.alt_fuel_code = (alt_codes.first if alt_codes.count == 1)
+        
+        type_names = amm.model_years.map(&:type_name).uniq
+        amm.type_name = (type_names.one? ? type_names.first : nil)
+        
+        amm.save!
+      end
+    end
+    
+    process "Derive fuel efficiencies from AutomobileMakeModelYear" do
       models = arel_table
-      variants = AutomobileMakeModelYearVariant.arel_table
-      conditional_relation = models[:make_name].eq(variants[:make_name]).and(models[:model_name].eq(variants[:model_name]))
-      %w{ city highway }.each do |type|
-        null_check = variants[:"fuel_efficiency_#{type}"].not_eq(nil)
-        relation = variants.project(variants[:"fuel_efficiency_#{type}"].average).where(conditional_relation).where(null_check)
-        update_all(%{
-          fuel_efficiency_#{type} = (#{relation.to_sql}),
-          fuel_efficiency_#{type}_units = 'kilometres_per_litre'
-        })
+      model_years = AutomobileMakeModelYear.arel_table
+      join_relation = model_years[:make_name].eq(models[:make_name]).and(model_years[:model_name].eq(models[:model_name]))
+      
+      %w{ fuel_efficiency_city fuel_efficiency_highway }.each do |fe|
+        fe_sql = AutomobileMakeModelYear.where(join_relation).weighted_average_relation("#{fe}").to_sql
+        update_all %{
+          #{fe} = (#{fe_sql}),
+          #{fe}_units = 'kilometres_per_litre'
+        }
+      end
+      
+      %w{ alt_fuel_efficiency_city alt_fuel_efficiency_highway }.each do |fe|
+        fe_sql = AutomobileMakeModelYear.where(join_relation).weighted_average_relation("#{fe}").to_sql
+        where("alt_fuel_code IS NOT NULL").update_all %{
+          #{fe} = (#{fe_sql}),
+          #{fe}_units = 'kilometres_per_litre'
+        }
       end
     end
   end

data/lib/earth/automobile/automobile_make_model_year.rb

@@ -1,18 +1,29 @@
-require 'earth/fuel'
 class AutomobileMakeModelYear < ActiveRecord::Base
   self.primary_key = "name"
   
-  # Need this so Automobile and AutomobileTrip can do characteristics[:make_model_year].automobile_fuel
-  belongs_to :automobile_fuel, :foreign_key => 'fuel_code', :primary_key => 'code'
+  # Used by Automobile and AutomobileTrip to look up auto fuel and alt auto fuel
+  belongs_to :automobile_fuel,     :foreign_key => 'fuel_code',     :primary_key => 'code'
+  belongs_to :alt_automobile_fuel, :foreign_key => 'alt_fuel_code', :primary_key => 'code', :class_name => 'AutomobileFuel'
   
-  col :name # make + model + year
+  col :name
   col :make_name
   col :model_name
-  col :year,                    :type => :integer
+  col :year, :type => :integer
+  col :hybridity,:type => :boolean
   col :fuel_code
-  col :hybridity,               :type => :boolean
-  col :fuel_efficiency_city,    :type => :float
+  col :fuel_efficiency_city, :type => :float
   col :fuel_efficiency_city_units
   col :fuel_efficiency_highway, :type => :float
   col :fuel_efficiency_highway_units
+  col :alt_fuel_code
+  col :alt_fuel_efficiency_city, :type => :float
+  col :alt_fuel_efficiency_city_units
+  col :alt_fuel_efficiency_highway, :type => :float
+  col :alt_fuel_efficiency_highway_units
+  col :type_name # whether the vehicle is a passenger car or light-duty truck
+  col :weighting, :type => :float # for calculating AutomobileMakeModel fuel efficiencies
+  
+  warn_unless_size 10997
+  warn_if_nulls_except :alt_fuel_code
+  warn_if_nulls /alt_fuel_efficiency/, :conditions => 'alt_fuel_code IS NOT NULL'
 end

data/lib/earth/automobile/automobile_make_model_year/data_miner.rb

@@ -1,4 +1,3 @@
-require 'earth/fuel/data_miner'
 AutomobileMakeModelYear.class_eval do
   data_miner do
     process "Start from scratch" do
@@ -9,7 +8,7 @@ AutomobileMakeModelYear.class_eval do
       AutomobileMakeModelYearVariant.run_data_miner!
     end
     
-    process "Derive model year names from automobile make model year variants" do
+    process "Derive names AutomobileMakeModelYearVariant" do
       ::Earth::Utils.insert_ignore(
         :src => AutomobileMakeModelYearVariant,
         :dest => AutomobileMakeModelYear,
@@ -22,40 +21,57 @@ AutomobileMakeModelYear.class_eval do
       )
     end
     
-    process "Derive fuel from automobile make model year variants" do
-      model_years = arel_table
-      variants = AutomobileMakeModelYearVariant.arel_table
-      conditional_relation = variants[:make_name].eq(model_years[:make_name]).and(variants[:model_name].eq(model_years[:model_name])).and(variants[:year].eq(model_years[:year]))
-      where("(#{variants.project('COUNT(DISTINCT fuel_code)').where(conditional_relation).to_sql}) = 1").update_all "fuel_code = (#{variants.project('DISTINCT fuel_code').where(conditional_relation).to_sql})"
-    end
-    
-    # FIXME TODO not in automobile_make_model_year_variants because they're missing from the EPA FEG download files:
-    # 2005 Honda Accord Hybrid
-    # 2006 Honda Accord Hybrid
-    # 2009 Chrysler Aspen HEV
-    # 2009 Dodge Durango HEV
-    import "A list of hybrid make model years derived from the EPA fuel economy guide",
-           :url => 'https://docs.google.com/spreadsheet/pub?hl=en_US&hl=en_US&key=0AoQJbWqPrREqdGtzekE4cGNoRGVmdmZMaTNvOWluSnc&output=csv' do
-      key 'name', :synthesize => proc { |record| [record['make_name'], record['model_name'], record['year']].join(' ') }
-      store 'hybridity'
+    process "Identify hybrid vehicles" do
+      where("model_name LIKE '%HYBRID%'").update_all :hybridity => true
+      where("make_name = 'Infiniti' AND model_name REGEXP '[A-Z][0-9]{2}H'").update_all :hybridity => true
+      where("make_name = 'Lexus' AND model_name REGEXP '[A-Z]{2}[0-9]{3}H'").update_all :hybridity => true
+      where(:make_name => 'Chevrolet', :model_name => 'VOLT').update_all :hybridity => true
+      where(:make_name => 'Honda', :model_name => 'INSIGHT').update_all :hybridity => true
+      where(:make_name => 'Honda', :model_name => 'CR-Z').update_all :hybridity => true
+      where("make_name = 'Toyota' AND model_name LIKE 'PRIUS%'").update_all :hybridity => true
+      where(:hybridity => nil).update_all :hybridity => false
     end
     
-    process "Set hybridity to FALSE for all other make model years" do
-      where(:hybridity => nil).update_all :hybridity => false
+    process "Derive fuel codes from AutomobileMakeModelYearVariant" do
+      model_years = arel_table
+      variants = AutomobileMakeModelYearVariant.arel_table
+      join_relation = variants[:make_name].eq(model_years[:make_name]).and(variants[:model_name].eq(model_years[:model_name])).and(variants[:year].eq(model_years[:year]))
+      
+      update_all "fuel_code = (#{variants.project('GROUP_CONCAT(DISTINCT fuel_code)').where(join_relation).to_sql})"
+      update_all "alt_fuel_code = (#{variants.project('GROUP_CONCAT(DISTINCT alt_fuel_code)').where(join_relation).to_sql})"
+      where(:fuel_code => ['R,P', 'P,R']).update_all "fuel_code = 'G'"
     end
     
     # FIXME TODO: weight by volume somehow
-    process "Calculate city and highway fuel efficiency from automobile make model year variants" do
+    process "Calculate city and highway fuel efficiency from AutomobileMakeModelYearVariants" do
       model_years = arel_table
       variants = AutomobileMakeModelYearVariant.arel_table
-      conditional_relation = model_years[:make_name].eq(variants[:make_name]).and(model_years[:model_name].eq(variants[:model_name])).and(model_years[:year].eq(variants[:year]))
-      %w{ city highway }.each do |type|
-        null_check = variants[:"fuel_efficiency_#{type}"].not_eq(nil)
-        relation = variants.project(variants[:"fuel_efficiency_#{type}"].average).where(conditional_relation).where(null_check)
+      
+      join_relation = model_years[:make_name].eq(variants[:make_name]).and(model_years[:model_name].eq(variants[:model_name])).and(model_years[:year].eq(variants[:year]))
+      %w{ fuel_efficiency_city fuel_efficiency_highway }.each do |fe|
         update_all(%{
-          fuel_efficiency_#{type} = (#{relation.to_sql}),
-          fuel_efficiency_#{type}_units = 'kilometres_per_litre'
+          #{fe} = (#{variants.project(variants["#{fe}"].average).where(join_relation).to_sql}),
+          #{fe}_units = 'kilometres_per_litre'
         })
+        where("alt_fuel_code IS NOT NULL").update_all(%{
+          alt_#{fe} = (#{variants.project(variants["alt_#{fe}"].average).where(join_relation).to_sql}),
+          alt_#{fe}_units = 'kilometres_per_litre'
+        })
+      end
+    end
+    
+    process "Derive type name from AutomobileMakeModelYearVariant" do
+      find_each do |ammy|
+        type_names = AutomobileMakeModelYearVariant.where(:make_name => ammy.make_name, :model_name => ammy.model_name, :year => ammy.year).map(&:type_name).uniq
+        ammy.type_name = (type_names.one? ? type_names.first : nil)
+        ammy.save!
+      end
+    end
+    
+    # For calculating AutomobileMakeModel fuel efficiencies
+    process "Derive weighting from AutomobileYear" do
+      connection.select_values("SELECT DISTINCT year FROM #{quoted_table_name}").each do |year|
+        where(:year => year).update_all "weighting = #{AutomobileYear.weighting(year)}"
       end
     end
   end