earth 0.11.20 → 0.12.0

data/lib/earth/automobile/automobile_model/data_miner.rb

@@ -1,4 +1,3 @@
-require 'earth/fuel/data_miner'
 AutomobileModel.class_eval do
   data_miner do
     process "Start from scratch" do
@@ -9,7 +8,7 @@ AutomobileModel.class_eval do
       AutomobileMakeModelYearVariant.run_data_miner!
     end
     
-    process "Derive model names from automobile make model year variants" do
+    process "Derive model names from AutomobileMakeModelYearVariant" do
       ::Earth::Utils.insert_ignore(
         :src => AutomobileMakeModelYearVariant,
         :dest => AutomobileModel,

data/lib/earth/automobile/automobile_size_class.rb

@@ -1,4 +1,3 @@
-require 'earth/fuel'
 class AutomobileSizeClass < ActiveRecord::Base
   self.primary_key = "name"
   
@@ -10,8 +9,6 @@ class AutomobileSizeClass < ActiveRecord::Base
   
   col :name
   col :type_name
-  col :annual_distance, :type => :float
-  col :annual_distance_units
   col :fuel_efficiency_city, :type => :float
   col :fuel_efficiency_city_units
   col :fuel_efficiency_highway, :type => :float
@@ -21,69 +18,11 @@ class AutomobileSizeClass < ActiveRecord::Base
   col :conventional_fuel_efficiency_city_multiplier, :type => :float
   col :conventional_fuel_efficiency_highway_multiplier, :type => :float
   
-  # FIXME TODO verify type_name?
-  
-  # verify "Annual distance should be greater than zero" do
-  #   AutomobileSizeClass.all.each do |size_class|
-  #     unless size_class.annual_distance > 0
-  #       raise "Invalid annual distance for AutomobileSizeClass #{size_class.name}: #{size_class.annual_distance} (should be > 0)"
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Annual distance units should be kilometres" do
-  #   AutomobileSizeClass.all.each do |size_class|
-  #     unless size_class.annual_distance_units == "kilometres"
-  #       raise "Invalid annual distance units for AutomobileSizeClass #{size_class.name}: #{size_class.annual_distance_units} (should be kilometres)"
-  #     end
-  #   end
-  # end
-  # 
-  #  verify "Fuel efficiencies should be greater than zero" do
-  #   AutomobileSizeClass.all.each do |size_class|
-  #     %w{ city highway }.each do |type|
-  #       fuel_efficiency = size_class.send(:"fuel_efficiency_#{type}")
-  #       unless fuel_efficiency > 0
-  #         raise "Invalid fuel efficiency #{type} for AutomobileSizeClass #{size_class.name}: #{fuel_efficiency} (should be > 0)"
-  #       end
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Fuel efficiency units should be kilometres per litre" do
-  #   AutomobileSizeClass.all.each do |size_class|
-  #     %w{ city highway }.each do |type|
-  #       units = size_class.send(:"fuel_efficiency_#{type}_units")
-  #       unless units == "kilometres_per_litre"
-  #         raise "Invalid fuel efficiency #{type} units for AutomobileSizeClass #{size_class.name}: #{units} (should be kilometres_per_litre)"
-  #       end
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Any fuel efficiency multipliers should be greater than zero" do
-  #   AutomobileSizeClass.all.each do |size_class|
-  #     %w{ hybrid conventional }.each do |hybridity|
-  #       %w{ city highway }.each do |type|
-  #         multiplier = size_class.send(:"#{hybridity}_fuel_efficiency_#{type}_multiplier")
-  #         if multiplier.present?
-  #           unless multiplier > 0
-  #             raise "Invalid #{hybridity} fuel efficiency #{type} multiplier for AutomobileSizeClass #{size_class.name}: #{multiplier} (should be > 0)"
-  #           end
-  #         end
-  #       end
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Fallback fuel efficiency multipliers should be greater than zero" do
-  #   %w{ hybrid conventional }.each do |hybridity|
-  #     %w{ city highway }.each do |type|
-  #       multiplier = AutomobileSizeClass.fallback.send(:"#{hybridity}_fuel_efficiency_#{type}_multiplier")
-  #       unless multiplier > 0
-  #         raise "Invalid AutomobileSizeClass fallback #{hybridity} fuel efficiency #{type} multiplier: #{multiplier} (should be > 0)"
-  #       end
-  #     end
-  #   end
-  # end
+  warn_unless_size 15
+  warn_if_nulls_except(
+    :hybrid_fuel_efficiency_city_multiplier,
+    :hybrid_fuel_efficiency_highway_multiplier,
+    :conventional_fuel_efficiency_city_multiplier,
+    :conventional_fuel_efficiency_highway_multiplier
+  )
 end

data/lib/earth/automobile/automobile_size_class/data_miner.rb

@@ -1,61 +1,26 @@
-require 'earth/fuel/data_miner'
 AutomobileSizeClass.class_eval do
   data_miner do
-    import "a list of size classes and pre-calculated fuel efficiencies",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHlRUE5IcWlrRENhN0EtUldPTy1rX1E&gid=0&output=csv' do
-      key 'name'
-      store 'type_name'
-      store 'fuel_efficiency_city', :units_field_name => 'fuel_efficiency_city_units'
-      store 'fuel_efficiency_highway', :units_field_name => 'fuel_efficiency_highway_units'
+    process "Start from scratch" do
+      delete_all
     end
     
-    # FIXME TODO derive everything from AutomobileSizeClassYear and AutomobileTypeFuelYearAge
-    # process "Start from scratch" do
-    #   delete_all
-    # end
-    # 
-    # process "Ensure AutomobileSizeClassYear is populated" do
-    #   AutomobileSizeClassYear.run_data_miner!
-    #   AutomobileTypeFuelYearAge.run_data_miner!
-    # end
-    # 
-    # process "Derive size classes from AutomobileSizeClassYear" do
-    #   ::Earth::Utils.insert_ignore(
-    #     :src => AutomobileSizeClassYear,
-    #     :dest => AutomobileSizeClass,
-    #     :cols => {
-    #       :size_class_name => :name,
-    #       :type_name => :type_name,
-    #     }
-    #   )
-    # end
-    # 
-    # process "somehow calculate fuel efficiency from size class year and type fuel year age" do
-    #   
-    # end
-    
-    process "Ensure AutomobileTypeFuelYearAge is populated" do
-      AutomobileTypeFuelYearAge.run_data_miner!
-    end
-    
-    # FIXME TODO make this a method on AutomobileSizeClass?
-    process "Calculate annual distance from AutomobileTypeFuelYearAge" do
-      classes = arel_table
-      ages = AutomobileTypeFuelYearAge.arel_table
-      conditional_relation = ages[:type_name].eq(classes[:type_name])
-      update_all(%{
-        annual_distance = (#{AutomobileTypeFuelYearAge.weighted_average_relation(:annual_distance, :weighted_by => :vehicles).where(conditional_relation).to_sql}),
-        annual_distance_units = 'kilometres'
-      })
+    import "pre-calculated size class data derived from the 2010 EPA Fuel Economy Trends report",
+           :url => "file://#{Earth::DATA_DIR}/automobile/sizes.csv" do
+      key   :name
+      store :type_name
+      store :fuel_efficiency_city
+      store :fuel_efficiency_city_units
+      store :fuel_efficiency_highway
+      store :fuel_efficiency_highway_units
     end
     
     import "pre-calculated fuel efficiency multipliers",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGt2NnhXLXUxNFRJSzczU3BkSHB3enc&hl=en&gid=0&output=csv' do
-      key 'name'
-      store 'hybrid_fuel_efficiency_city_multiplier'
-      store 'hybrid_fuel_efficiency_highway_multiplier'
-      store 'conventional_fuel_efficiency_city_multiplier'
-      store 'conventional_fuel_efficiency_highway_multiplier'
+           :url => "file://#{Earth::DATA_DIR}/automobile/hybridity_multipliers.csv" do
+      key :name
+      store :hybrid_fuel_efficiency_city_multiplier
+      store :hybrid_fuel_efficiency_highway_multiplier
+      store :conventional_fuel_efficiency_city_multiplier
+      store :conventional_fuel_efficiency_highway_multiplier
     end
   end
 end

data/lib/earth/automobile/automobile_type_fuel.rb

@@ -0,0 +1,23 @@
+class AutomobileTypeFuel < ActiveRecord::Base
+  self.primary_key = "name"
+  
+  # for calculating vehicles
+  def latest_activity_year_type_fuel
+    AutomobileActivityYearTypeFuel.latest.where(:type_name => type_name, :fuel_common_name => fuel_common_name).first
+  end
+  
+  col :name
+  col :type_name
+  col :fuel_common_name
+  col :annual_distance, :type => :float
+  col :annual_distance_units
+  col :fuel_consumption, :type => :float
+  col :fuel_consumption_units
+  col :ch4_emission_factor, :type => :float
+  col :ch4_emission_factor_units
+  col :n2o_emission_factor, :type => :float
+  col :n2o_emission_factor_units
+  col :vehicles, :type => :float
+  
+  warn_unless_size 4
+end

data/lib/earth/automobile/automobile_type_fuel/data_miner.rb

@@ -0,0 +1,52 @@
+AutomobileTypeFuel.class_eval do
+  data_miner do
+    process "Start from scratch" do
+      delete_all
+    end
+    
+    process "Ensure AutomobileTypeFuelYear is populated" do
+      AutomobileTypeFuelYear.run_data_miner!
+    end
+    
+    process "Derive from AutomobileTypeFuelYear" do
+      ::Earth::Utils.insert_ignore(
+        :src => AutomobileTypeFuelYear,
+        :dest => AutomobileTypeFuel,
+        :cols => {
+          [:type_name, :fuel_common_name] => :name,
+          :type_name => :type_name,
+          :fuel_common_name => :fuel_common_name
+        }
+      )
+    end
+    
+    process "Derive annual distance and emission factors from AutomobileTypeFuelYear" do
+      type_fuels = arel_table
+      type_fuel_years = AutomobileTypeFuelYear.arel_table
+      join_relation = type_fuel_years[:type_name].eq(type_fuels[:type_name]).and(type_fuel_years[:fuel_common_name].eq(type_fuels[:fuel_common_name]))
+      
+      %w{ annual_distance ch4_emission_factor n2o_emission_factor }.each do |item|
+        item_sql = AutomobileTypeFuelYear.where(join_relation).weighted_average_relation(:"#{item}", :weighted_by => :share_of_type).to_sql
+        item_units = AutomobileTypeFuelYear.first.send("#{item}_units")
+        
+        update_all %{
+          #{item} = (#{item_sql}),
+          #{item}_units = '#{item_units}'
+        }
+      end
+    end
+    
+    process "Ensure AutomobileActivityYearTypeFuel is populated" do
+      AutomobileActivityYearTypeFuel.run_data_miner!
+    end
+    
+    process "Derive number of vehicles and fuel consumption from AutomobileActivityYearTypeFuel" do
+      find_each do |atf|
+        atf.vehicles = atf.latest_activity_year_type_fuel.distance / atf.annual_distance
+        atf.fuel_consumption = atf.latest_activity_year_type_fuel.fuel_consumption
+        atf.fuel_consumption_units = atf.latest_activity_year_type_fuel.fuel_consumption_units
+        atf.save!
+      end
+    end
+  end
+end

data/lib/earth/automobile/automobile_type_fuel_control.rb

@@ -1,6 +1,6 @@
-require 'earth/fuel'
 class AutomobileTypeFuelControl < ActiveRecord::Base
   self.primary_key = "name"
+
   col :name
   col :type_name
   col :fuel_common_name
@@ -10,36 +10,5 @@ class AutomobileTypeFuelControl < ActiveRecord::Base
   col :n2o_emission_factor, :type => :float
   col :n2o_emission_factor_units
   
-  # verify "Type name, fuel common name, and control name should never be missing" do
-  #   AutomobileTypeFuelControl.all.each do |record|
-  #     %w{ type_name fuel_common_name control_name }.each do |attribute|
-  #       value = record.send(:"#{attribute}")
-  #       unless value.present?
-  #         raise "Missing #{attribute} for AutomobileTypeFuelControl '#{record.name}'"
-  #       end
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Emission factors should be greater than zero" do
-  #   AutomobileTypeFuelControl.all.each do |record|
-  #     %w{ ch4_emission_factor n2o_emission_factor }.each do |factor|
-  #       value = record.send(:"#{factor}")
-  #       unless value > 0
-  #         raise "Invalid #{factor} for AutomobileTypeFuelControl '#{record.name}': #{valuel} (should be > 0)"
-  #       end
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Emission factor units should be kilograms per kilometre" do
-  #   AutomobileTypeFuelControl.all.each do |record|
-  #     %w{ ch4_emission_factor_units n2o_emission_factor_units }.each do |attribute|
-  #       units = record.send(:"#{attribute}")
-  #       unless units == "kilograms_per_kilometre"
-  #         raise "Invalid #{attribute} for AutomobileTypeFuelControl '#{record.name}': #{units} (should be kilograms_per_kilometre)"
-  #       end
-  #     end
-  #   end
-  # end
+  warn_unless_size 20
 end

data/lib/earth/automobile/automobile_type_fuel_control/data_miner.rb

@@ -1,14 +1,29 @@
-require 'earth/fuel/data_miner'
+require 'earth/fuel/greenhouse_gas'
+require 'earth/fuel/greenhouse_gas/data_miner'
+
 AutomobileTypeFuelControl.class_eval do
   data_miner do
     import "automobile type fuel control data derived from the 2010 EPA GHG Inventory",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEloSTU5YUNOUXRFRUcxWHlTUi1GMkE&hl=en&gid=0&output=csv' do
+           :url => "file://#{Earth::DATA_DIR}/automobile/emission_control_techs.csv" do
       key   'name'
       store 'type_name'
       store 'fuel_common_name'
       store 'control_name'
-      store 'ch4_emission_factor', :units_field_name => 'ch4_emission_factor_units'
-      store 'n2o_emission_factor', :units_field_name => 'n2o_emission_factor_units'
+      store 'ch4_emission_factor', :from_units => :grams_per_mile, :to_units => :kilograms_per_kilometre
+      store 'n2o_emission_factor', :from_units => :grams_per_mile, :to_units => :kilograms_per_kilometre
+    end
+    
+    process "Ensure GreenhouseGas is populated" do
+      GreenhouseGas.run_data_miner!
+    end
+    
+    process "Convert emission factors to co2e" do
+      %w{ ch4 n2o }.each do |gas|
+        where("#{gas}_emission_factor_units = 'kilograms_per_kilometre'").update_all %{
+          #{gas}_emission_factor = #{gas}_emission_factor * #{GreenhouseGas[gas].global_warming_potential},
+          #{gas}_emission_factor_units = 'kilograms_co2e_per_kilometre'
+        }
+      end
     end
   end
 end

data/lib/earth/automobile/automobile_type_fuel_year.rb

@@ -1,68 +1,31 @@
-require 'earth/fuel'
 class AutomobileTypeFuelYear < ActiveRecord::Base
   self.primary_key = "name"
   
-  # FIXME TODO keep this until fix AutomobileFuel import so that it doesn't call type_fuel_year.type_year
-  belongs_to :type_year, :class_name => 'AutomobileTypeYear', :foreign_key => 'type_year_name'
+  # Used by Automobile and AutomobileTrip
+  def self.find_by_type_name_and_fuel_common_name_and_closest_year(type_name, fuel_common_name, year)
+    if year > maximum(:year)
+      where(:type_name => type_name, :fuel_common_name => fuel_common_name, :year => maximum(:year)).first
+    else
+      where(:type_name => type_name, :fuel_common_name => fuel_common_name, :year => [year, minimum(:year)].max).first
+    end
+  end
+  
+  # for calculating ch4 and n2o ef
+  def type_fuel_year_controls
+    AutomobileTypeFuelYearControl.find_all_by_type_name_and_fuel_common_name_and_closest_year(type_name, fuel_common_name, year)
+  end
   
   col :name
   col :type_name
   col :fuel_common_name
   col :year, :type => :integer
-  col :type_year_name
-  col :total_travel, :type => :float
-  col :total_travel_units
-  col :fuel_consumption, :type => :float
-  col :fuel_consumption_units
+  col :share_of_type, :type => :float
+  col :annual_distance, :type => :float
+  col :annual_distance_units
   col :ch4_emission_factor, :type => :float
   col :ch4_emission_factor_units
   col :n2o_emission_factor, :type => :float
   col :n2o_emission_factor_units
   
-  # %w{ type_name fuel_common_name type_year_name }.each do |attribute|
-  #   verify "#{attribute.humanize} should never be missing" do
-  #     AutomobileTypeFuelYear.all.each do |record|
-  #       value = record.send(:"#{attribute}")
-  #       unless value.present?
-  #         raise "Missing #{attribute.humanize.downcase} for AutomobileTypeFuelYear '#{record.name}'"
-  #       end
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Year should be from 1990 to 2008" do
-  #   AutomobileTypeFuelYear.all.each do |record|
-  #     year = record.send(:year)
-  #     unless year > 1989 and year < 2009
-  #       raise "Invalid year for AutomobileTypeFuelYear '#{record.name}': #{year} (should be from 1990 to 2008)"
-  #     end
-  #   end
-  # end
-  # 
-  # %w{ total_travel fuel_consumption ch4_emission_factor n2o_emission_factor }.each do |attribute|
-  #   verify "#{attribute.humanize} should be greater than zero" do
-  #     AutomobileTypeFuelYear.all.each do |record|
-  #       value = record.send(:"#{attribute}")
-  #       unless value > 0
-  #         raise "Invalid #{attribute.humanize.downcase} for AutomobileTypeFuelYear '#{record.name}': #{value} (should be > 0)"
-  #       end
-  #     end
-  #   end
-  # end
-  # 
-  # [["total_travel_units", "kilometres"],
-  #  ["fuel_consumption_units", "litres"],
-  #  ["ch4_emission_factor_units", "kilograms_per_litre"],
-  #  ["n2o_emission_factor_units", "kilograms_per_litre"]].each do |pair|
-  #   attribute = pair[0]
-  #   proper_units = pair[1]
-  #   verify "#{attribute.humanize} should be #{proper_units.humanize.downcase}" do
-  #     AutomobileTypeFuelYear.all.each do |record|
-  #       units = record.send(:"#{attribute}")
-  #       unless units == proper_units
-  #         raise "Invalid #{attribute.humanize.downcase} for AutomobileTypeFuelYear '#{record.name}': #{units} (should be #{proper_units})"
-  #       end
-  #     end
-  #   end
-  # end
+  warn_unless_size 124
 end

data/lib/earth/automobile/automobile_type_fuel_year/data_miner.rb

@@ -1,104 +1,50 @@
-require 'earth/fuel/data_miner'
 AutomobileTypeFuelYear.class_eval do
   data_miner do
-    import "total vehicle miles travelled by gasoline passenger cars from the 2010 EPA GHG Inventory",
-           :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
-           :filename => 'Annex Tables/Annex 3/Table A-87.csv',
-           :skip => 1,
-           :select => proc { |row| row['Year'].to_i.to_s == row['Year'] } do
-      key   'name', :synthesize => proc { |row| "Passenger cars gasoline #{row['Year']}" }
-      store 'type_name', :static => 'Passenger cars'
-      store 'fuel_common_name', :static => 'gasoline'
-      store 'year', :field_name => 'Year'
-      store 'total_travel', :field_name => 'Passenger Cars', :units => :billion_miles
-    end
-    
-    import "total vehicle miles travelled by gasoline light-duty trucks from the 2010 EPA GHG Inventory",
-           :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
-           :filename => 'Annex Tables/Annex 3/Table A-87.csv',
-           :skip => 1,
-           :select => proc { |row| row['Year'].to_i.to_s == row['Year'] } do
-      key   'name', :synthesize => proc { |row| "Light-duty trucks gasoline #{row['Year']}" }
-      store 'type_name', :static => 'Light-duty trucks'
-      store 'fuel_common_name', :static => 'gasoline'
-      store 'year', :field_name => 'Year'
-      store 'total_travel', :field_name => 'Light-Duty Trucks', :units => :billion_miles
-    end
-    
-    import "total vehicle miles travelled by diesel passenger cars from the 2010 EPA GHG Inventory",
-           :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
-           :filename => 'Annex Tables/Annex 3/Table A-88.csv',
-           :skip => 1,
-           :select => proc { |row| row['Year'].to_i.to_s == row['Year'] } do
-      key   'name', :synthesize => proc { |row| "Passenger cars diesel #{row['Year']}" }
-      store 'type_name', :static => 'Passenger cars'
-      store 'fuel_common_name', :static => 'diesel'
-      store 'year', :field_name => 'Year'
-      store 'total_travel', :field_name => 'Passenger Cars', :units => :billion_miles
-    end
-    
-    import "total vehicle miles travelled by diesel light-duty trucks from the 2010 EPA GHG Inventory",
-           :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
-           :filename => 'Annex Tables/Annex 3/Table A-88.csv',
-           :skip => 1,
-           :select => proc { |row| row['Year'].to_i.to_s == row['Year'] } do
-      key   'name', :synthesize => proc { |row| "Light-duty trucks diesel #{row['Year']}" }
-      store 'type_name', :static => 'Light-duty trucks'
-      store 'fuel_common_name', :static => 'diesel'
-      store 'year', :field_name => 'Year'
-      store 'total_travel', :field_name => 'Light-Duty Trucks', :units => :billion_miles
-    end
-    
-    process "Convert total travel from billion miles to kilometres" do
-      conversion_factor = 1_000_000_000.miles.to(:kilometres)
-      where(:total_travel_units => 'billion_miles').update_all(%{
-        total_travel = 1.0 * total_travel * #{conversion_factor},
-        total_travel_units = 'kilometres'
-      })
-    end
-    
-    import "fuel consumption derived from the 2010 EPA GHG Inventory",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHBCMFhLRTFTZENsd0dPUGUyYlJna0E&hl=en&gid=0&output=csv' do
-      key 'name'
-      store 'fuel_consumption', :units_field_name => 'fuel_consumption_units'
-    end
-    
-    process "Derive type year name for association with AutomobileTypeYear" do
-      update_all "type_year_name = type_name || ' ' || year"
+    process "Start from scratch" do
+      delete_all
+    end
+    
+    [['gasoline', 'G'], ['diesel', 'D']].each do |fuel, fuel_code|
+      [['Passenger cars', 'V'], ['Light-duty trucks', 'T']].each do |type, type_code|
+        code = "LD" + fuel_code + type_code
+        
+        import "age distribution of #{fuel} #{type.downcase} from the 2011 EPA GHG Inventory",
+               :url => 'http://www.epa.gov/climatechange/emissions/downloads11/Annex%20Tables.zip',
+               :filename => 'Annex Tables/Table A-93.csv',
+               :skip => 1,
+               :headers => %w{ age LDGV LDGT HDGV LDDV LDDT HDDT MC },
+               :select => proc { |row| row['age'].to_i.to_s == row['age'] } do
+          key 'name', :synthesize => proc { |row| "#{type} #{fuel} #{2009 - row['age'].to_i}" }
+          store 'type_name', :static => type
+          store 'fuel_common_name', :static => fuel
+          store 'year', :synthesize => proc { |row| 2009 - row['age'].to_i }
+          store 'share_of_type', :synthesize => proc { |row| row[code].to_f / (code == 'LDDV' ? 57.5 : 100) } # total percent only sums to 57.5 for diesel cars
+        end
+        
+        import "average distance per vehicle for gasoline passenger cars from the 2011 EPA GHG Inventory",
+               :url => 'http://www.epa.gov/climatechange/emissions/downloads11/Annex%20Tables.zip',
+               :filename => 'Annex Tables/Table A-94.csv',
+               :skip => 1,
+               :headers => %w{ age LDGV LDGT HDGV LDDV LDDT HDDT MC },
+               :select => proc { |row| row['age'].to_i.to_s == row['age'] } do
+          key 'name', :synthesize => proc { |row| "#{type} #{fuel} #{2009 - row['age'].to_i}" }
+          store 'annual_distance', :field_name => code, :from_units => :miles, :to_units => :kilometres
+        end
+      end
     end
     
-    process "Ensure AutomobileTypeFuelYearControl and AutomobileTypeFuelControl are populated" do
+    process "Ensure AutomobileTypeFuelYearControl is populated" do
       AutomobileTypeFuelYearControl.run_data_miner!
-      AutomobileTypeFuelControl.run_data_miner!
     end
     
-    process "Calculate CH4 and N2O emision factors from AutomobileTypeFuelYearControl and AutomobileTypeFuelControl" do
-      # FIXME TODO tried to do this with arel but '*' method is not defined
-      # fuel_years = arel_table
-      # year_controls = AutomobileTypeFuelYearControl.arel_table
-      # controls = AutomobileTypeFuelControl.arel_table
-      # year_controls.project((year_controls[:total_travel_percent] * controls[:ch4_emission_factor]).sum)
-      #   .join(controls)
-      #   .on(year_controls[:type_name].eq(controls[:type_name])
-      #   .and(year_controls[:fuel_common_name].eq(controls[:fuel_common_name]))
-      #   .and(year_controls[:control_name].eq(controls[:control_name])))
-      %w{ ch4 n2o }.each do |gas|
-        emission_factor = %{
-          ( SELECT SUM(1.0 * t1.total_travel_percent * t2.#{gas}_emission_factor)
-            FROM #{AutomobileTypeFuelYearControl.quoted_table_name} AS t1
-            INNER JOIN #{AutomobileTypeFuelControl.quoted_table_name} AS t2
-              ON t1.type_name = t2.type_name
-              AND t1.fuel_common_name = t2.fuel_common_name
-              AND t1.control_name = t2.control_name
-            WHERE
-              t1.type_name = #{quoted_table_name}.type_name
-              AND t1.fuel_common_name = #{quoted_table_name}.fuel_common_name
-              AND t1.year = #{quoted_table_name}.year )
-        }
-        update_all(%{
-          #{gas}_emission_factor = 1.0 * #{emission_factor} * total_travel / fuel_consumption,
-          #{gas}_emission_factor_units = 'kilograms_per_litre'
-        })
+    # Can't use an update all here b/c efs are methods defined on ATFYC that look up the ef from ATFC
+    process "Derive ch4 and n2o emission factor from AutomobileTypeFuelYearControl" do
+      find_each do |atfy|
+        atfy.ch4_emission_factor = atfy.type_fuel_year_controls.sum{ |atfyc| atfyc.total_travel_percent * atfyc.ch4_emission_factor }
+        atfy.n2o_emission_factor = atfy.type_fuel_year_controls.sum{ |atfyc| atfyc.total_travel_percent * atfyc.n2o_emission_factor }
+        atfy.ch4_emission_factor_units = atfy.type_fuel_year_controls.first.ch4_emission_factor_units
+        atfy.n2o_emission_factor_units = atfy.type_fuel_year_controls.first.n2o_emission_factor_units
+        atfy.save!
       end
     end
   end