earth 0.7.0 → 0.11.0

data/lib/earth/automobile/automobile_make_model_year/data_miner.rb

@@ -13,32 +13,48 @@ AutomobileMakeModelYear.class_eval do
         :src => AutomobileMakeModelYearVariant,
         :dest => AutomobileMakeModelYear,
         :cols => {
-          :make_model_year_name => :name,
+          [:make_name, :model_name, :year] => :name,
           :make_name => :make_name,
-          :name => :model_name,
-          :make_model_name => :make_model_name,
+          :model_name => :model_name,
           :year => :year,
-          :make_year_name => :make_year_name
         }
-        # :where => 'LENGTH(src.make_name) > 0 AND LENGTH(src.make_model_name) > 0'
       )
     end
     
-    # FIXME TODO make this a method on AutomobileMakeModelYear?
-    # TODO: weight by volume somehow
-    # note that we used to derive averages from make years, but here we get it from variants
-    # even without volume-weighting, the values are much better.
-    # for example, 20km/l for a toyota prius 2006 vs. 13km/l if you use make years
+    process "Derive fuel from automobile make model year variants" do
+      model_years = arel_table
+      variants = AutomobileMakeModelYearVariant.arel_table
+      conditional_relation = variants[:make_name].eq(model_years[:make_name]).and(variants[:model_name].eq(model_years[:model_name])).and(variants[:year].eq(model_years[:year]))
+      where("(#{variants.project('COUNT(DISTINCT fuel_code)').where(conditional_relation).to_sql}) = 1").update_all "fuel_code = (#{variants.project('DISTINCT fuel_code').where(conditional_relation).to_sql})"
+    end
+    
+    # FIXME TODO not in automobile_make_model_year_variants because they're missing from the EPA FEG download files:
+    # 2005 Honda Accord Hybrid
+    # 2006 Honda Accord Hybrid
+    # 2009 Chrysler Aspen HEV
+    # 2009 Dodge Durango HEV
+    import "A list of hybrid make model years derived from the EPA fuel economy guide",
+           :url => 'https://docs.google.com/spreadsheet/pub?hl=en_US&hl=en_US&key=0AoQJbWqPrREqdGtzekE4cGNoRGVmdmZMaTNvOWluSnc&output=csv' do
+      key 'name', :synthesize => lambda { |record| [record['make_name'], record['model_name'], record['year']].join(' ') }
+      store 'hybridity'
+    end
+    
+    process "Set hybridity to FALSE for all other make model years" do
+      where(:hybridity => nil).update_all :hybridity => false
+    end
+    
+    # FIXME TODO: weight by volume somehow
     process "Calculate city and highway fuel efficiency from automobile make model year variants" do
-      model_years = AutomobileMakeModelYear.arel_table
+      model_years = arel_table
       variants = AutomobileMakeModelYearVariant.arel_table
-      conditional_relation = model_years[:name].eq(variants[:make_model_year_name])
-      %w{ city highway }.each do |i|
-        null_check = variants[:"fuel_efficiency_#{i}"].not_eq(nil)
-        # sabshere 12/6/10 careful, don't use AutomobileMakeModelYearVariant.where here or you will be forced into projecting *
-        relation = variants.project(variants[:"fuel_efficiency_#{i}"].average).where(conditional_relation).where(null_check)
-        update_all "fuel_efficiency_#{i} = (#{relation.to_sql})"
-        update_all "fuel_efficiency_#{i}_units" => 'kilometres_per_litre'
+      conditional_relation = model_years[:make_name].eq(variants[:make_name]).and(model_years[:model_name].eq(variants[:model_name])).and(model_years[:year].eq(variants[:year]))
+      %w{ city highway }.each do |type|
+        null_check = variants[:"fuel_efficiency_#{type}"].not_eq(nil)
+        relation = variants.project(variants[:"fuel_efficiency_#{type}"].average).where(conditional_relation).where(null_check)
+        update_all(%{
+          fuel_efficiency_#{type} = (#{relation.to_sql}),
+          fuel_efficiency_#{type}_units = 'kilometres_per_litre'
+        })
       end
     end
   end

data/lib/earth/automobile/automobile_make_model_year_variant.rb

@@ -1,64 +1,42 @@
 class AutomobileMakeModelYearVariant < ActiveRecord::Base
   set_primary_key :row_hash
   
-  belongs_to :make,            :class_name => 'AutomobileMake',          :foreign_key => 'make_name'
-  belongs_to :make_model,      :class_name => 'AutomobileMakeModel',     :foreign_key => 'make_model_name'
-  belongs_to :make_model_year, :class_name => 'AutomobileMakeModelYear', :foreign_key => 'make_model_year_name'
-  belongs_to :fuel,            :class_name => 'AutomobileFuel',          :foreign_key => 'fuel_code', :primary_key => 'code'
+  # It looks like synthesizing a unique name would require including pretty much every column from the FEGs
+  # (e.g. creeper gear, automatic vs automatic with lockup, feedback fuel system, etc.)
+  # The advantages of not keying on row_hash is we could update variants to note whether they're hybrid or not and
+  # it would let users specify variants. But my conclusion is that trying to do this won't be much of an improvement
+  # because it will be a bunch of work to include all the columns and the names will be really long and obtuse
+  # -Ian 10/18/2011
   
   col :row_hash
-  col :name # short name!
   col :make_name
-  col :make_model_name # make + model
-  col :make_year_name # make + year
-  col :make_model_year_name # make + model + year
-  col :year, :type => :integer
-  col :fuel_efficiency, :type => :float
-  col :fuel_efficiency_units
-  col :fuel_efficiency_city, :type => :float
-  col :fuel_efficiency_city_units
-  col :fuel_efficiency_highway, :type => :float
-  col :fuel_efficiency_highway_units
-  col :fuel_code
+  col :model_name
+  col :year,         :type => :integer
   col :transmission
+  col :speeds
   col :drive
-  col :turbo, :type => :boolean
-  col :supercharger, :type => :boolean
-  col :cylinders, :type => :integer
+  col :fuel_code
+  col :cylinders,    :type => :integer
   col :displacement, :type => :float
-  col :raw_fuel_efficiency_city, :type => :float
+  col :turbo,        :type => :boolean
+  col :supercharger, :type => :boolean
+  col :injection,    :type => :boolean
+  col :fuel_efficiency,             :type => :float
+  col :fuel_efficiency_units
+  col :fuel_efficiency_city,        :type => :float
+  col :fuel_efficiency_city_units
+  col :fuel_efficiency_highway,     :type => :float
+  col :fuel_efficiency_highway_units
+  col :raw_fuel_efficiency_city,    :type => :float
   col :raw_fuel_efficiency_city_units
   col :raw_fuel_efficiency_highway, :type => :float
   col :raw_fuel_efficiency_highway_units
-  col :carline_mfr_code, :type => :integer
-  col :vi_mfr_code, :type => :integer
-  col :carline_code, :type => :integer
+  col :carline_mfr_code,   :type => :integer
+  col :vi_mfr_code,        :type => :integer
+  col :carline_code,       :type => :integer
   col :carline_class_code, :type => :integer
-  col :injection, :type => :boolean
   col :carline_class_name
-  col :speeds
   add_index :make_name
-  add_index :make_model_name
-  add_index :make_year_name
-  add_index :make_model_year_name
-  
-  # verify "Fuel code should appear in AutomobileFuel" do
-  #   if (violators = connection.select_values("SELECT DISTINCT fuel_code FROM #{quoted_table_name} WHERE fuel_code NOT IN (SELECT code FROM #{AutomobileFuel.quoted_table_name})")).any?
-  #     raise "Invalid fuel code(s): #{violators.join(', ')}"
-  #   end
-  #   true
-  # end
-  # 
-  # verify "Fuel efficiencies should be greater than zero" do
-  #   [:fuel_efficiency, :fuel_efficiency_city, :fuel_efficiency_highway].each do |field|
-  #     if AutomobileMakeModelYearVariant.where(field => nil).any?
-  #       raise "Invalid #{field} in automobile_make_model_year_variants: nil is not > 0"
-  #     else
-  #       min = AutomobileMakeModelYearVariant.minimum(field)
-  #       unless min > 0
-  #         raise "Invalid #{field} in automobile_make_model_year_variants: #{min} is not > 0"
-  #       end
-  #     end
-  #   end
-  # end
-end
+  add_index :model_name
+  add_index :year
+end

data/lib/earth/automobile/automobile_make_model_year_variant/data_miner.rb

@@ -1,69 +1,24 @@
 AutomobileMakeModelYearVariant.class_eval do
   # For errata
   class AutomobileMakeModelYearVariant::Guru
-    def transmission_is_blank?(row)
-      row['transmission'].blank?
-    end
-    
-    def is_a_2007_gmc_or_chevrolet?(row)
-      row['year'] == 2007 and %w(GMC CHEVROLET).include? row['MFR'].upcase
-    end
-    
-    def is_a_porsche?(row)
-      row['make'].to_s.upcase == 'PORSCHE'
-    end
-    
-    def is_not_a_porsche?(row)
-      !is_a_porsche? row
-    end
-    
-    def is_a_mercedes_benz?(row)
-      row['make'].to_s =~ /MERCEDES/i
-    end
-    
-    def is_a_lexus?(row)
-      row['make'].to_s.upcase == 'LEXUS'
-    end
-    
-    def is_a_bmw?(row)
-      row['make'].to_s.upcase == 'BMW'
-    end
-    
-    def is_a_ford?(row)
-      row['make'].to_s.upcase == 'FORD'
-    end
-    
-    def is_a_rolls_royce_and_model_contains_bentley?(row)
-      is_a_rolls_royce?(row) and model_contains_bentley?(row)
-    end
-    
-    def is_a_bentley?(row)
-      row['make'].to_s.upcase == 'BENTLEY'
+    %w{ alpina bentley chevrolet dodge ferrari jaguar kia lexus maybach mercedes-benz mitsubishi porsche toyota tvr volvo yugo }.each do |make|
+      make_for_method = make.gsub('-', '_')
+      method_name = :"is_a_#{make_for_method}?"
+      define_method method_name do |row|
+        row['make'].to_s.upcase == "#{make.humanize.upcase}"
+      end
     end
     
-    def is_a_rolls_royce?(row)
-      row['make'].to_s =~ /ROLLS/i
+    %w{ alpina bentley maybach smart }.each do |model|
+      method_name = :"model_contains_#{model}?"
+      define_method method_name do |row|
+        row['model'].to_s =~ /#{model.upcase}/i
+      end
     end
     
     def is_a_turbo_brooklands?(row)
       row['model'] =~ /TURBO R\/RL BKLDS/i
     end
-    
-    def model_contains_maybach?(row)
-      row['model'] =~ /MAYBACH/i
-    end
-    
-    def model_contains_bentley?(row)
-      row['model'] =~ /BENTLEY/i
-    end
-    
-    def name_contains_smart?(row)
-      row['model'] =~ / smart /i
-    end
-    
-    def source_code
-      IO.read __FILE__
-    end
   end
   
   # updated with 2010 names
@@ -78,18 +33,18 @@ AutomobileMakeModelYearVariant.class_eval do
     'CVT' => 'Continuously Variable',
     'OT'  => 'Other'
   }
-
+  
   ENGINE_TYPES = {
     '(GUZZLER)' => nil, # "gas guzzler"
     '(POLICE)' => nil, # police automobile_variant
     '(MPFI)' => 'injection',
     '(MPI*)' => 'injection',
     '(SPFI)' => 'injection',
-    '(FFS)' => 'injection',
+    '(FFS)' => nil, # doesn't necessarily mean fuel injection
     '(TURBO)' => 'turbo',
     '(TRBO)' => 'turbo',
     '(TC*)' => 'turbo',
-    '(FFS,TRBO)' => %w(injection turbo),
+    '(FFS,TRBO)' => 'turbo',
     '(S-CHARGE)' => 'supercharger',
     '(SC*)' => 'supercharger',
     '(DIESEL)' => nil, # diesel
@@ -165,26 +120,33 @@ AutomobileMakeModelYearVariant.class_eval do
     end
     def apply(row)
       row.merge!({
-        'make'           => row['carline_mfr_name'], # make it line up with the errata
-        'model'          => row['carline_name'],     # ditto
-        'transmission'   => TRANSMISSIONS[row['model_trans'][0, 1]],
-        'speeds'         => (row['model_trans'][1, 1] == 'V') ? 'variable' : row['model_trans'][1, 1],
-        'turbo'          => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('turbo'),
-        'supercharger'   => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('supercharger'),
-        'injection'      => [ENGINE_TYPES[row['engine_desc1']], ENGINE_TYPES[row['engine_desc2']]].flatten.include?('injection'),
-        'displacement'   => _displacement(row['opt_disp']),
-        'year'           => year
+        'make'           => row['carline_mfr_name'],    # make it line up with the errata
+        'model'          => row['carline_name'].upcase, # ditto
+        'year'           => year,
+        'transmission'   => TRANSMISSIONS[row['model_trans'][0,1].to_s],
+        'speeds'         => row['model_trans'][1,1] == 'V' ? 'variable' : row['model_trans'][1,1],
+        'displacement'   => _displacement(row),
+        'turbo'          => _turbo(row),
+        'supercharger'   => [ENGINE_TYPES[row['engine_desc1'].to_s], ENGINE_TYPES[row['engine_desc2'].to_s]].flatten.include?('supercharger'),
+        'injection'      => row['fuel_system'] == 'FI' ? true : false
       })
       row
     end
-    def _displacement(str)
-      str = str.gsub(/[\(\)]/, '').strip
-      if str =~ /^(.+)L$/
+    def _displacement(row)
+      optional_displacement = row['opt_disp'].gsub(/[\(\)]/, '').strip.to_s
+      if optional_displacement =~ /^(\d\.\d)L$/
         $1.to_f
-      elsif str =~ /^(.+)CC$/
-        $1.to_f / 1000
+      elsif optional_displacement =~ /^(\d{4})CC$/
+        ($1.to_f / 1000).round(1)
+      else
+        row['disp_cub_in'].to_f.cubic_inches.to(:litres).round(1)
       end
     end
+    def _turbo(row)
+      engine_types = [ENGINE_TYPES[row['engine_desc1'].to_s], ENGINE_TYPES[row['engine_desc2'].to_s]]
+      engine_types << (row['carline_name'].to_s.downcase.include?('turbo') ? 'turbo' : nil)
+      engine_types.flatten.include?('turbo')
+    end
   end
   
   class AutomobileMakeModelYearVariant::ParserC
@@ -195,12 +157,12 @@ AutomobileMakeModelYearVariant.class_eval do
     end
     def apply(row)
       row.merge!({
-        'make'           => row['Manufacturer'], # make it line up with the errata
-        'model'          => row['carline name'], # ditto
+        'make'           => row['Manufacturer'],        # make it line up with the errata
+        'model'          => row['carline name'].upcase, # ditto
         'drive'          => row['drv'] + 'WD',
-        'transmission'   => TRANSMISSIONS[row['trans'][-3, 1]], # only using prefix, probably a FIXME
+        'transmission'   => TRANSMISSIONS[row['trans'][-3, 1]],
         'speeds'         => (row['trans'][-2, 1] == 'V') ? '1' : row['trans'][-2, 1],
-        'turbo'          => row['T'] == 'T',
+        'turbo'          => (row['T'] == 'T' || row['carline name'].to_s.downcase.include?('turbo')) ? true : false,
         'supercharger'   => row['S'] == 'S',
         'injection'      => true,
         'year'           => year
@@ -217,10 +179,10 @@ AutomobileMakeModelYearVariant.class_eval do
     end
     def apply(row)
       row.merge!({
-        'make'           => row['MFR'],          # make it line up with the errata
-        'model'          => row['CAR LINE'],     # ditto
+        'make'           => row['MFR'],             # make it line up with the errata
+        'model'          => row['CAR LINE'].upcase, # ditto
         'drive'          => row['DRIVE SYS'] + 'WD',
-        'transmission'   => TRANSMISSIONS[row['TRANS'][-3, 1]], # only using prefix, probably a FIXME
+        'transmission'   => TRANSMISSIONS[row['TRANS'][-3, 1]],
         'speeds'         => (row['TRANS'][-2, 1] == 'V') ? '1' : row['TRANS'][-2, 1],
         'turbo'          => row['TURBO'] == 'T',
         'supercharger'   => row['SPCHGR'] == 'S',
@@ -249,8 +211,8 @@ AutomobileMakeModelYearVariant.class_eval do
     end
     def apply(row)
       row.merge!({
-        'make'           => row['Division'],          # make it line up with the errata
-        'model'          => row['Carline'],     # ditto
+        'make'           => row['Division'],       # make it line up with the errata
+        'model'          => row['Carline'].upcase, # ditto
         'drive'          => row['Drive Sys'] + 'WD',
         'transmission'   => TRANSMISSIONS[row['Trans']],
         'speeds'         => row['# Gears'],
@@ -266,46 +228,47 @@ AutomobileMakeModelYearVariant.class_eval do
   
   data_miner do
     # 1985---1997
-    # FIXME TODO one Jaguar in the 1990 FEG has no model name
     # FIXME TODO 14 records in the 1995 FEG are missing fuel efficiencies
     (85..97).each do |yy|
     # [85, 95, 96].each do |yy|
       filename = (yy == 96) ? "#{yy}MFGUI.ASC" : "#{yy}MFGUI.DAT"
-      import("19#{ yy } Fuel Economy Guide",
+      import "19#{ yy } Fuel Economy Guide",
              :url => "http://www.fueleconomy.gov/FEG/epadata/#{yy}mfgui.zip",
+             # :url => "file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/#{filename}",
              :format => :fixed_width,
              :cut => ((yy == 95) ? '13-' : nil),
              :schema_name => :fuel_economy_guide_b,
              :select => lambda { |row| row['model'].present? and (row['suppress_code'].blank? or row['suppress_code'].to_f == 0) and row['state_code'] == 'F' },
              :filename => filename,
              :transform => { :class => AutomobileMakeModelYearVariant::ParserB, :year => "19#{yy}".to_i },
-             :errata => { :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdEFqVXRvQjRGNUpxNHFCOXhqSjRmdlE&output=csv', :responder => AutomobileMakeModelYearVariant::Guru.new }) do
+             :errata => { :url => "https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDkxTElWRVlvUXB3Uy04SDhSYWkzakE&output=csv", :responder => AutomobileMakeModelYearVariant::Guru.new } do
+             # :errata => { :url => "file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/errata_new.csv", :responder => AutomobileMakeModelYearVariant::Guru.new } do
         key   'row_hash'
-        store 'name', :field_name => 'model'
-        store 'make_name', :field_name => 'make'
+        store 'make_name',  :field_name => 'make'
+        store 'model_name', :field_name => 'model'
         store 'year'
-        store 'fuel_code', :field_name => 'fuel_type'
-        store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
-        store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
-        store 'raw_fuel_efficiency_highway', :field_name => 'unadj_hwy_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
-        store 'raw_fuel_efficiency_city', :field_name => 'unadj_city_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
-        store 'cylinders', :field_name => 'no_cyc'
-        store 'drive', :field_name => 'drive_system'
-        store 'carline_mfr_code'
-        store 'vi_mfr_code'
-        store 'carline_code'
-        store 'carline_class_code', :field_name => 'carline_clss'
         store 'transmission'
         store 'speeds'
+        store 'drive',      :field_name => 'drive_system'
+        store 'fuel_code',  :field_name => 'fuel_type'
+        store 'cylinders',  :field_name => 'no_cyc'
+        store 'displacement'
         store 'turbo'
         store 'supercharger'
         store 'injection'
-        store 'displacement'
+        store 'raw_fuel_efficiency_city',    :field_name => 'unadj_city_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
+        store 'raw_fuel_efficiency_highway', :field_name => 'unadj_hwy_mpg',  :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
+        store 'carline_mfr_code'
+        store 'vi_mfr_code'
+        store 'carline_code'
+        store 'carline_class_code', :field_name => 'carline_clss'
       end
     end
     
     # 1998--2005
     {
+      # FIXME TODO 2005 Mercedes-Benz SLK55 AMG has NULL speeds (it does in the EPA FEG also)
+      # FIXME TODO numbers from xls files are getting imported as floats rather than integers (e.g. 4.0WD rather than 4WD)
       1998 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/98guide6.zip', :filename => '98guide6.csv' },
       1999 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/99guide.zip', :filename => '99guide6.csv' },
       2000 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip', :filename => 'G6080900.xls' },
@@ -314,30 +277,37 @@ AutomobileMakeModelYearVariant.class_eval do
       2003 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/03data.zip', :filename => 'guide_2003_feb04-03b.csv' },
       2004 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/04data.zip', :filename => 'gd04-Feb1804-RelDtFeb20.csv' },
       2005 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/05data.zip', :filename => 'guide2005-2004oct15.csv' }
+      # 1998 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/98guide6.csv' },
+      # 1999 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/99guide6.csv' },
+      # 2000 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/G6080900.csv' },
+      # 2001 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/01guide0918.csv' },
+      # 2002 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/guide_jan28.csv' },
+      # 2003 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/guide_2003_feb04-03b.csv' },
+      # 2004 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/gd04-Feb1804-RelDtFeb20.csv' },
+      # 2005 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/guide2005-2004oct15.csv' }
     }.each do |year, options|
       import "#{ year } Fuel Economy Guide",
              options.merge(:transform => { :class => AutomobileMakeModelYearVariant::ParserC, :year => year },
-                           :errata => { :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdEFqVXRvQjRGNUpxNHFCOXhqSjRmdlE&output=csv', :responder => AutomobileMakeModelYearVariant::Guru.new },
+                           :errata => { :url => "https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDkxTElWRVlvUXB3Uy04SDhSYWkzakE&output=csv", :responder => AutomobileMakeModelYearVariant::Guru.new },
+                           # :errata => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/errata_new.csv', :responder => AutomobileMakeModelYearVariant::Guru.new },
                            :select => lambda { |row| row['model'].present? }) do
         key   'row_hash'
-        store 'name', :field_name => 'model'
-        store 'make_name', :field_name => 'make'
-        store 'fuel_code', :field_name => 'fl'
-        store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
-        store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
-        store 'raw_fuel_efficiency_highway', :field_name => 'uhwy', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
-        store 'raw_fuel_efficiency_city', :field_name => 'ucty', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
-        store 'cylinders', :field_name => 'cyl'
-        store 'displacement', :field_name => 'displ'
-        store 'carline_class_code', :field_name => 'cls' if year >= 2000
-        store 'carline_class_name', :field_name => 'Class'
+        store 'make_name',  :field_name => 'make'
+        store 'model_name', :field_name => 'model'
         store 'year'
         store 'transmission'
         store 'speeds'
+        store 'drive'
+        store 'fuel_code',    :field_name => 'fl'
+        store 'cylinders',    :field_name => 'cyl'
+        store 'displacement', :field_name => 'displ'
         store 'turbo'
         store 'supercharger'
         store 'injection'
-        store 'drive'
+        store 'raw_fuel_efficiency_highway', :field_name => 'uhwy', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
+        store 'raw_fuel_efficiency_city',    :field_name => 'ucty', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
+        store 'carline_class_code',          :field_name => 'cls' if year >= 2000
+        store 'carline_class_name',          :field_name => 'Class'
       end
     end
     
@@ -347,15 +317,21 @@ AutomobileMakeModelYearVariant.class_eval do
       # the 07data.xls file provided by the government has a bad encoding
       2007 => { :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/2007_FE_guide_ALL_no_sales_May_01_2007.csv' },
       2008 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/08data.zip', :filename => '2008_FE_guide_ALL_rel_dates_-no sales-for DOE-5-1-08.csv' },
-      2009 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/09data.zip', :filename => '2009_FE_guide for DOE_ALL-rel dates-no-sales-8-28-08download.csv' },
+      2009 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/09data.zip', :filename => '2009_FE_guide for DOE_ALL-rel dates-no-sales-8-28-08download.csv' }
+      # 2006 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/2006_FE_Guide_14-Nov-2005_download.csv' },
+      # 2007 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/2007_FE_guide_ALL_no_sales_May_01_2007.csv' },
+      # 2008 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/2008_FE_guide.csv' },
+      # 2009 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/2009_FE_guide.csv' }
     }.each do |year, options|
       import "#{ year } Fuel Economy Guide",
              options.merge(:transform => { :class => AutomobileMakeModelYearVariant::ParserD, :year => year },
-                           :errata => { :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdEFqVXRvQjRGNUpxNHFCOXhqSjRmdlE&output=csv', :responder => AutomobileMakeModelYearVariant::Guru.new },
+                           :errata => { :url => "https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDkxTElWRVlvUXB3Uy04SDhSYWkzakE&output=csv", :responder => AutomobileMakeModelYearVariant::Guru.new },
+                           # :errata => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/errata_new.csv', :responder => AutomobileMakeModelYearVariant::Guru.new },
                            :select => lambda { |row| row['model'].present? }) do
         key   'row_hash'
-        store 'name', :field_name => 'model'
         store 'make_name', :field_name => 'make'
+        store 'model_name', :field_name => 'model'
+        store 'year'
         store 'fuel_code', :field_name => 'FUEL TYPE'
         store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
         store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
@@ -365,7 +341,6 @@ AutomobileMakeModelYearVariant.class_eval do
         store 'displacement', :field_name => 'DISPLACEMENT'
         store 'carline_class_code', :field_name => 'CLS'
         store 'carline_class_name', :field_name => 'CLASS'
-        store 'year'
         store 'transmission'
         store 'speeds'
         store 'turbo'
@@ -377,16 +352,22 @@ AutomobileMakeModelYearVariant.class_eval do
     
     # 2010--?
     {
+      # FIXME TODO numbers from xlsx files are getting imported as floats rather than integers (e.g. speeds = 6.0 rather than 6)
+      # Note: it's ok for electric vehicles to be missing cylinders and displacement
       2010 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/10data.zip', :filename => '2010FEGuide for DOE-all rel dates-no-sales-02-22-2011public.xlsx' },
       2011 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/11data.zip', :filename => '2011FEGuide-for DOE rel-dates before 1-23-2011-no-sales-01-10-2011_All_public.xlsx' }
+      # 2010 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/2010FEGuide.xlsx' },
+      # 2011 => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/2011FEGuide.xlsx' }
     }.each do |year, options|
       import "#{ year } Fuel Economy Guide",
              options.merge(:transform => { :class => AutomobileMakeModelYearVariant::ParserE, :year => year },
-                           :errata => { :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdEFqVXRvQjRGNUpxNHFCOXhqSjRmdlE&output=csv', :responder => AutomobileMakeModelYearVariant::Guru.new },
+                           :errata => { :url => "https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDkxTElWRVlvUXB3Uy04SDhSYWkzakE&output=csv", :responder => AutomobileMakeModelYearVariant::Guru.new },
+                           # :errata => { :url => 'file:///Users/ian/Documents/brighter_planet/documents1/science/data/transport/automobiles/fuel_economy_guide/errata_new.csv', :responder => AutomobileMakeModelYearVariant::Guru.new },
                            :select => lambda { |row| row['model'].present? }) do
         key   'row_hash'
-        store 'name', :field_name => 'model'
         store 'make_name', :field_name => 'make'
+        store 'model_name', :field_name => 'model'
+        store 'year'
         store 'fuel_code', :field_name => 'fuel_type_code'
         store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
         store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
@@ -396,7 +377,6 @@ AutomobileMakeModelYearVariant.class_eval do
         store 'displacement', :field_name => 'Eng Displ'
         store 'carline_class_code', :field_name => 'Carline Class'
         store 'carline_class_name', :field_name => 'Carline Class Desc'
-        store 'year'
         store 'transmission'
         store 'speeds'
         store 'turbo'
@@ -406,36 +386,27 @@ AutomobileMakeModelYearVariant.class_eval do
       end
     end
     
-    process "Derive model and model year names" do
-      update_all "make_model_name = make_name || ' ' || name"
-      update_all "make_year_name = make_name || ' ' || year"
-      update_all "make_model_year_name = make_name || ' ' || name || ' ' || year"
-    end
-    
     # Note: need to divide by 0.425143707 b/c equation is designed for miles / gallon not km / l
     # Note: EPA seems to adjust differently for plug-in hybrid electric vehicles (e.g. Leaf and Volt)
     process "Calculate adjusted fuel efficiency using the latest EPA equations from EPA Fuel Economy Trends report Appendix A including conversion from miles per gallon to kilometres per litre" do
-      update_all 'fuel_efficiency_city = 1.0 / ((0.003259 / 0.425143707) + (1.1805 / raw_fuel_efficiency_city))', 'raw_fuel_efficiency_city > 0'
-      update_all 'fuel_efficiency_highway = 1.0 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))', 'raw_fuel_efficiency_highway > 0'
+      where("raw_fuel_efficiency_city > 0").update_all(%{
+        fuel_efficiency_city = 1.0 / ((0.003259 / 0.425143707) + (1.1805 / raw_fuel_efficiency_city)),
+        fuel_efficiency_city_units = 'kilometres_per_litre'
+      })
+      where("raw_fuel_efficiency_highway > 0").update_all(%{
+        fuel_efficiency_highway = 1.0 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway)),
+        fuel_efficiency_highway_units = 'kilometres_per_litre'
+      })
     end
     
     # This will be useful later for calculating MakeModel and Make fuel efficiency based on Variant
     # NOTE: we use a 43/57 city/highway weighting per the latest EPA analysis of real-world driving behavior
     # This results in a deviation from EPA fuel economy label values which use a historical 55/45 weighting
     process "Calculate combined adjusted fuel efficiency using the latest EPA equation" do
-      update_all "fuel_efficiency = 1.0 / ((0.43 / fuel_efficiency_city) + (0.57 / fuel_efficiency_highway))"
-    end
-    
-    process "Set units" do
-      update_all :fuel_efficiency_units => 'kilometres_per_litre'
-      update_all :fuel_efficiency_city_units => 'kilometres_per_litre'
-      update_all :fuel_efficiency_highway_units => 'kilometres_per_litre'
-    end
-    
-    process "Ensure related tables are populated" do
-      AutomobileMakeModelYear.run_data_miner!
-      AutomobileMakeModel.run_data_miner!
-      AutomobileFuel.run_data_miner!
+      update_all(%{
+        fuel_efficiency = 1.0 / ((0.43 / fuel_efficiency_city) + (0.57 / fuel_efficiency_highway)),
+        fuel_efficiency_units = 'kilometres_per_litre'
+      })
     end
   end
 end