earth 0.3.15 → 0.4.0

data/lib/earth/automobile/automobile_fuel_type.rb

@@ -1,3 +1,4 @@
+# DEPRECATED - use AutomobileFuel
 class AutomobileFuelType < ActiveRecord::Base
   set_primary_key :code
   

data/lib/earth/automobile/automobile_fuel_type/data_miner.rb

@@ -1,3 +1,4 @@
+# DEPRECATED - use AutomobileFuel
 AutomobileFuelType.class_eval do
   data_miner do
     schema Earth.database_options do
@@ -105,7 +106,7 @@ AutomobileFuelType.class_eval do
     
     verify "Emission factor units should never be missing" do
       AutomobileFuelType.all.each do |fuel_type|
-        if fuel_type.emission_factor_units.nil?
+        unless fuel_type.emission_factor_units.present?
           raise "Missing emission factor units for AutomobileFuelType #{fuel_type.name}"
         end
       end

data/lib/earth/automobile/automobile_make/data_miner.rb

@@ -19,6 +19,17 @@ AutomobileMake.class_eval do
       }
     end
     
+    # sabshere 1/31/11 add Avanti, DaimlerChrysler, IHC, Tesla, etc.
+    process "Derive extra manufacturer names from CAFE data" do
+      AutomobileMakeFleetYear.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO automobile_makes(name)
+        SELECT DISTINCT automobile_make_fleet_years.make_name
+        FROM automobile_make_fleet_years
+      }
+    end
+    
+    # FIXME TODO make this a method on AutomobileMake?
     process "Calculate fuel efficiency from automobile make fleet years for makes with CAFE data" do
       AutomobileMakeFleetYear.run_data_miner!
       make_fleet_years = AutomobileMakeFleetYear.arel_table

data/lib/earth/automobile/automobile_make_fleet_year.rb

@@ -3,11 +3,11 @@ class AutomobileMakeFleetYear < ActiveRecord::Base
   
   belongs_to :make,      :class_name => 'AutomobileMake',     :foreign_key => 'make_name'
   belongs_to :make_year, :class_name => 'AutomobileMakeYear', :foreign_key => 'make_year_name'
-
+  
   data_miner do
     tap "Brighter Planet's sanitized auto make fleet year data", Earth.taps_server
     
-    process "bring in dependencies" do
+    process "Pull dependencies" do
       run_data_miner_on_belongs_to_associations
     end
   end

data/lib/earth/automobile/automobile_make_model.rb

@@ -7,7 +7,7 @@ class AutomobileMakeModel < ActiveRecord::Base
   data_miner do
     tap "Brighter Planet's auto model data", Earth.taps_server
     
-    process "bring in dependencies" do
+    process "Pull dependencies" do
       run_data_miner_on_belongs_to_associations
     end
   end

data/lib/earth/automobile/automobile_make_model/data_miner.rb

@@ -21,6 +21,7 @@ AutomobileMakeModel.class_eval do
       }
     end
     
+    # FIXME TODO make this a method on AutomobileMakeModel?
     # TODO not weighted until we get weightings on auto variants
     process "Derive average fuel economy from automobile make model year variants" do
       models = AutomobileMakeModel.arel_table

data/lib/earth/automobile/automobile_make_model_year/data_miner.rb

@@ -24,6 +24,7 @@ AutomobileMakeModelYear.class_eval do
       }
     end
     
+    # FIXME TODO make this a method on AutomobileMakeModelYear?
     # TODO: weight by volume somehow
     # note that we used to derive averages from make years, but here we get it from variants
     # even without volume-weighting, the values are much better.

data/lib/earth/automobile/automobile_make_model_year_variant.rb

@@ -5,7 +5,8 @@ class AutomobileMakeModelYearVariant < ActiveRecord::Base
   belongs_to :make_model,      :class_name => 'AutomobileMakeModel',     :foreign_key => 'make_model_name'
   belongs_to :make_model_year, :class_name => 'AutomobileMakeModelYear', :foreign_key => 'make_model_year_name'
   belongs_to :fuel_type,       :class_name => 'AutomobileFuelType',      :foreign_key => 'fuel_type_code'
-
+  belongs_to :fuel,            :class_name => 'AutomobileFuel',          :foreign_key => 'fuel_code', :primary_key => 'code'
+  
   data_miner do
     tap "Brighter Planet's sanitized automobile make model year variant data", Earth.taps_server
     

data/lib/earth/automobile/automobile_make_model_year_variant/data_miner.rb

@@ -272,6 +272,7 @@ AutomobileMakeModelYearVariant.class_eval do
       float    'fuel_efficiency_highway'
       string   'fuel_efficiency_highway_units'
       string   'fuel_type_code'
+      string   'fuel_code'
       string   'transmission'
       string   'drive'
       boolean  'turbo'
@@ -306,7 +307,7 @@ AutomobileMakeModelYearVariant.class_eval do
              :format => :fixed_width,
              :cut => ((yy == 95) ? '13-' : nil),
              :schema_name => :fuel_economy_guide_b,
-             :select => lambda { |row| (row['suppress_code'].blank? or row['suppress_code'].to_f == 0) and row['state_code'] == 'F' },
+             :select => lambda { |row| row['model'].present? and (row['suppress_code'].blank? or row['suppress_code'].to_f == 0) and row['state_code'] == 'F' },
              :filename => filename,
              :transform => { :class => AutomobileMakeModelYearVariant::ParserB, :year => "19#{yy}".to_i },
              :errata => { 'url' => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv' }) do
@@ -315,6 +316,7 @@ AutomobileMakeModelYearVariant.class_eval do
         store 'make_name', :field_name => 'make'
         store 'year'
         store 'fuel_type_code', :field_name => 'fuel_type'
+        store 'fuel_code', :field_name => 'fuel_type'
         store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
         store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
         store 'raw_fuel_efficiency_highway', :field_name => 'unadj_hwy_mpg', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
@@ -347,11 +349,13 @@ AutomobileMakeModelYearVariant.class_eval do
     }.sort { |a, b| a.first <=> b.first }.each do |year, options|
       import "#{ year } Fuel Economy Guide",
              options.merge(:transform => { :class => AutomobileMakeModelYearVariant::ParserC, :year => year },
-                           :errata => { 'url' => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv' }) do
+                           :errata => { 'url' => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv' },
+                           :select => lambda { |row| row['model'].present? }) do
         key   'row_hash'
         store 'name', :field_name => 'model'
         store 'make_name', :field_name => 'make'
         store 'fuel_type_code', :field_name => 'fl'
+        store 'fuel_code', :field_name => 'fl'
         store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
         store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
         store 'raw_fuel_efficiency_highway', :field_name => 'uhwy', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
@@ -380,11 +384,13 @@ AutomobileMakeModelYearVariant.class_eval do
     }.sort { |a, b| a.first <=> b.first }.each do |year, options|
       import "#{ year } Fuel Economy Guide",
              options.merge(:transform => { :class => AutomobileMakeModelYearVariant::ParserD, :year => year },
-                           :errata => { 'url' => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv' }) do
+                           :errata => { 'url' => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv' },
+                           :select => lambda { |row| row['model'].present? }) do
         key   'row_hash'
         store 'name', :field_name => 'model'
         store 'make_name', :field_name => 'make'
         store 'fuel_type_code', :field_name => 'FUEL TYPE'
+        store 'fuel_code', :field_name => 'FUEL TYPE'
         store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
         store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
         store 'raw_fuel_efficiency_highway', :field_name => 'UNRND HWY (EPA)', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
@@ -409,11 +415,13 @@ AutomobileMakeModelYearVariant.class_eval do
     }.sort { |a, b| a.first <=> b.first }.each do |year, options|
       import "#{ year } Fuel Economy Guide",
              options.merge(:transform => { :class => AutomobileMakeModelYearVariant::ParserE, :year => year },
-                           :errata => { 'url' => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv' }) do
+                           :errata => { 'url' => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv' },
+                           :select => lambda { |row| row['model'].present? }) do
         key   'row_hash'
         store 'name', :field_name => 'model'
         store 'make_name', :field_name => 'make'
         store 'fuel_type_code'
+        store 'fuel_code', :field_name => 'fuel_type_code'
         store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
         store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
         store 'raw_fuel_efficiency_highway', :field_name => 'Hwy Unadj FE - Conventional Fuel', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
@@ -433,9 +441,15 @@ AutomobileMakeModelYearVariant.class_eval do
     end
     
     process "Derive model and model year names" do
-      update_all "make_model_name = CONCAT(make_name, ' ', name)"
-      update_all "make_year_name = CONCAT(make_name, ' ', year)"
-      update_all "make_model_year_name = CONCAT(make_name, ' ', name, ' ', year)"
+      if ActiveRecord::Base.connection.adapter_name == 'sqlite'
+        update_all "make_model_name = make_name || ' ' || name"
+        update_all "make_year_name = make_name || ' ' || year"
+        update_all "make_model_year_name = make_name || ' ' || name || ' ' || year"
+      else
+        update_all "make_model_name = CONCAT(make_name, ' ', name)"
+        update_all "make_year_name = CONCAT(make_name, ' ', year)"
+        update_all "make_model_year_name = CONCAT(make_name, ' ', name, ' ', year)"
+      end
     end
     
     # Note: need to divide by 0.425143707 b/c equation is designed for miles / gallon not km / l

data/lib/earth/automobile/automobile_make_year.rb

@@ -3,11 +3,11 @@ class AutomobileMakeYear < ActiveRecord::Base
   
   belongs_to :make,      :class_name => 'AutomobileMake',          :foreign_key => 'make_name'
   has_many :fleet_years, :class_name => 'AutomobileMakeFleetYear', :foreign_key => 'make_year_name'
-
+  
   data_miner do
     tap "Brighter Planet's make year data", Earth.taps_server
     
-    process "bring in dependencies" do
+    process "Pull dependencies" do
       run_data_miner_on_belongs_to_associations
     end
   end

data/lib/earth/automobile/automobile_make_year/data_miner.rb

@@ -25,6 +25,7 @@ AutomobileMakeYear.class_eval do
       }
     end
     
+    # FIXME TODO make this a method on AutomobileMakeYear?
     process "Calculate fuel efficiency from make fleet years for makes with CAFE data" do
       AutomobileMakeFleetYear.run_data_miner!
       make_fleet_years = AutomobileMakeFleetYear.arel_table
@@ -54,6 +55,8 @@ AutomobileMakeYear.class_eval do
       }
     end
     
+    # FIXME TODO verify make_name and volume?
+    
     verify "Year should be from 1985 to 2010" do
       AutomobileMakeYear.all.each do |make_year|
         unless make_year.year > 1984 and make_year.year < 2011

data/lib/earth/automobile/automobile_size_class/data_miner.rb

@@ -15,7 +15,8 @@ AutomobileSizeClass.class_eval do
       float  'conventional_fuel_efficiency_highway_multiplier'
     end
     
-    import "A list of size classes and pre-calculated fuel efficiencies",
+    # FIXME TODO pull this from AutomobileSizeClassYear
+    import "a list of size classes and pre-calculated fuel efficiencies",
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHlRUE5IcWlrRENhN0EtUldPTy1rX1E&single=true&gid=0&output=csv' do
       key 'name'
       store 'type_name'
@@ -23,16 +24,17 @@ AutomobileSizeClass.class_eval do
       store 'fuel_efficiency_highway', :units_field_name => 'fuel_efficiency_highway_units'
     end
     
-    process "Calculate annual distance from AutomobileTypeFuelAge" do
-      AutomobileTypeFuelAge.run_data_miner!
-      ages = AutomobileTypeFuelAge.arel_table
+    # FIXME TODO make this a method on AutomobileSizeClass?
+    process "Calculate annual distance from AutomobileTypeFuelYearAge" do
+      AutomobileTypeFuelYearAge.run_data_miner!
+      ages = AutomobileTypeFuelYearAge.arel_table
       classes = AutomobileSizeClass.arel_table
       conditional_relation = ages[:type_name].eq(classes[:type_name])
-      update_all "annual_distance = (#{AutomobileTypeFuelAge.weighted_average_relation(:annual_distance, :weighted_by => :vehicles).where(conditional_relation).to_sql})"
+      update_all "annual_distance = (#{AutomobileTypeFuelYearAge.weighted_average_relation(:annual_distance, :weighted_by => :vehicles).where(conditional_relation).to_sql})"
       update_all "annual_distance_units = 'kilometres'"
     end
     
-    import "pre-calculated hybridity multipliers",
+    import "pre-calculated fuel efficiency multipliers",
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGt2NnhXLXUxNFRJSzczU3BkSHB3enc&hl=en&single=true&gid=0&output=csv' do
       key 'name'
       store 'hybrid_fuel_efficiency_city_multiplier'
@@ -41,6 +43,8 @@ AutomobileSizeClass.class_eval do
       store 'conventional_fuel_efficiency_highway_multiplier'
     end
     
+    # FIXME TODO verify type_name?
+    
     verify "Annual distance should be greater than zero" do
       AutomobileSizeClass.all.each do |size_class|
         unless size_class.annual_distance > 0
@@ -84,7 +88,7 @@ AutomobileSizeClass.class_eval do
         %w{ hybrid conventional }.each do |hybridity|
           %w{ city highway }.each do |type|
             multiplier = size_class.send(:"#{hybridity}_fuel_efficiency_#{type}_multiplier")
-            if not multiplier.nil?
+            if multiplier.present?
               unless multiplier > 0
                 raise "Invalid #{hybridity} fuel efficiency #{type} multiplier for AutomobileSizeClass #{size_class.name}: #{multiplier} (should be > 0)"
               end

data/lib/earth/automobile/automobile_size_class_year/data_miner.rb

@@ -21,7 +21,8 @@ AutomobileSizeClassYear.class_eval do
       store 'fuel_efficiency_highway', :units_field_name => 'fuel_efficiency_highway_units'
     end
     
-    # FIXME TODO verify that type_name appears in AutomobileTypeFuelAges
+    # FIXME TODO verify that size_class_name is never missing?
+    # FIXME TODO verify that type_name appears in AutomobileTypeFuelYearAges
     
     verify "Year should be from 1975 to 2010" do
       AutomobileSizeClassYear.all.each do |record|

data/lib/earth/automobile/automobile_type_fuel_age.rb

@@ -1,3 +1,4 @@
+# DEPRECATED - use AutomobileTypeFuelYearAge
 class AutomobileTypeFuelAge < ActiveRecord::Base
   set_primary_key :name
   

data/lib/earth/automobile/automobile_type_fuel_age/data_miner.rb

@@ -1,3 +1,4 @@
+# DEPRECATED - use AutomobileTypeFuelYearAge
 AutomobileTypeFuelAge.class_eval do
   data_miner do
     schema Earth.database_options do
@@ -15,8 +16,9 @@ AutomobileTypeFuelAge.class_eval do
     import "age distribution of gasoline passenger cars from the 2010 EPA GHG Inventory",
            :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
            :filename => 'Annex Tables/Annex 3/Table A-91.csv',
-           :skip => 1 do
-      key 'name', :synthesize => lambda { |row| "Passenger cars gasoline age #{row['Vehicle Age']}" if row['Vehicle Age'].length < 3 }
+           :skip => 1,
+           :select => lambda { |row| row['Vehicle Age'].to_i.to_s == row['Vehicle Age'] } do
+      key 'name', :synthesize => lambda { |row| "Passenger cars gasoline age #{row['Vehicle Age']}" }
       store 'type_name', :static => 'Passenger cars'
       store 'fuel_common_name', :static => 'gasoline'
       store 'age', :field_name => 'Vehicle Age'
@@ -26,8 +28,9 @@ AutomobileTypeFuelAge.class_eval do
     import "age distribution of gasoline light-duty trucks from the 2010 EPA GHG Inventory",
            :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
            :filename => 'Annex Tables/Annex 3/Table A-91.csv',
-           :skip => 1 do
-      key 'name', :synthesize => lambda { |row| "Light-duty trucks gasoline age #{row['Vehicle Age']}" if row['Vehicle Age'].length < 3 }
+           :skip => 1,
+           :select => lambda { |row| row['Vehicle Age'].to_i.to_s == row['Vehicle Age'] } do
+      key 'name', :synthesize => lambda { |row| "Light-duty trucks gasoline age #{row['Vehicle Age']}" }
       store 'type_name', :static => 'Light-duty trucks'
       store 'fuel_common_name', :static => 'gasoline'
       store 'age', :field_name => 'Vehicle Age'
@@ -37,8 +40,9 @@ AutomobileTypeFuelAge.class_eval do
     import "age distribution of diesel passenger cars from the 2010 EPA GHG Inventory",
            :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
            :filename => 'Annex Tables/Annex 3/Table A-91.csv',
-           :skip => 1 do
-      key 'name', :synthesize => lambda { |row| "Passenger cars diesel age #{row['Vehicle Age']}" if row['Vehicle Age'].length < 3 }
+           :skip => 1,
+           :select => lambda { |row| row['Vehicle Age'].to_i.to_s == row['Vehicle Age'] } do
+      key 'name', :synthesize => lambda { |row| "Passenger cars diesel age #{row['Vehicle Age']}" }
       store 'type_name', :static => 'Passenger cars'
       store 'fuel_common_name', :static => 'diesel'
       store 'age', :field_name => 'Vehicle Age'
@@ -48,8 +52,9 @@ AutomobileTypeFuelAge.class_eval do
     import "age distribution of diesel light-duty trucks from the 2010 EPA GHG Inventory",
            :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
            :filename => 'Annex Tables/Annex 3/Table A-91.csv',
-           :skip => 1 do
-      key 'name', :synthesize => lambda { |row| "Light-duty trucks diesel age #{row['Vehicle Age']}" if row['Vehicle Age'].length < 3 }
+           :skip => 1,
+           :select => lambda { |row| row['Vehicle Age'].to_i.to_s == row['Vehicle Age'] } do
+      key 'name', :synthesize => lambda { |row| "Light-duty trucks diesel age #{row['Vehicle Age']}" }
       store 'type_name', :static => 'Light-duty trucks'
       store 'fuel_common_name', :static => 'diesel'
       store 'age', :field_name => 'Vehicle Age'
@@ -59,64 +64,72 @@ AutomobileTypeFuelAge.class_eval do
     import "total travel distribution of gasoline passenger cars from the 2010 EPA GHG Inventory",
            :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
            :filename => 'Annex Tables/Annex 3/Table A-93.csv',
-           :skip => 1 do
-      key 'name', :synthesize => lambda { |row| "Passenger cars gasoline age #{row['Vehicle Age']}" if row['Vehicle Age'].length < 3 }
+           :skip => 1,
+           :select => lambda { |row| row['Vehicle Age'].to_i.to_s == row['Vehicle Age'] } do
+      key 'name', :synthesize => lambda { |row| "Passenger cars gasoline age #{row['Vehicle Age']}" }
       store 'total_travel_percent', :synthesize => lambda { |row| row['LDGV'].to_f / 100 }
     end
     
     import "total travel distribution of gasoline light-duty trucks from the 2010 EPA GHG Inventory",
            :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
            :filename => 'Annex Tables/Annex 3/Table A-93.csv',
-           :skip => 1 do
-      key 'name', :synthesize => lambda { |row| "Light-duty trucks gasoline age #{row['Vehicle Age']}" if row['Vehicle Age'].length < 3 }
+           :skip => 1,
+           :select => lambda { |row| row['Vehicle Age'].to_i.to_s == row['Vehicle Age'] } do
+      key 'name', :synthesize => lambda { |row| "Light-duty trucks gasoline age #{row['Vehicle Age']}" }
       store 'total_travel_percent', :synthesize => lambda { |row| row['LDGT'].to_f / 100 }
     end
     
     import "total travel distribution of diesel passenger cars from the 2010 EPA GHG Inventory",
            :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
            :filename => 'Annex Tables/Annex 3/Table A-93.csv',
-           :skip => 1 do
-      key 'name', :synthesize => lambda { |row| "Passenger cars diesel age #{row['Vehicle Age']}" if row['Vehicle Age'].length < 3 }
+           :skip => 1,
+           :select => lambda { |row| row['Vehicle Age'].to_i.to_s == row['Vehicle Age'] } do
+      key 'name', :synthesize => lambda { |row| "Passenger cars diesel age #{row['Vehicle Age']}" }
       store 'total_travel_percent', :synthesize => lambda { |row| row['LDDV'].to_f / 100 }
     end
     
     import "total travel distribution of diesel light-duty trucks from the 2010 EPA GHG Inventory",
            :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
            :filename => 'Annex Tables/Annex 3/Table A-93.csv',
-           :skip => 1 do
-      key 'name', :synthesize => lambda { |row| "Light-duty trucks diesel age #{row['Vehicle Age']}" if row['Vehicle Age'].length < 3 }
+           :skip => 1,
+           :select => lambda { |row| row['Vehicle Age'].to_i.to_s == row['Vehicle Age'] } do
+      key 'name', :synthesize => lambda { |row| "Light-duty trucks diesel age #{row['Vehicle Age']}" }
       store 'total_travel_percent', :synthesize => lambda { |row| row['LDDT'].to_f / 100 }
     end
     
     import "average annual distance for gasoline passenger cars from the 2010 EPA GHG Inventory",
            :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
            :filename => 'Annex Tables/Annex 3/Table A-92.csv',
-           :skip => 1 do
-      key 'name', :synthesize => lambda { |row| "Passenger cars gasoline age #{row['Vehicle Age']}" if row['Vehicle Age'].length < 3 }
+           :skip => 1,
+           :select => lambda { |row| row['Vehicle Age'].to_i.to_s == row['Vehicle Age'] } do
+      key 'name', :synthesize => lambda { |row| "Passenger cars gasoline age #{row['Vehicle Age']}" }
       store 'annual_distance', :synthesize => lambda { |row| row['LDGV'].to_s.sub(',', '') }, :units => :miles
     end
     
     import "average annual distance for gasoline light-duty trucks from the 2010 EPA GHG Inventory",
            :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
            :filename => 'Annex Tables/Annex 3/Table A-92.csv',
-           :skip => 1 do
-      key 'name', :synthesize => lambda { |row| "Light-duty trucks gasoline age #{row['Vehicle Age']}" if row['Vehicle Age'].length < 3 }
+           :skip => 1,
+           :select => lambda { |row| row['Vehicle Age'].to_i.to_s == row['Vehicle Age'] } do
+      key 'name', :synthesize => lambda { |row| "Light-duty trucks gasoline age #{row['Vehicle Age']}" }
       store 'annual_distance', :synthesize => lambda { |row| row['LDGT'].to_s.sub(',', '') }, :units => :miles
     end
     
     import "average annual distance for diesel passenger cars from the 2010 EPA GHG Inventory",
            :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
            :filename => 'Annex Tables/Annex 3/Table A-92.csv',
-           :skip => 1 do
-      key 'name', :synthesize => lambda { |row| "Passenger cars diesel age #{row['Vehicle Age']}" if row['Vehicle Age'].length < 3 }
+           :skip => 1,
+           :select => lambda { |row| row['Vehicle Age'].to_i.to_s == row['Vehicle Age'] } do
+      key 'name', :synthesize => lambda { |row| "Passenger cars diesel age #{row['Vehicle Age']}" }
       store 'annual_distance', :synthesize => lambda { |row| row['LDDV'].to_s.sub(',', '') }, :units => :miles
     end
     
     import "average annual distance for diesel light-duty trucks from the 2010 EPA GHG Inventory",
            :url => 'http://www.epa.gov/climatechange/emissions/downloads10/2010-Inventory-Annex-Tables.zip',
            :filename => 'Annex Tables/Annex 3/Table A-92.csv',
-           :skip => 1 do
-      key 'name', :synthesize => lambda { |row| "Light-duty trucks diesel age #{row['Vehicle Age']}" if row['Vehicle Age'].length < 3 }
+           :skip => 1,
+           :select => lambda { |row| row['Vehicle Age'].to_i.to_s == row['Vehicle Age'] } do
+      key 'name', :synthesize => lambda { |row| "Light-duty trucks diesel age #{row['Vehicle Age']}" }
       store 'annual_distance', :synthesize => lambda { |row| row['LDDT'].to_s.sub(',', '') }, :units => :miles
     end
     
@@ -126,18 +139,23 @@ AutomobileTypeFuelAge.class_eval do
       update_all "annual_distance_units = 'kilometres'"
     end
     
-    process "Calculate number of vehicles from total travel and total travel percent" do
+    # FIXME TODO maybe make this a method on AutomobileTypeFuelAge?
+    process "Calculate number of vehicles from total travel percent and AutomobileTypeFuelYear" do
       AutomobileTypeFuelYear.run_data_miner!
       connection.execute %{
         UPDATE automobile_type_fuel_ages
-        SET automobile_type_fuel_ages.vehicles =
-          ((SELECT automobile_type_fuel_years.total_travel
-          FROM automobile_type_fuel_years
-          WHERE automobile_type_fuel_years.year =
-          (SELECT max(automobile_type_fuel_years.year) FROM automobile_type_fuel_years)
-          AND automobile_type_fuel_years.type_name = automobile_type_fuel_ages.type_name
-          AND automobile_type_fuel_years.fuel_common_name = automobile_type_fuel_ages.fuel_common_name
-          ) * automobile_type_fuel_ages.total_travel_percent / automobile_type_fuel_ages.annual_distance )
+        SET vehicles =
+          (
+            (
+              SELECT automobile_type_fuel_years.total_travel
+              FROM automobile_type_fuel_years
+              WHERE automobile_type_fuel_years.`year` =
+              (SELECT max(automobile_type_fuel_years.`year`) FROM automobile_type_fuel_years)
+              AND automobile_type_fuel_years.`type_name` = automobile_type_fuel_ages.`type_name`
+              AND automobile_type_fuel_years.fuel_common_name = automobile_type_fuel_ages.fuel_common_name
+            ) *
+            automobile_type_fuel_ages.total_travel_percent / automobile_type_fuel_ages.annual_distance
+          )
       }
     end
     
@@ -145,7 +163,7 @@ AutomobileTypeFuelAge.class_eval do
       AutomobileTypeFuelAge.all.each do |record|
         %w{ type_name fuel_common_name }.each do |attribute|
           value = record.send(:"#{attribute}")
-          if value.nil?
+          unless value.present?
             raise "Missing #{attribute} for AutomobileTypeFuelAge '#{record.name}'"
           end
         end