earth 0.4.4 → 0.4.5

data/README.markdown

@@ -8,7 +8,7 @@ The data that these models represent can be pulled from http://data.brighterplan
 
     require 'earth'
     Earth.init :automobile, :locality
-    ft = AutomobileFuelType.first
+    ft = AutomobileFuel.first
     ...
 
 You can also run data imports via the data_miner gem.

data/Rakefile

@@ -5,23 +5,51 @@ Bundler::GemHelper.install_tasks
 require 'bueller'
 Bueller::Tasks.new
 
-begin
-  require 'rake/rdoctask'
-  Rake::RDocTask.new do |rdoc|
-    rdoc.rdoc_dir = 'rdoc'
-    rdoc.title = 'emitter'
-    rdoc.options << '--line-numbers' << '--inline-source'
-    rdoc.rdoc_files.include('README*')
-    rdoc.rdoc_files.include('lib/**/*.rb')
+task :console do
+  Earth.init :all
+
+  require 'irb'
+  ARGV.clear
+  IRB.start
+end
+
+desc 'Run all cucumber tests'
+Cucumber::Rake::Task.new(:features) do |t|
+  if ENV['CUCUMBER_FORMAT']
+    t.cucumber_opts = "features --format #{ENV['CUCUMBER_FORMAT']}"
+  else
+    t.cucumber_opts = 'features --format pretty'
   end
-rescue LoadError
-   puts "Rdoc is not available"
 end
 
-require 'sniff'
-require 'sniff/rake_tasks'
-Sniff::RakeTasks.define_tasks do |t|
-  t.earth_domains = :all
-  t.cucumber = true
-  t.rspec = true
+desc "Run cucumber tests with RCov"
+Cucumber::Rake::Task.new(:features_with_coverage) do |t|
+  t.cucumber_opts = "features --format pretty"
+  t.rcov = true
+  t.rcov_opts = ['--exclude', 'features']
+end
+
+require 'rspec/core/rake_task'
+
+desc "Run all examples"
+RSpec::Core::RakeTask.new(:examples) do |c|
+  c.rspec_opts = '-Ispec'
+end
+
+desc "Run specs with RCov"
+RSpec::Core::RakeTask.new(:examples_with_coverage) do |t|
+  t.rcov = true
+  t.rcov_opts = ['--exclude', 'spec']
+  t.rspec_opts = '-Ispec'
+end
+
+task :test => [:features, :examples]
+task :default => :test
+
+require 'earth/version'
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "#{gemname} #{Earth::VERSION}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
 end

data/earth.gemspec

@@ -4,8 +4,8 @@ require "earth/version"
 
 Gem::Specification.new do |s|
   s.name        = "earth"
-  s.version = "0.4.4"
-  s.date = "2011-02-25"
+  s.version     = Earth::VERSION
+  s.date = "2011-03-04"
   s.platform    = Gem::Platform::RUBY
   s.authors = ["Seamus Abshere", "Derek Kastner", "Andy Rossmeissl"]
   s.email = %q{andy@rossmeissl.net}
@@ -28,14 +28,10 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.7}
 
   s.add_runtime_dependency 'activerecord', '~> 3'
+  s.add_runtime_dependency 'cohort_scope', '>= 0.0.7'
   s.add_runtime_dependency 'data_miner', '~> 1.1.1'
-  s.add_runtime_dependency 'errata'
   s.add_runtime_dependency 'falls_back_on', '>= 0.0.3'
-  s.add_runtime_dependency 'geokit', '>= 1.5.0'
   s.add_runtime_dependency 'geokit-rails'
-  s.add_runtime_dependency 'cohort_scope', '>= 0.0.7'
-  s.add_runtime_dependency 'conversions', '>= 1.4.5'
-  s.add_runtime_dependency 'weighted_average', '>= 0.0.4'
   s.add_runtime_dependency 'loose_tight_dictionary', '>= 0.0.8'
-  s.add_development_dependency 'sniff', '~> 0.6.1'
+  s.add_runtime_dependency 'weighted_average', '>= 0.0.4'
 end

data/features/automobile_make_model_year_variant.feature

@@ -9,9 +9,9 @@ Feature: Data import for AutomobileMakeModelYearVariant
     Then the verification should be successful
 
   # FIXME TODO
-  # Scenario: Successfully verifying that fuel type code is found in AutomobileFuelType
+  # Scenario: Successfully verifying that fuel type code is found in AutomobileFuel
   #   Given a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_good"
-  #   When a data import verifies "Fuel type code should be found in AutomobileFuelType"
+  #   When a data import verifies "Fuel type code should be found in AutomobileFuel"
   #   Then the verification should be successful
 
   Scenario: Successfully verifying that fuel efficiencies are greater than zero
@@ -30,9 +30,9 @@ Feature: Data import for AutomobileMakeModelYearVariant
     Then the verification should not be successful
 
   # FIXME TODO
-  # Scenario: Failing to verify that fuel type code is found in AutomobileFuelType
+  # Scenario: Failing to verify that fuel type code is found in AutomobileFuel
   #   Given a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_bad"
-  #   When a data import verifies "Fuel type code should be found in AutomobileFuelType"
+  #   When a data import verifies "Fuel type code should be found in AutomobileFuel"
   #   Then the verification should not be successful
 
   Scenario: Failing to verify that fuel efficiencies are greater than zero

data/features/step_definitions/data_steps.rb

@@ -0,0 +1,32 @@
+Given /^a "([^\"]*)" data import fetches results listed in "(.*)"$/ do |model, file_name|
+  @data_import_model = model.constantize
+  table_name = @data_import_model.table_name
+
+  fixture_path = File.expand_path("features/support/imports/#{file_name}", Dir.pwd) 
+
+  fixture = Fixtures.new @data_import_model.connection, table_name, model, fixture_path
+
+  @data_import_model.connection.transaction(:requires_new => true) do
+    fixture.delete_existing_fixtures
+    fixture.insert_fixtures
+  end
+end
+
+When /^a data import verifies "(.*)"$/ do |verification_step_name|
+  @verification = @data_import_model.data_miner_config.steps.find { |f|
+    f.respond_to?(:description) and f.description == verification_step_name
+  }
+  @verification_result = begin
+                           @verification.run
+                           true
+                         rescue => e
+                           false
+                         end
+end
+
+Then /^the verification should be successful$/ do
+  @verification_result.should be_true
+end
+Then /^the verification should not be successful$/ do
+  @verification_result.should be_false
+end

data/features/support/env.rb

@@ -4,7 +4,10 @@ require 'bundler'
 Bundler.setup
 
 require 'data_miner'
-DataMiner.logger = Logger.new(nil)
+DataMiner.logger = Logger.new nil
 
-require 'sniff'
-Sniff.init File.expand_path('../..', File.dirname(__FILE__)), :earth => :all, :cucumber => true
+require 'active_record'
+require 'sqlite3'
+ActiveRecord::Base.logger = Logger.new nil
+ActiveRecord::Base.establish_connection :adapter => 'sqlite3', :database => ':memory:'
+Earth.init :all, :apply_schemas => true

data/features/support/imports/automobile_make_model_year_variant_bad.csv

@@ -1,3 +1,3 @@
-row_hash,year,fuel_type_code,fuel_efficiency,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_units,fuel_efficiency_city_units,fuel_efficiency_highway_units
+row_hash,year,fuel_code,fuel_efficiency,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_units,fuel_efficiency_city_units,fuel_efficiency_highway_units
 a1,2000,D,12.0,10.0,15.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
 b2,2011,P,0,-10,0,foo,bar

data/features/support/imports/automobile_make_model_year_variant_good.csv

@@ -1,3 +1,3 @@
-row_hash,year,fuel_type_code,fuel_efficiency,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_units,fuel_efficiency_city_units,fuel_efficiency_highway_units
+row_hash,year,fuel_code,fuel_efficiency,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_units,fuel_efficiency_city_units,fuel_efficiency_highway_units
 a1,2000,D,12.0,10.0,15.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
 b2,2010,X,20.5,20.0,21.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre

data/lib/earth/air/aircraft/data_miner.rb

@@ -108,7 +108,7 @@ Aircraft.class_eval do
     end
     
     import "a curated list of aircraft",
-            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDJFblR4MDE1RGtnLVM1S2JHRGZpT3c&hl=en&single=true&gid=0&output=csv' do
+            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDJFblR4MDE1RGtnLVM1S2JHRGZpT3c&hl=en&gid=0&output=csv' do
       key 'bp_code'
       store 'icao_code',              :nullify => true
       store 'bts_code',               :nullify => true

data/lib/earth/air/aircraft_class/data_miner.rb

@@ -11,7 +11,7 @@ AircraftClass.class_eval do
     end
     
     import "a list of Brighter Planet-defined aircraft classes",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGNBbHFibmxJUFprQkUwZHp6VU51Smc&hl=en&single=true&gid=0&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGNBbHFibmxJUFprQkUwZHp6VU51Smc&hl=en&gid=0&output=csv' do
       key 'code', :field_name => 'aircraft_class_code'
       store 'name'
     end

data/lib/earth/air/flight_distance_class/data_miner.rb

@@ -7,7 +7,7 @@ FlightDistanceClass.class_eval do
     end
     
     import "a list of distance classes taken from the WRI business travel tool and UK DEFRA/DECC GHG Conversion Factors for Company Reporting",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdFBKM0xWaUhKVkxDRmdBVkE3VklxY2c&hl=en&single=true&gid=0&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdFBKM0xWaUhKVkxDRmdBVkE3VklxY2c&hl=en&gid=0&output=csv' do
       key   'name'
       store 'distance', :units_field_name => 'distance_units'
     end

data/lib/earth/air/flight_seat_class/data_miner.rb

@@ -7,8 +7,8 @@ FlightSeatClass.class_eval do
       float    'multiplier'
     end
     
-    import "a list of Brighter Planet-defined seat classes and pre-calculated multipliers",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdG5zSER1QmFVVkhNcTM2cmhCMEJtWVE&hl=en&single=true&gid=0&output=csv' do
+    import "seat classes used in the WRI GHG Protocol calculation tools",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdG5zSER1QmFVVkhNcTM2cmhCMEJtWVE&hl=en&gid=0&output=csv' do
       key   'name'
       store 'distance_class_name'
       store 'seat_class_name'

data/lib/earth/automobile/automobile_fuel/data_miner.rb

@@ -20,10 +20,12 @@ AutomobileFuel.class_eval do
       string 'n2o_emission_factor_units'
       float  'hfc_emission_factor'
       string 'hfc_emission_factor_units'
+      float  'emission_factor' # DEPRECATED but motorcycle needs this
+      string 'emission_factor_units' # DEPRECATED but motorcycle needs this
     end
     
     import "a list of pure automobile fuels",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdE9xTEdueFM2R0diNTgxUlk1QXFSb2c&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdE9xTEdueFM2R0diNTgxUlk1QXFSb2c&gid=0&output=html' do
       key   'name'
       store 'code'
       store 'base_fuel_name'
@@ -32,7 +34,7 @@ AutomobileFuel.class_eval do
     end
     
     import "a list of blended automobile fuels",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEswNGIxM0U4U0N1UUppdWw2ejJEX0E&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEswNGIxM0U4U0N1UUppdWw2ejJEX0E&gid=0&output=csv' do
       key   'name'
       store 'code'
       store 'base_fuel_name'
@@ -96,6 +98,11 @@ AutomobileFuel.class_eval do
       end
     end
     
+    process "Derive emission factor" do
+      update_all "emission_factor = co2_emission_factor + ch4_emission_factor + n2o_emission_factor + hfc_emission_factor,
+                  emission_factor_units = 'kilograms_co2e_per_litre'"
+    end
+    
     # FIXME TODO verify code somehow
     
     %w{ base_fuel_name distance_key ef_key }.each do |attribute|

data/lib/earth/automobile/automobile_make/data_miner.rb

@@ -23,7 +23,7 @@ AutomobileMake.class_eval do
     # sabshere 1/31/11 add Avanti, DaimlerChrysler, IHC, Tesla, etc.
     process "Derive extra manufacturer names from CAFE data" do
       AutomobileMakeFleetYear.run_data_miner!
-      INSERT IGNORE %{INTO automobile_makes(name)
+      INSERT_IGNORE %{INTO automobile_makes(name)
         SELECT DISTINCT automobile_make_fleet_years.make_name
         FROM automobile_make_fleet_years
       }

data/lib/earth/automobile/automobile_make_fleet_year/data_miner.rb

@@ -12,7 +12,7 @@ AutomobileMakeFleetYear.class_eval do
     end
     
     import "annual corporate average fuel economy data for domestic and imported vehicle fleets from the NHTSA",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEdXWXB6dkVLWkowLXhYSFVUT01sS2c&hl=en&single=true&gid=0&output=csv',
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEdXWXB6dkVLWkowLXhYSFVUT01sS2c&hl=en&gid=0&output=csv',
            :errata => { 'url' => 'http://static.brighterplanet.com/science/data/transport/automobiles/make_fleet_years/errata.csv' },
            :select => lambda { |row| row['volume'].to_i > 0 } do
       key   'name', :synthesize => lambda { |row| [ row['manufacturer_name'], row['year_content'], row['fleet'][2,2] ].join ' ' }

data/lib/earth/automobile/automobile_make_model_year_variant/data_miner.rb

@@ -271,7 +271,6 @@ AutomobileMakeModelYearVariant.class_eval do
       string   'fuel_efficiency_city_units'
       float    'fuel_efficiency_highway'
       string   'fuel_efficiency_highway_units'
-      string   'fuel_type_code'
       string   'fuel_code'
       string   'transmission'
       string   'drive'
@@ -315,7 +314,6 @@ AutomobileMakeModelYearVariant.class_eval do
         store 'name', :field_name => 'model'
         store 'make_name', :field_name => 'make'
         store 'year'
-        store 'fuel_type_code', :field_name => 'fuel_type'
         store 'fuel_code', :field_name => 'fuel_type'
         store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
         store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
@@ -354,7 +352,6 @@ AutomobileMakeModelYearVariant.class_eval do
         key   'row_hash'
         store 'name', :field_name => 'model'
         store 'make_name', :field_name => 'make'
-        store 'fuel_type_code', :field_name => 'fl'
         store 'fuel_code', :field_name => 'fl'
         store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
         store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
@@ -389,7 +386,6 @@ AutomobileMakeModelYearVariant.class_eval do
         key   'row_hash'
         store 'name', :field_name => 'model'
         store 'make_name', :field_name => 'make'
-        store 'fuel_type_code', :field_name => 'FUEL TYPE'
         store 'fuel_code', :field_name => 'FUEL TYPE'
         store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
         store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
@@ -420,7 +416,6 @@ AutomobileMakeModelYearVariant.class_eval do
         key   'row_hash'
         store 'name', :field_name => 'model'
         store 'make_name', :field_name => 'make'
-        store 'fuel_type_code'
         store 'fuel_code', :field_name => 'fuel_type_code'
         store 'fuel_efficiency_highway', :static => nil, :units => :kilometres_per_litre
         store 'fuel_efficiency_city', :static => nil, :units => :kilometres_per_litre
@@ -479,17 +474,8 @@ AutomobileMakeModelYearVariant.class_eval do
       end
     end
     
-    # FIXME TODO
-    # verify "Fuel type code should be found in AutomobileFuelType" do
-    #   valid_codes = AutomobileFuelType.all.map(&:code)
-    #   puts valid_codes
-    #   AutomobileMakeModelVariant.all.each do |variant|
-    #     unless valid_codes.include? variant.fuel_type_code
-    #       raise "Invalid fuel type code for AutomobileMakeModelYearVariant #{variant.row_hash}: #{variant.fuel_type_code} (should be found in AutomobileFuelType)"
-    #     end
-    #   end
-    # end
-    # 
+    # FIXME TODO verify fuel code appears in AutomobileFuel
+    
     verify "Fuel efficiencies should be greater than zero" do
       AutomobileMakeModelYearVariant.all.each do |variant|
         [:fuel_efficiency, :fuel_efficiency_city, :fuel_efficiency_highway].each do |type|

data/lib/earth/automobile/automobile_make_model_year_variant.rb

@@ -4,7 +4,6 @@ class AutomobileMakeModelYearVariant < ActiveRecord::Base
   belongs_to :make,            :class_name => 'AutomobileMake',          :foreign_key => 'make_name'
   belongs_to :make_model,      :class_name => 'AutomobileMakeModel',     :foreign_key => 'make_model_name'
   belongs_to :make_model_year, :class_name => 'AutomobileMakeModelYear', :foreign_key => 'make_model_year_name'
-  belongs_to :fuel_type,       :class_name => 'AutomobileFuelType',      :foreign_key => 'fuel_type_code'
   belongs_to :fuel,            :class_name => 'AutomobileFuel',          :foreign_key => 'fuel_code', :primary_key => 'code'
   
   data_miner do

data/lib/earth/automobile/automobile_size_class/data_miner.rb

@@ -17,7 +17,7 @@ AutomobileSizeClass.class_eval do
     
     # FIXME TODO pull this from AutomobileSizeClassYear
     import "a list of size classes and pre-calculated fuel efficiencies",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHlRUE5IcWlrRENhN0EtUldPTy1rX1E&single=true&gid=0&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHlRUE5IcWlrRENhN0EtUldPTy1rX1E&gid=0&output=csv' do
       key 'name'
       store 'type_name'
       store 'fuel_efficiency_city', :units_field_name => 'fuel_efficiency_city_units'
@@ -35,7 +35,7 @@ AutomobileSizeClass.class_eval do
     end
     
     import "pre-calculated fuel efficiency multipliers",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGt2NnhXLXUxNFRJSzczU3BkSHB3enc&hl=en&single=true&gid=0&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGt2NnhXLXUxNFRJSzczU3BkSHB3enc&hl=en&gid=0&output=csv' do
       key 'name'
       store 'hybrid_fuel_efficiency_city_multiplier'
       store 'hybrid_fuel_efficiency_highway_multiplier'

data/lib/earth/automobile/automobile_size_class_year/data_miner.rb

@@ -12,7 +12,7 @@ AutomobileSizeClassYear.class_eval do
     end
     
     import "automobile size class year fuel efficiencies from the 2010 EPA Fuel Economy Trends report",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDZOLWdDdlZja04xZkJYc3NjeGxQamc&hl=en&single=true&gid=0&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDZOLWdDdlZja04xZkJYc3NjeGxQamc&hl=en&gid=0&output=csv' do
       key 'name'
       store 'size_class_name'
       store 'year'

data/lib/earth/automobile/automobile_type_fuel_control/data_miner.rb

@@ -12,7 +12,7 @@ AutomobileTypeFuelControl.class_eval do
     end
     
     import "automobile type fuel control data derived from the 2010 EPA GHG Inventory",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEloSTU5YUNOUXRFRUcxWHlTUi1GMkE&hl=en&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEloSTU5YUNOUXRFRUcxWHlTUi1GMkE&hl=en&gid=0&output=csv' do
       key   'name'
       store 'type_name'
       store 'fuel_common_name'

data/lib/earth/automobile/automobile_type_fuel_year/data_miner.rb

@@ -75,7 +75,7 @@ AutomobileTypeFuelYear.class_eval do
     end
     
     import "fuel consumption derived from the 2010 EPA GHG Inventory",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHBCMFhLRTFTZENsd0dPUGUyYlJna0E&hl=en&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHBCMFhLRTFTZENsd0dPUGUyYlJna0E&hl=en&gid=0&output=csv' do
       key 'name'
       store 'fuel_consumption', :units_field_name => 'fuel_consumption_units'
     end

data/lib/earth/automobile/automobile_type_fuel_year_control/data_miner.rb

@@ -12,7 +12,7 @@ AutomobileTypeFuelYearControl.class_eval do
     end
     
     import "automobile type fuel year control data derived from the 2010 EPA GHG Inventory",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGpQV2xMdlZkV1JzVlVTeU5ZalF6elE&hl=en&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGpQV2xMdlZkV1JzVlVTeU5ZalF6elE&hl=en&gid=0&output=csv' do
       key   'name'
       store 'type_name'
       store 'fuel_common_name'

data/lib/earth/automobile/automobile_type_year/data_miner.rb

@@ -11,7 +11,7 @@ AutomobileTypeYear.class_eval do
     end
     
     import "automobile type year air conditioning emissions derived from the 2010 EPA GHG Inventory",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdFoyTWhDeHpndTV5Ny1aX0sxR1ljSFE&hl=en&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdFoyTWhDeHpndTV5Ny1aX0sxR1ljSFE&hl=en&gid=0&output=csv' do
       key   'name'
       store 'type_name'
       store 'year'

data/lib/earth/automobile/data_miner.rb

@@ -1,5 +1,4 @@
 require 'earth/automobile/automobile_fuel/data_miner'
-require 'earth/automobile/automobile_fuel_type/data_miner'
 require 'earth/automobile/automobile_make/data_miner'
 require 'earth/automobile/automobile_make_fleet_year/data_miner'
 require 'earth/automobile/automobile_make_year/data_miner'

data/lib/earth/automobile.rb

@@ -1,5 +1,4 @@
 require 'earth/automobile/automobile_fuel'
-require 'earth/automobile/automobile_fuel_type'
 require 'earth/automobile/automobile_make'
 require 'earth/automobile/automobile_make_fleet_year'
 require 'earth/automobile/automobile_make_year'

data/lib/earth/bus/bus_class/data_miner.rb

@@ -30,7 +30,7 @@ BusClass.class_eval do
     end
     
     import "a list of bus classes and pre-calculated trip and fuel use characteristics",
-           :url => 'https://spreadsheets.google.com/pub?hl=en&hl=en&key=0AoQJbWqPrREqdGRsSnJoS1hraGJvR012cDROWXFPbVE&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?hl=en&hl=en&key=0AoQJbWqPrREqdGRsSnJoS1hraGJvR012cDROWXFPbVE&gid=0&output=csv' do
       key   'name'
       store 'distance', :units_field_name => 'distance_units'
       store 'passengers'

data/lib/earth/bus/bus_fuel/data_miner.rb

@@ -22,13 +22,13 @@ BusFuel.class_eval do
     end
     
     import "a list of bus fuels without emission factors",
-           :url => 'https://spreadsheets.google.com/pub?hl=en&hl=en&key=0AoQJbWqPrREqdGNscUdpbkdBUmJkTlFxdEtfSEdJS2c&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGNscUdpbkdBUmJkTlFxdEtfSEdJS2c&gid=0&output=csv' do
       key 'name'
       store 'fuel_name'
     end
     
     import "a list of bus fuels and their emission factors",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHAtZ1YyWmxuS1JWUzZYSVhhdUViYmc&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHAtZ1YyWmxuS1JWUzZYSVhhdUViYmc&gid=0&output=csv' do
       key   'name'
       store 'fuel_name'
       store 'ch4_emission_factor', :units_field_name => 'ch4_emission_factor_units'
@@ -36,7 +36,7 @@ BusFuel.class_eval do
     end
     
     import "energy contents for compressed/liquified natural gas fuels",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDFfdHMycVB6dTkyUFFNM0x4b19TQVE&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDFfdHMycVB6dTkyUFFNM0x4b19TQVE&gid=0&output=csv' do
       key   'name'
       store 'energy_content', :units_field_name => 'energy_content_units'
     end

data/lib/earth/bus/bus_fuel_control/data_miner.rb

@@ -11,7 +11,7 @@ BusFuelControl.class_eval do
     end
     
     import "a list of bus fuel controls",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEw1QW80VVJhaXRkUURQSFhHczNyVWc&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEw1QW80VVJhaXRkUURQSFhHczNyVWc&gid=0&output=csv' do
       key   'name'
       store 'bus_fuel_name'
       store 'control'

data/lib/earth/bus/bus_fuel_year_control/data_miner.rb

@@ -14,7 +14,7 @@ BusFuelYearControl.class_eval do
     end
     
     import "a list of bus fuel year controls",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGhHQkZPZW4zbXYzZ3NkYThBWnQ2QXc&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGhHQkZPZW4zbXYzZ3NkYThBWnQ2QXc&gid=0&output=csv' do
       key   'name'
       store 'bus_fuel_name'
       store 'year'

data/lib/earth/computation/computation_platform/data_miner.rb

@@ -6,7 +6,7 @@ ComputationPlatform.class_eval do
     end
     
     import "a list of computation platforms and the data center company they use",
-           :url => 'https://spreadsheets.google.com/pub?key=0AkCJNpm9Ks6JdG5zVER5THhXRlE4N0RXTklVaS03Ymc&single=true&gid=0&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AkCJNpm9Ks6JdG5zVER5THhXRlE4N0RXTklVaS03Ymc&gid=0&gid=0&output=csv' do
       key   'name'
       store 'data_center_company_name'
     end

data/lib/earth/computation/data_center_company/data_miner.rb

@@ -6,7 +6,7 @@ DataCenterCompany.class_eval do
     end
     
     import "a list of data center companies and their power usage effectiveness",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdFlxZVZLai01WWJOOTFiaUd1blF6VkE&hl=en&single=true&gid=0&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdFlxZVZLai01WWJOOTFiaUd1blF6VkE&hl=en&gid=0&output=csv' do
       key   'name'
       store 'power_usage_effectiveness'
     end

data/lib/earth/computation/server_type/data_miner.rb

@@ -12,7 +12,7 @@ ServerType.class_eval do
     end
     
     import "a list of server types and their characteristics",
-           :url => 'https://spreadsheets.google.com/pub?key=0AkCJNpm9Ks6JdGExaW1ic2c0d2s1ZmpzeUdOa2kyRlE&hl=en&single=true&gid=0&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AkCJNpm9Ks6JdGExaW1ic2c0d2s1ZmpzeUdOa2kyRlE&hl=en&gid=0&output=csv' do
       key   'name'
       store 'data_center_company_name'
       store 'cpu',              :units_field_name => 'cpu_units'

data/lib/earth/computation/server_type_alias/data_miner.rb

@@ -7,7 +7,7 @@ ServerTypeAlias.class_eval do
     end
     
     import "a list of server type aliases and their characteristics",
-           :url => 'https://spreadsheets.google.com/pub?key=0AkCJNpm9Ks6JdDdkVlBzSUYweFN0OHBreEN6YTdTZ3c&hl=en&single=true&gid=0&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AkCJNpm9Ks6JdDdkVlBzSUYweFN0OHBreEN6YTdTZ3c&hl=en&gid=0&output=csv' do
       key   'name', :field_name => 'alias'
       store 'server_type_name'
       store 'platform_name'

data/lib/earth/diet/food_group/data_miner.rb

@@ -16,7 +16,7 @@ FoodGroup.class_eval do
     end
     
     import "energy and units",
-           :url => 'http://spreadsheets.google.com/pub?key=trkBEq5oFnUQE76gQ6VpQgw' do
+           :url => 'http://spreadsheets.google.com/pub?key=trkBEq5oFnUQE76gQ6VpQgw&gid=0&output=csv' do
       key   'name'
       store 'energy', :units => 'unknown' #FIXME this needs to be "kilocalories per" and then whatever the suggested imperial measurement is
       store 'suggested_imperial_measurement', :field_name => 'friendly_units'

data/lib/earth/fuel/fuel/data_miner.rb

@@ -24,7 +24,7 @@ Fuel.class_eval do
     end
     
     import "fuels with non-variable characteristics",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGJmYkdtajZyV3Byb0lrd21xLVhXUGc&hl=en&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGJmYkdtajZyV3Byb0lrd21xLVhXUGc&hl=en&gid=0&output=csv' do
       key 'name'
       store 'energy_content', :units_field_name => 'energy_content_units'
       store 'carbon_content', :units_field_name => 'carbon_content_units'
@@ -33,7 +33,7 @@ Fuel.class_eval do
     end
     
     import "densities for aircraft fuels",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHBjTVE4NmRlc05iUHVZR1E3eEJwOGc&hl=en&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHBjTVE4NmRlc05iUHVZR1E3eEJwOGc&hl=en&gid=0&output=csv' do
       key 'name'
       store 'density', :units_field_name => 'density_units'
     end

data/lib/earth/fuel/fuel_price/data_miner.rb

@@ -11,7 +11,7 @@ FuelPrice.class_eval do
     end
     
     import 'fuel prices derived from the EIA',
-           :url => 'http://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHlSdXJoOFB5aEpHenJQbTVJdS1pMVE',
+           :url => 'http://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHlSdXJoOFB5aEpHenJQbTVJdS1pMVE&gid=0&output=csv',
            :select => lambda { |row| row['fuel_type_name'].present? } do
       key   'name', :field_name => 'fuel_type_name'
       store 'price', :units_field_name => 'price_units'

data/lib/earth/fuel/fuel_type/data_miner.rb

@@ -28,7 +28,7 @@ FuelType.class_eval do
     # end
     
     import "a list of fuels and their emission factors and densities",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDR3RjlTcWlsLTc2TzQ0cERTMElJbHc&single=true&gid=0&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDR3RjlTcWlsLTc2TzQ0cERTMElJbHc&gid=0&output=csv' do
       key 'name', :field_name => 'fuel'
       store 'emission_factor', :units_field_name => 'emission_factor_units'
       store 'average_purchase_volume', :units_field_name => 'average_purchase_volume_units'

data/lib/earth/fuel/fuel_year/data_miner.rb

@@ -17,7 +17,7 @@ FuelYear.class_eval do
     end
     
     import "fuels with annually variable characteristics, derived from the 2010 EPA GHG Inventory",
-           :url => 'https://spreadsheets.google.com/pub?hl=en&hl=en&key=0AoQJbWqPrREqdFZVSlZ3SUZsTzZLVTB5bVk5THdBN2c&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?hl=en&hl=en&key=0AoQJbWqPrREqdFZVSlZ3SUZsTzZLVTB5bVk5THdBN2c&gid=0&output=csv' do
       key 'name'
       store 'fuel_name'
       store 'year'

data/lib/earth/fuel/greenhouse_gas/data_miner.rb

@@ -10,7 +10,7 @@ GreenhouseGas.class_eval do
     end
     
     import "greenhouse gas global warming potentials taken from the IPCC AR4",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdE1tUjBwS1ZHZnBUUG1XcVpkOTVHOVE&single=true&gid=0&output=csv' do
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdE1tUjBwS1ZHZnBUUG1XcVpkOTVHOVE&gid=0&output=csv' do
       key 'name'
       store 'abbreviation'
       store 'ipcc_report'