earth-ruby19 0.2.3

data/lib/earth/all.rb

@@ -0,0 +1,11 @@
+require 'earth/air'
+require 'earth/automobile'
+require 'earth/bus'
+require 'earth/diet'
+require 'earth/fuel'
+require 'earth/hospitality'
+require 'earth/industry'
+require 'earth/locality'
+require 'earth/pet'
+require 'earth/rail'
+require 'earth/residence'

data/lib/earth/automobile.rb

@@ -0,0 +1,8 @@
+require 'earth/automobile/automobile_fuel_type'
+require 'earth/automobile/automobile_make'
+require 'earth/automobile/automobile_make_fleet_year'
+require 'earth/automobile/automobile_make_year'
+require 'earth/automobile/automobile_model'
+require 'earth/automobile/automobile_model_year'
+require 'earth/automobile/automobile_size_class'
+require 'earth/automobile/automobile_variant'

data/lib/earth/automobile/automobile_fuel_type.rb

@@ -0,0 +1,18 @@
+class AutomobileFuelType < ActiveRecord::Base
+  set_primary_key :code
+  
+  has_many :automobiles, :foreign_key => 'fuel_type_id'
+  
+  scope :ordered, :order => 'name'
+  
+  falls_back_on :emission_factor => 20.781.pounds_per_gallon.to(:kilograms_per_litre) # https://brighterplanet.sifterapp.com/projects/30/issues/428
+  
+  data_miner do
+    tap "Brighter Planet's sanitized automobile fuel type data", Earth.taps_server
+  end
+  
+  CODES = {
+    :electricity => 'El',
+    :diesel => 'D'
+  }
+end

data/lib/earth/automobile/automobile_fuel_type/data_miner.rb

@@ -0,0 +1,45 @@
+AutomobileFuelType.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'code'
+      string   'name'
+      float    'emission_factor'
+      string   'emission_factor_units'
+      float    'annual_distance'
+      string   'annual_distance_units'
+    end
+    
+    import("the fuels used in the 2000 EPA fuel economy guide",
+           :url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
+           :filename => 'Gd6-dsc.txt',
+           :format => :fixed_width,
+           :crop => 21..26, # inclusive
+           :cut => '2-',
+           :select => lambda { |row| /\A[A-Z]/.match row[:code] },
+           :schema => [[ 'code',   2,  { :type => :string }  ],
+                       [ 'spacer', 2 ],
+                       [ 'name',   52, { :type => :string } ]]) do
+      key   'code'
+      store 'name'
+    end
+
+    import "a pre-calculated emission factor and average annual distance for each fuel",
+           :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/models_export/automobile_fuel_type.csv' do
+      key   'code', :field_name => 'code'
+      store 'name'
+      store 'annual_distance', :units => :kilometres
+      store 'emission_factor', :units => :kilograms_per_litre
+    end
+
+    # pull electricity emission factor from residential electricity
+    import "a pre-calculated emission factor for electricity",
+           :url => 'http://spreadsheets.google.com/pub?key=rukxnmuhhsOsrztTrUaFCXQ',
+           :select => lambda { |row| row['name'] == 'electricity' } do
+      key 'name'
+      store 'emission_factor', :units => :kilograms_per_litre
+    end
+    
+    # still need distance estimate for electric cars
+  end
+end
+

data/lib/earth/automobile/automobile_make.rb

@@ -0,0 +1,14 @@
+class AutomobileMake < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :make_years,   :class_name => 'AutomobileMakeYear',      :foreign_key => 'make_name'
+  has_many :models,       :class_name => 'AutomobileModel',         :foreign_key => 'make_name'
+  has_many :fleet_years,  :class_name => 'AutomobileMakeFleetYear', :foreign_key => 'make_name'
+  has_many :variants,     :class_name => 'AutomobileVariant',       :foreign_key => 'make_name'
+  
+  scope :major, :conditions => { :major => true }, :order => :name
+
+  data_miner do
+    tap "Brighter Planet's make year data", Earth.taps_server
+  end
+end

data/lib/earth/automobile/automobile_make/data_miner.rb

@@ -0,0 +1,68 @@
+AutomobileMake.class_eval do
+  data_miner do
+    process "Start from scratch" do
+      connection.drop_table table_name
+    end
+    
+    schema Earth.database_options do
+      string  'name'
+      boolean 'major'
+      float   'fuel_efficiency'
+      string  'fuel_efficiency_units'
+    end
+
+    process "Derive automobile makes from automobile variants" do
+      AutomobileVariant.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO automobile_makes(name)
+        SELECT automobile_variants.make_name FROM automobile_variants WHERE LENGTH(automobile_variants.make_name) > 0
+      }
+    end
+    
+    import "a Brighter-Planet defined list of major automobile manufacturers",
+           :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/makes/make_importance.csv' do
+      key 'name'
+      store 'major'
+    end
+    
+    process "Derive average fuel efficiency from automobile makes and years" do
+      AutomobileMakeYear.run_data_miner!
+      make_years = AutomobileMakeYear.arel_table
+      makes = AutomobileMake.arel_table
+      conditional_relation = makes[:name].eq(make_years[:make_name])
+      relation = AutomobileMakeYear.weighted_average_relation(:fuel_efficiency, :weighted_by => :volume).where(conditional_relation)
+      update_all "fuel_efficiency = (#{relation.to_sql})"
+      update_all "fuel_efficiency_units = 'kilometres_per_litre'"
+    end
+  end
+end
+
+# leave this for later if we need it
+# SUBSIDIARIES = {
+#   'Chevrolet' => 'GM',
+#   'Pontiac' => 'GM',
+#   'Audi' => 'Volkswagen',
+#   'Dodge' => 'Chrysler',
+#   'Lincoln' => 'Ford',
+#   'Plymouth' => 'Chrysler',
+#   'Buick' => 'GM',
+#   'Cadillac' => 'GM',
+#   'Merkur' => 'Ford',
+#   'Oldsmobile' => 'GM',
+#   'GMC' => 'GM',
+#   'Bentley' => 'Rolls-Royce', # currently owned by Volkswagen, but a Flying Spur is hardly a rebranded Passat
+#   'Acura' => 'Honda',
+#   'Land Rover' => 'Ford',
+#   'Eagle' => 'Chrysler',
+#   'Geo' => 'GM',
+#   'Laforza' => 'Ford',
+#   'Infiniti' => 'Nissan',
+#   'Lexus' => 'Toyota',
+#   'Saturn' => 'GM',
+#   'Mercury' => 'Ford',
+#   'Alpina' => 'BMW',
+#   'Mini' => 'BMW',
+#   'Maybach' => 'Mercedes',
+#   'Hummer' => 'GM'
+# }
+

data/lib/earth/automobile/automobile_make_fleet_year.rb

@@ -0,0 +1,15 @@
+class AutomobileMakeFleetYear < ActiveRecord::Base
+  set_primary_key :name
+  
+  belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'make_name'
+  # WRONG belongs_to :model_year, :class_name => 'AutomobileModelYear', :foreign_key => 'automobile_model_year_name'
+  belongs_to :make_year, :class_name => 'AutomobileMakeYear', :foreign_key => 'make_year_name'
+
+  data_miner do
+    tap "Brighter Planet's sanitized auto make fleet year data", Earth.taps_server
+    
+    process "bring in dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+end

data/lib/earth/automobile/automobile_make_fleet_year/data_miner.rb

@@ -0,0 +1,29 @@
+AutomobileMakeFleetYear.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'name'
+      string   'make_year_name'
+      string   'make_name'
+      string   'fleet'
+      integer  'year'
+      float    'fuel_efficiency'
+      string   'fuel_efficiency_units'
+      integer  'volume'
+    end
+
+    # CAFE data privately emailed to Andy from Terry Anderson at the DOT/NHTSA
+    import "annual corporate average fuel economy data for domestic and imported vehicle fleets from the NHTSA",
+           :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/make_fleet_years/make_fleet_years.csv',
+           :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/make_fleet_years/errata.csv',
+           :select => lambda { |row| row['volume'].to_i > 0 } do
+      key   'name', :synthesize => lambda { |row| [ row['manufacturer_name'], row['year_content'], row['fleet'][2,2] ].join ' ' }
+      store 'make_year_name', :synthesize => lambda { |row| [ row['manufacturer_name'], row['year_content'] ].join ' ' }
+      store 'make_name', :field_name => 'manufacturer_name'
+      store 'year', :field_name => 'year_content'
+      store 'fleet', :chars => 2..3 # zero-based
+      store 'fuel_efficiency', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
+      store 'volume'
+    end
+  end
+end
+

data/lib/earth/automobile/automobile_make_year.rb

@@ -0,0 +1,14 @@
+class AutomobileMakeYear < ActiveRecord::Base
+  set_primary_key :name
+  
+  belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'make_name'
+  has_many :fleet_years, :class_name => 'AutomobileMakeFleetYear', :foreign_key => 'make_year_name'
+
+  data_miner do
+    tap "Brighter Planet's make year data", Earth.taps_server
+    
+    process "bring in dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+end

data/lib/earth/automobile/automobile_make_year/data_miner.rb

@@ -0,0 +1,45 @@
+AutomobileMakeYear.class_eval do
+  data_miner do
+    process "Start from scratch" do
+      connection.drop_table table_name
+    end
+    
+    schema Earth.database_options do
+      string   'name'
+      string   'make_name'
+      integer  'year'
+      float    'fuel_efficiency'
+      string   'fuel_efficiency_units'
+      integer  'volume'
+    end
+
+    process "Derive manufacturer names and years from automobile make fleet years" do
+      AutomobileMakeFleetYear.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO automobile_make_years(name, make_name, year)
+        SELECT
+          automobile_make_fleet_years.make_year_name,
+          automobile_make_fleet_years.make_name,
+          automobile_make_fleet_years.year
+        FROM automobile_make_fleet_years
+      }
+    end
+    
+    process "Derive annual corporate average fuel economy across all vehicles from automobile make fleet years" do
+      AutomobileMakeFleetYear.run_data_miner!
+      make_fleet_years = AutomobileMakeFleetYear.arel_table
+      make_years = AutomobileMakeYear.arel_table
+      conditional_relation = make_years[:name].eq(make_fleet_years[:make_year_name])
+      relation = AutomobileMakeFleetYear.weighted_average_relation(:fuel_efficiency, :weighted_by => :volume).where(conditional_relation)
+      update_all "fuel_efficiency = (#{relation.to_sql})"
+      update_all "fuel_efficiency_units = 'kilometres_per_litre'"
+    end
+    
+    process "Derive sales volume across all vehicles from automobile make fleet years" do
+      connection.execute %{
+        UPDATE automobile_make_years SET automobile_make_years.volume = (SELECT SUM(automobile_make_fleet_years.volume) FROM automobile_make_fleet_years WHERE automobile_make_fleet_years.make_year_name = automobile_make_years.name)
+      }
+    end
+  end
+end
+

data/lib/earth/automobile/automobile_model.rb

@@ -0,0 +1,14 @@
+class AutomobileModel < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :variants, :class_name => 'AutomobileVariant', :foreign_key => 'model_name'
+  belongs_to :make, :class_name => 'AutomobileMake', :foreign_key => 'make_name'
+  
+  data_miner do
+    tap "Brighter Planet's auto model data", Earth.taps_server
+    
+    process "bring in dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+end

data/lib/earth/automobile/automobile_model/data_miner.rb

@@ -0,0 +1,38 @@
+AutomobileModel.class_eval do
+  data_miner do
+    process "Start from scratch" do
+      connection.drop_table table_name
+    end
+    
+    schema Earth.database_options do
+      string  'name' # make + model
+      string  'make_name'
+      float   'fuel_efficiency_city'
+      string  'fuel_efficiency_city_units'
+      float   'fuel_efficiency_highway'
+      string  'fuel_efficiency_highway_units'
+    end
+
+    process "Derive model names from automobile variants" do
+      AutomobileVariant.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO automobile_models(name, make_name)
+        SELECT automobile_variants.model_name, automobile_variants.make_name FROM automobile_variants WHERE LENGTH(automobile_variants.model_name) > 0 AND LENGTH(automobile_variants.make_name) > 0
+      }
+    end
+    
+    # TODO not weighted until we get weightings on auto variants
+    process "Derive average fuel economy from automobile variants" do
+      AutomobileVariant.run_data_miner!
+      automobile_models = AutomobileModel.arel_table
+      automobile_variants = AutomobileVariant.arel_table
+      conditional_relation = automobile_models[:name].eq(automobile_variants[:model_name])
+      %w{ city highway }.each do |i|
+        relation = AutomobileVariant.where(conditional_relation).where("`automobile_variants`.`fuel_efficiency_#{i}` IS NOT NULL").project("AVG(`automobile_variants`.`fuel_efficiency_#{i}`)")
+        update_all "fuel_efficiency_#{i} = (#{relation.to_sql})"
+        update_all "fuel_efficiency_#{i}_units = 'kilometres_per_litre'"
+      end
+    end
+  end
+end
+

data/lib/earth/automobile/automobile_model_year.rb

@@ -0,0 +1,15 @@
+class AutomobileModelYear < ActiveRecord::Base
+  set_primary_key :name
+  
+  belongs_to :make_year, :class_name => 'AutomobileMakeYear', :foreign_key => 'make_year_name'
+  has_many :variants, :class_name => 'AutomobileVariant', :foreign_key => 'model_year_name'
+  has_many :automobiles, :foreign_key => 'model_year_id'
+  
+  data_miner do
+    tap "Brighter Planet's model year data", Earth.taps_server
+    
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+end

data/lib/earth/automobile/automobile_model_year/data_miner.rb

@@ -0,0 +1,51 @@
+AutomobileModelYear.class_eval do
+  data_miner do
+    process "Start from scratch" do
+      connection.drop_table table_name
+    end
+    
+    schema Earth.database_options do
+      string   'name'
+      string   'make_name'
+      string   'model_name'
+      integer  'year'
+      string   'make_year_name'
+      float    'fuel_efficiency'
+      string   'fuel_efficiency_units'
+      float    'fuel_efficiency_city'
+      string   'fuel_efficiency_city_units'
+      float    'fuel_efficiency_highway'
+      string   'fuel_efficiency_highway_units'
+    end
+
+    process "Derive model year names from automobile variants" do
+      AutomobileVariant.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO automobile_model_years(name, make_name, model_name, year, make_year_name)
+        SELECT automobile_variants.model_year_name, automobile_variants.make_name, automobile_variants.model_name, automobile_variants.year, automobile_variants.make_year_name FROM automobile_variants WHERE LENGTH(automobile_variants.make_name) > 0 AND LENGTH(automobile_variants.model_name) > 0
+      }
+    end
+    
+    # TODO: weight by volume somehow
+    # note that we used to derive averages from make years, but here we get it from variants
+    # even without volume-weighting, the values are much better.
+    # for example, 20km/l for a toyota prius 2006 vs. 13km/l if you use make years
+    process "Calculate city and highway fuel efficiency from automobile variants" do
+      AutomobileVariant.run_data_miner!
+      automobile_model_years = AutomobileModelYear.arel_table
+      automobile_variants = AutomobileVariant.arel_table
+      conditional_relation = automobile_model_years[:name].eq(automobile_variants[:model_year_name])
+      %w{ city highway }.each do |i|
+        relation = AutomobileVariant.where(conditional_relation).where("`automobile_variants`.`fuel_efficiency_#{i}` IS NOT NULL").project("AVG(`automobile_variants`.`fuel_efficiency_#{i}`)")
+        update_all "fuel_efficiency_#{i} = (#{relation.to_sql})"
+        update_all "fuel_efficiency_#{i}_units = 'kilometres_per_litre'"
+      end
+    end
+    
+    process "Calculate overall fuel efficiency using the latest EPA equation" do
+      update_all "fuel_efficiency = 1 / ((0.43 / fuel_efficiency_city) + (0.57 / fuel_efficiency_highway))"
+      update_all "fuel_efficiency_units = 'kilometres_per_litre'"
+    end
+  end
+end
+

data/lib/earth/automobile/automobile_size_class.rb

@@ -0,0 +1,14 @@
+class AutomobileSizeClass < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :automobiles, :foreign_key => 'size_class_id'
+  
+  falls_back_on :hybrid_fuel_efficiency_city_multiplier => 1.651, # https://brighterplanet.sifterapp.com/issue/667
+                :hybrid_fuel_efficiency_highway_multiplier => 1.213,
+                :conventional_fuel_efficiency_city_multiplier => 0.987,
+                :conventional_fuel_efficiency_highway_multiplier => 0.996
+  
+  data_miner do
+    tap "Brighter Planet's sanitized automobile size class data", Earth.taps_server
+  end
+end

data/lib/earth/automobile/automobile_size_class/data_miner.rb

@@ -0,0 +1,43 @@
+AutomobileSizeClass.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'name'
+      float    'fuel_efficiency_city'
+      string   'fuel_efficiency_city_units'
+      float    'fuel_efficiency_highway'
+      string   'fuel_efficiency_highway_units'
+      float    'annual_distance'
+      string   'annual_distance_units'
+      string   'emblem'
+      float    'hybrid_fuel_efficiency_city_multiplier'
+      float    'hybrid_fuel_efficiency_highway_multiplier'
+      float    'conventional_fuel_efficiency_city_multiplier'
+      float    'conventional_fuel_efficiency_highway_multiplier'
+    end
+
+    import "a list of size classes and pre-calculated annual distances and fuel efficiencies",
+           :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/models_export/automobile_size_class.csv' do
+      key 'name'
+      store 'annual_distance', :units => :kilometres
+      store 'fuel_efficiency_city', :units => :kilometres_per_litre
+      store 'fuel_efficiency_highway', :units => :kilometres_per_litre
+    end
+
+    # Ian 5/27/2010 I'm pretty sure we don't need emblems in middleware
+    # import "",
+    #        :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/models_export/automobile_size_class_emblems.csv' do
+    #   key 'name'
+    #   store 'emblem'
+    # end
+    
+    import "pre-calculated hybridity multipliers",
+           :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/vehicle_classes/fuel_efficiency_multipliers.csv' do
+      key 'name'
+      store 'hybrid_fuel_efficiency_city_multiplier'
+      store 'hybrid_fuel_efficiency_highway_multiplier'
+      store 'conventional_fuel_efficiency_city_multiplier'
+      store 'conventional_fuel_efficiency_highway_multiplier'
+    end
+  end
+end
+