earth 0.3.10 → 0.3.11

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    earth (0.3.9)
+    earth (0.3.10)
       activerecord (~> 3)
       cohort_scope (>= 0.0.7)
       conversions (>= 1.4.5)
@@ -176,5 +176,5 @@ DEPENDENCIES
   falls_back_on (>= 0.0.3)
   geokit (>= 1.5.0)
   loose_tight_dictionary (>= 0.0.8)
-  sniff (~> 0.4.9)
+  sniff (~> 0.4.11)
   weighted_average (>= 0.0.4)

data/LICENSE-PREAMBLE

@@ -0,0 +1,54 @@
+BRIGHTER PLANET ADDITIONAL PREAMBLE TO THE GNU AFFERO GENERAL PUBLIC LICENSE
+January 7, 2011
+
+This preamble is meant to help the licensee understand exactly what
+software is being received under the license and set expectations as to
+what is not covered.
+
+IDENTITY OF THE SOFTWARE MODULE COVERED BY LICENSE:
+
+This license covers the Brighter Planet reference data import module,
+referred to as 'earth,' officially distributed from
+http://github.com/brighterplanet/earth.
+
+SCOPE OF THE SOFTWARE MODULE COVERED BY LICENSE:
+
+The scope of the license includes only those files included in the
+official Brighter Planet distribution package for the reference data
+import module, including only the reference data import scripts.
+
+WHAT IS NOT COVERED BY LICENSE:
+
+This license does not extend to the following:
+
+  * Any element of any Brighter Planet website, including but not
+    limited the copyrights, trademarks, graphic design, text,
+    documentation, method of interaction with clients, runtime
+    environment, technical infrastructure, and software code associated
+    therewith;
+
+  * Any element of any Brighter Planet website user interface, including
+    but not limited to the copyrights, trademarks, graphic design, text,
+    documentation, method of interaction with clients, runtime
+    environment, technical infrastructure, and software code associated
+    therewith;
+
+  * Any element of other software modules licensed under any form of
+    license including but not limited to other forms of open source
+    license, commercial license, and the like;
+
+  * Any element of any Brighter Planet Carbon Middleware web service,
+    including but not limited to the copyrights, trademarks, graphic
+    design, text, documentation, method of interaction with clients,
+    runtime environment, technical infrastructure, and software code, is
+    not covered by the license; and
+
+  * Any aspect or use of http://data.brighterplanet.com.
+
+  * Any use of the imported data that would infringe on the rights of
+    the original copyright holder of the imported data, such as resale
+    for profit.
+
+If you have any questions, please contact licensing@brighterplanet.com
+
+Copyright 2010, 2011 Brighter Planet, Inc.

data/features/automobile_make_model_year.feature

@@ -3,9 +3,9 @@ Feature: Data import for AutomobileMakeModelYear
   I want to import MakeModelYear data
   So that I can perform model year-based calculations
 
-  Scenario: Successfully verifying that year is between 1985 and 2009
+  Scenario: Successfully verifying that year is between 1985 and 2010
     Given a "AutomobileMakeModelYear" data import fetches results listed in "automobile_make_model_year_good"
-    When a data import verifies "Year should be between 1985 and 2009"
+    When a data import verifies "Year should be between 1985 and 2010"
     Then the verification should be successful
 
   Scenario: Successfully verifying that fuel efficiences are greater than zero
@@ -18,9 +18,9 @@ Feature: Data import for AutomobileMakeModelYear
     When a data import verifies "Fuel efficiency units should be kilometres per litre"
     Then the verification should be successful
 
-  Scenario: Failing to verify that year is between 1985 and 2009
+  Scenario: Failing to verify that year is between 1985 and 2010
     Given a "AutomobileMakeModelYear" data import fetches results listed in "automobile_make_model_year_bad"
-    When a data import verifies "Year should be between 1985 and 2009"
+    When a data import verifies "Year should be between 1985 and 2010"
     Then the verification should not be successful
 
   Scenario: Failing to verify that fuel efficiences are greater than zero

data/features/automobile_make_model_year_variant.feature

@@ -3,9 +3,9 @@ Feature: Data import for AutomobileMakeModelYearVariant
   I want to import MakeModelYearVariant data
   So that I can perform variant-based calculations
 
-  Scenario: Successfully verifying that year is between 1985 and 2009
+  Scenario: Successfully verifying that year is between 1985 and 2010
     Given a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_good"
-    When a data import verifies "Year should be between 1985 and 2009"
+    When a data import verifies "Year should be between 1985 and 2010"
     Then the verification should be successful
 
   # FIXME TODO
@@ -24,9 +24,9 @@ Feature: Data import for AutomobileMakeModelYearVariant
     When a data import verifies "Fuel efficiency units should be kilometres per litre"
     Then the verification should be successful
 
-  Scenario: Failing to verify that year is between 1985 and 2009
+  Scenario: Failing to verify that year is between 1985 and 2010
     Given a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_bad"
-    When a data import verifies "Year should be between 1985 and 2009"
+    When a data import verifies "Year should be between 1985 and 2010"
     Then the verification should not be successful
 
   # FIXME TODO

data/features/automobile_make_year.feature

@@ -3,9 +3,9 @@ Feature: Data import for AutomobileMakeYear
   I want to import MakeYear data
   So that I can perform year-based calculations
   
-  Scenario: Successfully verifying that year is between 1978 and 2007
+  Scenario: Successfully verifying that year is between 1985 and 2010
     Given a "AutomobileMakeYear" data import fetches results listed in "automobile_make_year_good"
-    When a data import verifies "Year should be between 1978 and 2007"
+    When a data import verifies "Year should be between 1985 and 2010"
     Then the verification should be successful
   
   Scenario: Successfully verifying that fuel efficiency is greater than zero
@@ -13,19 +13,14 @@ Feature: Data import for AutomobileMakeYear
     When a data import verifies "Fuel efficiency should be greater than zero"
     Then the verification should be successful
   
-  Scenario: Successfully verifying that volume is greater than zero
-    Given a "AutomobileMakeYear" data import fetches results listed in "automobile_make_year_good"
-    When a data import verifies "Volume should be greater than zero"
-    Then the verification should be successful
-  
   Scenario: Successfully verifying that fuel efficiency units are kilometres per litre
     Given a "AutomobileMakeYear" data import fetches results listed in "automobile_make_year_good"
     When a data import verifies "Fuel efficiency units should be kilometres per litre"
     Then the verification should be successful
   
-  Scenario: Failing to verify that year is between 1978 and 2007
+  Scenario: Failing to verify that year is between 1985 and 2010
     Given a "AutomobileMakeYear" data import fetches results listed in "automobile_make_year_bad"
-    When a data import verifies "Year should be between 1978 and 2007"
+    When a data import verifies "Year should be between 1985 and 2010"
     Then the verification should not be successful
   
   Scenario: Failing to verify that fuel efficiency is greater than zero
@@ -33,11 +28,6 @@ Feature: Data import for AutomobileMakeYear
     When a data import verifies "Fuel efficiency should be greater than zero"
     Then the verification should not be successful
   
-  Scenario: Failing to verify that volume is greater than zero
-    Given a "AutomobileMakeYear" data import fetches results listed in "automobile_make_year_bad"
-    When a data import verifies "Volume should be greater than zero"
-    Then the verification should not be successful
-  
   Scenario: Failing to verify that fuel efficiency units are kilometres per litre
     Given a "AutomobileMakeYear" data import fetches results listed in "automobile_make_year_bad"
     When a data import verifies "Fuel efficiency units should be kilometres per litre"

data/features/support/imports/automobile_make_model_year_bad.csv

@@ -1,3 +1,3 @@
 name,year,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_city_units,fuel_efficiency_highway_units
 Toyota,2000,10,10.5,kilometres_per_litre,kilometres_per_litre
-Honda,2010,0,-10,foo
+Honda,2011,0,-10,foo

data/features/support/imports/automobile_make_model_year_good.csv

@@ -1,3 +1,3 @@
 name,year,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_city_units,fuel_efficiency_highway_units
 Toyota,2000,10,10.5,kilometres_per_litre,kilometres_per_litre
-Honda,2009,20,21,kilometres_per_litre,kilometres_per_litre
+Honda,2010,20,21,kilometres_per_litre,kilometres_per_litre

data/features/support/imports/automobile_make_model_year_variant_bad.csv

@@ -1,3 +1,3 @@
-row_hash,year,fuel_type_code,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_city_units,fuel_efficiency_highway_units
-a1,2000,D,10.0,15.0,kilometres_per_litre,kilometres_per_litre
-b2,2010,P,-10,0,foo
+row_hash,year,fuel_type_code,fuel_efficiency,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_units,fuel_efficiency_city_units,fuel_efficiency_highway_units
+a1,2000,D,12.0,10.0,15.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
+b2,2011,P,0,-10,0,foo,bar

data/features/support/imports/automobile_make_model_year_variant_good.csv

@@ -1,3 +1,3 @@
-row_hash,year,fuel_type_code,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_city_units,fuel_efficiency_highway_units
-a1,2000,D,10.0,15.0,kilometres_per_litre,kilometres_per_litre
-b2,2009,X,20.0,21.0,kilometres_per_litre,kilometres_per_litre
+row_hash,year,fuel_type_code,fuel_efficiency,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_units,fuel_efficiency_city_units,fuel_efficiency_highway_units
+a1,2000,D,12.0,10.0,15.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
+b2,2010,X,20.5,20.0,21.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre

data/features/support/imports/automobile_make_year_bad.csv

@@ -1,3 +1,3 @@
-name,year,fuel_efficiency,volume,fuel_efficiency_units
-Toyota 2000,2000,10,1000,kilometres_per_litre
-Honda 2008,2008,0,,foo
+name,year,fuel_efficiency,fuel_efficiency_units,volume
+Toyota 2000,2000,10,kilometres_per_litre,1000
+Honda 2011,2011,-10,foo,2000

data/features/support/imports/automobile_make_year_good.csv

@@ -1,3 +1,3 @@
-name,year,fuel_efficiency,volume,fuel_efficiency_units
-Toyota 2000,2000,10,1000,kilometres_per_litre
-Honda 2007,2007,15,2000,kilometres_per_litre
+name,year,fuel_efficiency,fuel_efficiency_units,volume
+Toyota 2000,2000,10,kilometres_per_litre,1000
+Honda 2010,2010,15,kilometres_per_litre

data/lib/earth/automobile/automobile_make.rb

@@ -6,8 +6,6 @@ class AutomobileMake < ActiveRecord::Base
   has_many :fleet_years,              :class_name => 'AutomobileMakeFleetYear',        :foreign_key => 'make_name'
   has_many :make_model_year_variants, :class_name => 'AutomobileMakeModelYearVariant', :foreign_key => 'make_name'
   
-  scope :major, :conditions => { :major => true }, :order => :name
-
   data_miner do
     tap "Brighter Planet's make year data", Earth.taps_server
   end

data/lib/earth/automobile/automobile_make/data_miner.rb

@@ -6,26 +6,20 @@ AutomobileMake.class_eval do
     
     schema Earth.database_options do
       string  'name'
-      boolean 'major'
       float   'fuel_efficiency'
       string  'fuel_efficiency_units'
     end
-
-    process "Derive manufacturer names from automobile make fleet years" do
-      AutomobileMakeFleetYear.run_data_miner!
+    
+    process "Derive manufacturer names from automobile make model year variants" do
+      AutomobileMakeModelYearVariant.run_data_miner!
       connection.execute %{
         INSERT IGNORE INTO automobile_makes(name)
-        SELECT DISTINCT automobile_make_fleet_years.make_name FROM automobile_make_fleet_years
+        SELECT DISTINCT automobile_make_model_year_variants.make_name
+        FROM automobile_make_model_year_variants
       }
     end
     
-    import "a Brighter-Planet defined list of major automobile manufacturers",
-           :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/makes/make_importance.csv' do
-      key 'name'
-      store 'major'
-    end
-    
-    process "Derive average fuel efficiency from automobile make fleet years" do
+    process "Calculate fuel efficiency from automobile make fleet years for makes with CAFE data" do
       AutomobileMakeFleetYear.run_data_miner!
       make_fleet_years = AutomobileMakeFleetYear.arel_table
       makes = AutomobileMake.arel_table
@@ -35,6 +29,19 @@ AutomobileMake.class_eval do
       update_all "fuel_efficiency_units = 'kilometres_per_litre'"
     end
     
+    process "Calculate fuel effeciency from automobile make model year variants for makes without CAFE data" do
+      connection.execute %{
+        UPDATE automobile_makes
+        SET automobile_makes.fuel_efficiency = (SELECT AVG(automobile_make_model_year_variants.fuel_efficiency) FROM automobile_make_model_year_variants WHERE automobile_makes.name = automobile_make_model_year_variants.make_name)
+        WHERE automobile_makes.fuel_efficiency IS NULL
+      }
+      connection.execute %{
+        UPDATE automobile_makes
+        SET automobile_makes.fuel_efficiency_units = 'kilometres_per_litre'
+        WHERE automobile_makes.fuel_efficiency_units IS NULL
+      }
+    end
+    
     verify "Fuel efficiency should be greater than zero" do
       AutomobileMake.all.each do |make|
         unless make.fuel_efficiency > 0

data/lib/earth/automobile/automobile_make_model_year/data_miner.rb

@@ -10,8 +10,6 @@ AutomobileMakeModelYear.class_eval do
       string   'make_model_name'
       integer  'year'
       string   'make_year_name'
-      float    'fuel_efficiency'
-      string   'fuel_efficiency_units'
       float    'fuel_efficiency_city'
       string   'fuel_efficiency_city_units'
       float    'fuel_efficiency_highway'
@@ -44,15 +42,10 @@ AutomobileMakeModelYear.class_eval do
       end
     end
     
-    process "Calculate overall fuel efficiency using the latest EPA equation" do
-      update_all "fuel_efficiency = 1 / ((0.43 / fuel_efficiency_city) + (0.57 / fuel_efficiency_highway))"
-      update_all "fuel_efficiency_units = 'kilometres_per_litre'"
-    end
-    
-    verify "Year should be between 1985 and 2009" do
+    verify "Year should be between 1985 and 2010" do
       AutomobileMakeModelYear.all.each do |model_year|
-        unless model_year.year > 1984 and model_year.year < 2010
-          raise "Invalid year for AutomobileMakeModelYear #{model_year.name}: #{model_year.year} (should be between 1985 and 2009)"
+        unless model_year.year.to_i > 1984 and model_year.year.to_i < 2011
+          raise "Invalid year for AutomobileMakeModelYear #{model_year.name}: #{model_year.year} (should be between 1985 and 2010)"
         end
       end
     end
@@ -61,7 +54,7 @@ AutomobileMakeModelYear.class_eval do
       AutomobileMakeModelYear.all.each do |model_year|
         %w{ city highway }.each do |type|
           fuel_efficiency = model_year.send(:"fuel_efficiency_#{type}")
-          unless fuel_efficiency > 0
+          unless fuel_efficiency.to_f > 0
             raise "Invalid fuel efficiency #{type} for AutomobileMakeModelYear #{model_year.name}: #{fuel_efficiency} (should be > 0)"
           end
         end

data/lib/earth/automobile/automobile_make_model_year_variant/data_miner.rb

@@ -283,6 +283,8 @@ AutomobileMakeModelYearVariant.class_eval do
       string   'make_year_name' # make + year
       string   'make_model_year_name' # make + model + year
       integer  'year'
+      float    'fuel_efficiency'
+      string   'fuel_efficiency_units'
       float    'fuel_efficiency_city'
       string   'fuel_efficiency_city_units'
       float    'fuel_efficiency_highway'
@@ -451,16 +453,22 @@ AutomobileMakeModelYearVariant.class_eval do
       update_all 'fuel_efficiency_highway = 1 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))'
     end
     
+    # This will be useful later for calculating MakeModel and Make fuel efficiency based on Variant
+    process "Calculate combined adjusted fuel efficiency using the latest EPA equation" do
+      update_all "fuel_efficiency = 1 / ((0.43 / fuel_efficiency_city) + (0.57 / fuel_efficiency_highway))"
+      update_all "fuel_efficiency_units = 'kilometres_per_litre'"
+    end
+    
     %w{ AutomobileMakeModelYear AutomobileMakeModel }.each do |synthetic_resource|
       process "Synthesize #{synthetic_resource}" do
         synthetic_resource.constantize.run_data_miner!
       end
     end
     
-    verify "Year should be between 1985 and 2009" do
+    verify "Year should be between 1985 and 2010" do
       AutomobileMakeModelYearVariant.all.each do |variant|
-        unless variant.year > 1984 and variant.year < 2010
-          raise "Invalid year for AutomobileMakeModelYearVariant #{variant.row_hash}: #{variant.year} (should be between 1985 and 2009)"
+        unless variant.year > 1984 and variant.year < 2011
+          raise "Invalid year for AutomobileMakeModelYearVariant #{variant.row_hash}: #{variant.year} (should be between 1985 and 2010)"
         end
       end
     end
@@ -478,8 +486,8 @@ AutomobileMakeModelYearVariant.class_eval do
     # 
     verify "Fuel efficiencies should be greater than zero" do
       AutomobileMakeModelYearVariant.all.each do |variant|
-        %w{ city highway }.each do |type|
-          fuel_efficiency = variant.send(:"fuel_efficiency_#{type}")
+        [:fuel_efficiency, :fuel_efficiency_city, :fuel_efficiency_highway].each do |type|
+          fuel_efficiency = variant.send(type)
           unless fuel_efficiency > 0
             raise "Invalid fuel efficiency #{type} for AutomobileMakeModelYearVariant #{variant.row_hash}: #{fuel_efficiency} (should be < 0)"
           end
@@ -489,8 +497,8 @@ AutomobileMakeModelYearVariant.class_eval do
     
     verify "Fuel efficiency units should be kilometres per litre" do
       AutomobileMakeModelYearVariant.all.each do |variant|
-        %w{ city highway }.each do |type|
-          units = variant.send(:"fuel_efficiency_#{type}_units")
+        [:fuel_efficiency_units, :fuel_efficiency_city_units, :fuel_efficiency_highway_units].each do |type|
+          units = variant.send(type)
           unless units == "kilometres_per_litre"
             raise "Invalid fuel efficiency #{type} units for AutomobileMakeModelYearVariant #{variant.row_hash}: #{units} (should be kilometres_per_litre)"
           end

data/lib/earth/automobile/automobile_make_year/data_miner.rb

@@ -10,22 +10,22 @@ AutomobileMakeYear.class_eval do
       integer  'year'
       float    'fuel_efficiency'
       string   'fuel_efficiency_units'
-      integer  'volume'
+      integer  'volume' # This will sometimes be null because not all make_years have CAFE data
     end
-
-    process "Derive manufacturer names and years from automobile make fleet years" do
-      AutomobileMakeFleetYear.run_data_miner!
+    
+    process "Derive manufacturer names and years from automobile make model year variants" do
+      AutomobileMakeModelYearVariant.run_data_miner!
       connection.execute %{
         INSERT IGNORE INTO automobile_make_years(name, make_name, year)
         SELECT
-          automobile_make_fleet_years.make_year_name,
-          automobile_make_fleet_years.make_name,
-          automobile_make_fleet_years.year
-        FROM automobile_make_fleet_years
+          automobile_make_model_year_variants.make_year_name,
+          automobile_make_model_year_variants.make_name,
+          automobile_make_model_year_variants.year
+        FROM automobile_make_model_year_variants
       }
     end
     
-    process "Derive annual corporate average fuel economy across all vehicles from automobile make fleet years" do
+    process "Calculate fuel efficiency from make fleet years for makes with CAFE data" do
       AutomobileMakeFleetYear.run_data_miner!
       make_fleet_years = AutomobileMakeFleetYear.arel_table
       make_years = AutomobileMakeYear.arel_table
@@ -35,16 +35,29 @@ AutomobileMakeYear.class_eval do
       update_all "fuel_efficiency_units = 'kilometres_per_litre'"
     end
     
-    process "Derive sales volume across all vehicles from automobile make fleet years" do
+    process "Calculate fuel effeciency from automobile make model year variants for makes without CAFE data" do
+      connection.execute %{
+        UPDATE automobile_make_years
+        SET automobile_make_years.fuel_efficiency = (SELECT AVG(automobile_make_model_year_variants.fuel_efficiency) FROM automobile_make_model_year_variants WHERE automobile_make_years.name = automobile_make_model_year_variants.make_year_name)
+        WHERE automobile_make_years.fuel_efficiency IS NULL
+      }
+      connection.execute %{
+        UPDATE automobile_make_years
+        SET automobile_make_years.fuel_efficiency_units = 'kilometres_per_litre'
+        WHERE automobile_make_years.fuel_efficiency_units IS NULL
+      }
+    end
+    
+    process "Calculate sales volume from automobile make fleet years for makes with CAFE data" do
       connection.execute %{
         UPDATE automobile_make_years SET automobile_make_years.volume = (SELECT SUM(automobile_make_fleet_years.volume) FROM automobile_make_fleet_years WHERE automobile_make_fleet_years.make_year_name = automobile_make_years.name)
       }
     end
     
-    verify "Year should be between 1978 and 2007" do
+    verify "Year should be between 1985 and 2010" do
       AutomobileMakeYear.all.each do |make_year|
-        unless make_year.year > 1977 and make_year.year < 2008
-          raise "Invalid year for AutomobileMakeYear #{make_year.name}: #{make_year.year} (should be between 1978 and 2007)"
+        unless make_year.year > 1984 and make_year.year < 2011
+          raise "Invalid year for AutomobileMakeYear #{make_year.name}: #{make_year.year} (should be between 1985 and 2010)"
         end
       end
     end
@@ -64,13 +77,5 @@ AutomobileMakeYear.class_eval do
         end
       end
     end
-    
-    verify "Volume should be greater than zero" do
-      AutomobileMakeYear.all.each do |make_year|
-        unless make_year.volume > 0
-          raise "Invalid volume for AutomobileMakeYear #{make_year.name}: #{make_year.volume} (should be > 0)"
-        end
-      end
-    end
   end
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 3
-  - 10
-  version: 0.3.10
+  - 11
+  version: 0.3.11
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-30 00:00:00 -05:00
+date: 2011-01-11 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -160,6 +160,7 @@ extensions: []
 
 extra_rdoc_files: 
 - LICENSE
+- LICENSE-PREAMBLE
 - README.markdown
 files: 
 - LICENSE
@@ -344,6 +345,7 @@ files:
 - vendor/geokit-rails/test/schema.rb
 - vendor/geokit-rails/test/tasks.rake
 - vendor/geokit-rails/test/test_helper.rb
+- LICENSE-PREAMBLE
 - features/automobile_make_model_year.feature
 - features/automobile_make_year.feature
 - features/shipment_mode.feature
@@ -409,7 +411,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -847600877
+      hash: 1003851629
       segments: 
       - 0
       version: "0"