earth 0.7.0 → 0.11.0

data/certification_changelog.markdown

@@ -0,0 +1,21 @@
+# Material changes
+- Add energy content to AutomobileFuel (c0edd440e89bb201c37b449dbf0db8f3f5e69726)
+- Add hybridity and fuel code to AutomobileMakeModelYear (8d619575bc399b26abef16d4a033abb94fb9b60b)
+- Add European rail data (82d4a301f58012d38ca66a93a23703a41f5c2f94)
+- New seat class data (3e1bd6f4b41961f420a17aac566244054b0a790b)
+- Update curated airlines (4ded9ce3ce1485e1051da4b313b61e18d760229d)
+- Add US average eGRID region loss factor (a6c0176d9bb1b51c695d101513d1c32940c86e89)
+- Air improvements: ICAO data in CM1, fuzzy matching aircraft -> flight segment, fuel use data in AircraftFuelUseEquation, improve Airport and Aircraft import errata (6e2becd74c625905ae3ca40f0c0bd9b9fb652939)
+
+# Immaterial changes
+- New zipcode eGRID subregion data (ebbb08a5ffcc118f311d2fca61b7ce9e0c4d4c2d)
+- Allow users to input automobile 'make', 'model', and 'year' (97d793ea3de14a4e7eef0b0b4ca883011f52038e)
+- New 2010 FEG file (2c678d7d452da97b41bdf5311c399f7a84efe9cf)
+- New zipcode eGRID subregion data (a64add803285c836c03ed3f66f70bf6d3e4affc5)
+- Add 2011 FEG data (7c94ad50e53c6a064c862715c0afd5fdbd183873)
+- Revise computation data (4ced1779d2e45948143a84b2fc1699ef8d298d63)
+- New zipcode eGRID subregion data (af754e84468b47fa8fe7847072bd884a0c6a2b27)
+- Move some automobile data and fallbacks to Country (253215c0f914bc8bab31f64acdbfd3d7cecdd644)
+
+# Latest 3rd-party review
+f08c4cad8f30542bda5d49d0a4bbafffc451edaa

data/lib/earth/air.rb

@@ -1,6 +1,4 @@
 require 'earth/air/aircraft'
-require 'earth/air/aircraft_class'
-require 'earth/air/aircraft_fuel_use_equation'
 require 'earth/air/aircraft_instance'
 require 'earth/air/aircraft_instance_seat_class'
 require 'earth/air/airline'

data/lib/earth/air/aircraft.rb

@@ -1,25 +1,9 @@
 class Aircraft < ActiveRecord::Base
   set_primary_key :icao_code
   
-  belongs_to :aircraft_class,    :foreign_key => 'class_code',    :primary_key => 'code'
-  belongs_to :fuel_use_equation, :foreign_key => 'fuel_use_code', :primary_key => 'code', :class_name => 'AircraftFuelUseEquation'
-  
-  col :icao_code
-  col :manufacturer_name
-  col :model_name
-  col :description
-  col :aircraft_type
-  col :engine_type
-  col :engines, :type => :integer
-  col :weight_class
-  col :class_code
-  col :fuel_use_code
-  col :seats, :type => :float
-  col :passengers, :type => :float
-  
-  # Enable aircraft.flight_segments
+  # Fuzzy association with FlightSegment
   cache_loose_tight_dictionary_matches_with :flight_segments, :primary_key => :description, :foreign_key => :aircraft_description
-
+  
   class << self
     # set up a loose_tight_dictionary for matching Aircraft description with FlightSegment aircraft_description
     def loose_tight_dictionary
@@ -31,18 +15,49 @@ class Aircraft < ActiveRecord::Base
           :must_match_blocking => true,
           :first_blocking_decides => true)
     end
-  
+    
     # FIXME TODO do we want to restrict this to certain years?
     # Derive some average characteristics from flight segments
     def update_averages!
+      # Setup fuzzy matches with FlightSegment
       manually_cache_flight_segments!
-      find_each do |aircraft|
-        aircraft.seats = aircraft.flight_segments.weighted_average :seats_per_flight, :weighted_by => :passengers
-        aircraft.passengers = aircraft.flight_segments.sum :passengers
-        aircraft.save!
+      
+      # Calculate seats and passengers for each aircraft based on associated flight_segments
+      find_each do |a|
+        if a.seats = a.flight_segments.weighted_average(:seats_per_flight, :weighted_by => :passengers)
+          a.seats_specificity = 'aircraft'
+        end
+        if (passengers = a.flight_segments.sum(:passengers)) > 0
+          a.passengers = passengers
+        end
+        a.save!
+      end
+      
+      # Calculate seats for any aircraft that don't have any flight_segments by averaging across all aircraft with flight segments in the aircraft class
+      where(:seats => nil).find_each do |a|
+        if a.seats = where(:class_code => a.class_code, :seats_specificity => 'aircraft').weighted_average(:seats, :weighted_by => :passengers)
+          a.seats_specificity = 'aircraft_class'
+          a.save!
+        end
+      end
+      
+      # Calculate any missing fuel use coefficients by averaging across all aircraft with fuel use coefficients in the same aircraft class
+      where(:m3 => nil).find_each do |a|
+        a.m3 = where(:class_code => a.class_code, :fuel_use_specificity => 'aircraft').weighted_average(:m3, :weighted_by => :passengers)
+        a.m2 = where(:class_code => a.class_code, :fuel_use_specificity => 'aircraft').weighted_average(:m2, :weighted_by => :passengers)
+        a.m1 = where(:class_code => a.class_code, :fuel_use_specificity => 'aircraft').weighted_average(:m1, :weighted_by => :passengers)
+        a.b =  where(:class_code => a.class_code, :fuel_use_specificity => 'aircraft').weighted_average(:b,  :weighted_by => :passengers)
+        if a.valid_fuel_use_equation?
+          a.m3_units = 'kilograms_per_cubic_nautical_mile'
+          a.m2_units = 'kilograms_per_square_nautical_mile'
+          a.m1_units = 'kilograms_per_nautical_mile'
+          a.b_units =  'kilograms'
+          a.fuel_use_specificity = 'aircraft_class'
+          a.save!
+        end
       end
     end
-  
+    
     # Cache fuzzy matches between FlightSegment aircraft_description and Aircraft description
     def manually_cache_flight_segments!
       FlightSegment.run_data_miner!
@@ -56,16 +71,16 @@ class Aircraft < ActiveRecord::Base
           # Pull out the complete first aircraft description
           # e.g. 'boeing 747-100'
           first_description = original_description.split('/')[0]
-        
+          
           # Pull out the root of the description - the text up to and including the last ' ' or '-'
           # e.g. 'boeing 747-'
           root_length = first_description.rindex(/[ \-]/)
           root = first_description.slice(0..root_length)
-        
+          
           # Pull out the suffixes - the text separated by forward slashes
           # e.g. ['100', '200']
           suffixes = original_description.split(root)[1].split('/')
-        
+          
           # Create an array of synthesized descriptions by appending each suffix to the root
           # e.g. ['boeing 747-100', 'boeing 747-200']
           suffixes.map{ |suffix| root + suffix }.each do |synthesized_description|
@@ -91,4 +106,39 @@ class Aircraft < ActiveRecord::Base
       end
     end
   end
+  
+  def valid_fuel_use_equation?
+    [m3, m2, m1, b].all?(&:present?) and [m3, m2, m1, b].any?(&:nonzero?)
+  end
+  
+  falls_back_on :m3 => lambda { weighted_average(:m3, :weighted_by => :passengers) }, # 9.73423082858437e-08   r7110: 8.6540464368905e-8      r6972: 8.37e-8
+                :m2 => lambda { weighted_average(:m2, :weighted_by => :passengers) }, # -0.000134350543484608  r7110: -0.00015337661447817    r6972: -4.09e-5
+                :m1 => lambda { weighted_average(:m1, :weighted_by => :passengers) }, # 6.7728101555467        r7110: 4.7781966869412         r6972: 7.85
+                :b  => lambda { weighted_average(:b,  :weighted_by => :passengers) }, # 1527.81790006167       r7110: 1065.3476555284         r6972: 1.72e3
+                :m3_units => 'kilograms_per_cubic_nautical_mile',
+                :m2_units => 'kilograms_per_square_nautical_mile',
+                :m1_units => 'kilograms_per_nautical_mile',
+                :b_units  => 'kilograms'
+  
+  col :icao_code
+  col :manufacturer_name
+  col :model_name
+  col :description
+  col :aircraft_type
+  col :engine_type
+  col :engines, :type => :integer
+  col :weight_class
+  col :class_code
+  col :passengers, :type => :float
+  col :seats, :type => :float
+  col :seats_specificity
+  col :m3, :type => :float
+  col :m3_units
+  col :m2, :type => :float
+  col :m2_units
+  col :m1, :type => :float
+  col :m1_units
+  col :b, :type => :float
+  col :b_units
+  col :fuel_use_specificity
 end

data/lib/earth/air/aircraft/data_miner.rb

@@ -51,34 +51,27 @@ Aircraft.class_eval do
       store 'weight_class'
     end
     
-    import "a curated list of aircraft fuel use codes",
-           :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGxqRjFJdDlQWVVLYS11NnJVcDZsYWc&output=csv' do
+    import "aircraft fuel use equations derived from EMEP/EEA and ICAO",
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdEhYenF3dGt1T0Y1cTdneUNsNjV0dEE&output=csv' do
       key 'icao_code'
-      store 'fuel_use_code'
+      store 'm3', :units_field_name => 'm3_units'
+      store 'm2', :units_field_name => 'm2_units'
+      store 'm1', :units_field_name => 'm1_units'
+      store 'b',  :units_field_name => 'b_units'
+      store 'fuel_use_specificity', :static => 'aircraft'
     end
     
     process "Synthesize description from manufacturer name and model name" do
-      find_each do |aircraft|
-        aircraft.update_attribute :description, [aircraft.manufacturer_name, aircraft.model_name].join(' ').downcase
-      end
+      update_all "description = LOWER(manufacturer_name || ' ' || model_name)"
     end
     
     process "Synthesize class code from engine type and weight class" do
-      find_each do |aircraft|
-        size = case aircraft.weight_class
-        when 'Small', 'Small+', 'Light'
-          'Light'
-        when 'Large', 'Medium'
-          'Medium'
-        else
-          'Heavy'
-        end
-        aircraft.update_attribute :class_code, [size, aircraft.engines.to_s, 'engine', aircraft.engine_type].join(' ')
-      end
+      where(:weight_class => ['Small', 'Small+', 'Light']).update_all "class_code = 'Light '  || engines || ' engine ' || engine_type"
+      where(:weight_class => ['Medium', 'Large']         ).update_all "class_code = 'Medium ' || engines || ' engine ' || engine_type"
+      where(:weight_class => 'Heavy'                     ).update_all "class_code = 'Heavy '  || engines || ' engine ' || engine_type"
     end
     
+    # Calculate seats and passengers from flight_segments
     process :update_averages!
-    
-    # FIXME TODO verify this
   end
 end

data/lib/earth/air/aircraft_instance.rb

@@ -6,4 +6,6 @@ class AircraftInstance < ActiveRecord::Base
   col :serial_number
   col :aircraft_description
   col :airline_name
+  add_index :registration
+  add_index :serial_number
 end

data/lib/earth/air/aircraft_instance_seat_class.rb

@@ -9,4 +9,5 @@ class AircraftInstanceSeatClass < ActiveRecord::Base
   col :seat_pitch_units
   col :seat_width, :type => :float
   col :seat_width_units
+  add_index :aircraft_instance_id
 end

data/lib/earth/air/airport/data_miner.rb

@@ -46,7 +46,7 @@ Airport.class_eval do
     process "Fill in blank country codes" do
       Country.find_each do |country|
         next unless country.name.present? and country.iso_3166_code.present?
-        update_all({ :country_iso_3166_code => country.iso_3166_code }, [ "country_name LIKE ?", country.name ])
+        where(["country_name LIKE ?", country.name]).update_all :country_iso_3166_code => country.iso_3166_code
       end
     end
     

data/lib/earth/air/data_miner.rb

@@ -1,6 +1,4 @@
 require 'earth/air/aircraft/data_miner'
-require 'earth/air/aircraft_class/data_miner'
-require 'earth/air/aircraft_fuel_use_equation/data_miner'
 require 'earth/air/airline/data_miner'
 require 'earth/air/airport/data_miner'
 require 'earth/air/bts_aircraft/data_miner'

data/lib/earth/air/flight_distance_class.rb

@@ -2,7 +2,7 @@ class FlightDistanceClass < ActiveRecord::Base
   set_primary_key :name
   
   def self.find_by_distance(distance)
-    first :conditions => arel_table[:min_distance].lteq(distance).and(arel_table[:max_distance].gt(distance))
+    first :conditions => arel_table[:min_distance].lteq(distance.to_f).and(arel_table[:max_distance].gt(distance.to_f))
   end
   
   col :name
@@ -12,5 +12,4 @@ class FlightDistanceClass < ActiveRecord::Base
   col :min_distance_units
   col :max_distance, :type => :float
   col :max_distance_units
-  col :passengers, :type => :float
-end
+end

data/lib/earth/air/flight_distance_class/data_miner.rb

@@ -11,18 +11,5 @@ FlightDistanceClass.class_eval do
     # FIXME TODO verify that min_distance >= 0
     # FIXME TODO verify that max_distance > 0
     # FIXME TODO verify that distance class distance bounds don't overlap
-    
-    process "Ensure FlightSegment is populated" do
-      FlightSegment.run_data_miner!
-    end
-    
-    process "Calculate passengers for each distance class" do
-      find_each do |distance_class|
-        distance_class.passengers = FlightSegment.distances_between(distance_class.min_distance, distance_class.max_distance).sum(:passengers)
-        distance_class.save!
-      end
-    end
-    
-    # FIXME TODO verify this
   end
 end

data/lib/earth/air/flight_distance_class_seat_class.rb

@@ -1,10 +1,11 @@
 class FlightDistanceClassSeatClass < ActiveRecord::Base
   set_primary_key :name
   
-  belongs_to :distance_class, :foreign_key => 'distance_class_name', :primary_key => 'name', :class_name => 'FlightDistanceClass'
+  falls_back_on :name => 'fallback',
+                :multiplier => 1.0
   
   col :name
   col :distance_class_name
   col :seat_class_name
   col :multiplier, :type => :float
-end
+end

data/lib/earth/air/flight_seat_class.rb

@@ -1,11 +1,5 @@
 class FlightSeatClass < ActiveRecord::Base
   set_primary_key :name
   
-  has_many :distance_class_seat_classes, :foreign_key => 'seat_class_name', :primary_key => 'name', :class_name => 'FlightDistanceClassSeatClass'
-  
-  falls_back_on :name => 'fallback',
-                :multiplier => 1
-  
   col :name
-  col :multiplier, :type => :float
 end

data/lib/earth/air/flight_seat_class/data_miner.rb

@@ -9,12 +9,5 @@ FlightSeatClass.class_eval do
         find_or_create_by_name seat_class_name
       end
     end
-    
-    process "Calculate multipliers from FlightDistanceClassSeatClass and FlightDistanceClass" do
-      find_each do |seat_class|
-        seat_class.multiplier = seat_class.distance_class_seat_classes.weighted_average(:multiplier, :weighted_by => [:distance_class, :passengers])
-        seat_class.save!
-      end
-    end
   end
 end

data/lib/earth/air/flight_segment.rb

@@ -14,41 +14,57 @@ class FlightSegment < ActiveRecord::Base
   # Enable flight_segment.aircraft
   cache_loose_tight_dictionary_matches_with :aircraft, :primary_key => :aircraft_description, :foreign_key => :description
   
+  class << self
+    def update_averages!
+      # Derive load factor, which is passengers divided by available seats
+      where('seats > 0').update_all 'load_factor = 1.0 * passengers / seats'
+      
+      # Assume a load factor of 1 where passengers > available seats
+      where('passengers > seats AND seats > 0').update_all 'load_factor = 1'
+      
+      # FIXME TODO: what is 90.718474
+      # Derive freight share as a fraction of the total weight carried
+      where('(freight + mail + passengers) > 0').update_all 'freight_share = 1.0 * (freight + mail) / (freight + mail + (passengers * 90.718474))'
+      
+      # Derive average seats per flight
+      where('flights > 0').update_all 'seats_per_flight = 1.0 * seats / flights'
+    end
+  end
+  
   falls_back_on :distance         => lambda { weighted_average(:distance,         :weighted_by => :passengers) }, # 2077.1205         data1 10-12-2010
                 :seats_per_flight => lambda { weighted_average(:seats_per_flight, :weighted_by => :passengers) }, # 144.15653537046   data1 10-12-2010
                 :load_factor      => lambda { weighted_average(:load_factor,      :weighted_by => :passengers) }, # 0.78073233770097  data1 10-12-2010
                 :freight_share    => lambda { weighted_average(:freight_share,    :weighted_by => :passengers) }  # 0.022567224170157 data1 10-12-2010
   
-  col :row_hash                          # auto-generated primary key
-  col :origin_airport_iata_code          # iata code
-  col :origin_airport_city               # city
-  col :origin_country_iso_3166_code      # iso code
-  col :destination_airport_iata_code     # iata code
-  col :destination_airport_city          # city
-  col :destination_country_iso_3166_code # iso code
-  col :airline_bts_code                  # bts code
-  col :airline_icao_code                 # icao code
-  col :airline_name                      # text description derived from bts or icao code
-  col :aircraft_bts_code                 # bts code
-  col :aircraft_description              # text description derived from BTS T100 or ICAO TFS
-  col :flights, :type => :integer                           # number of flights over month or year
-  col :passengers, :type => :integer                        # total passengers on all flights
-  col :seats, :type => :integer                             # total seats on all flights
-  col :seats_per_flight, :type => :float                  # average seats per flight; make this a float
-  col :load_factor, :type => :float                       # passengers / seats
-  col :freight_share, :type => :float                     # (freight + mail) / (freight + mail + (passengers * average passenger weight))
-  col :distance, :type => :float                          # flight distance
-  col :distance_units                    # 'kilometres'
-  col :payload_capacity, :type => :float                  # aircraft maximum payload capacity rating; float b/c unit conversion
-  col :payload_capacity_units            # 'kilograms'
-  col :freight, :type => :float                           # total freight on all flights performed; float b/c unit conversion
-  col :freight_units                     # 'kilograms'
-  col :mail, :type => :float                              # total mail on all flights performed; float b/c unit conversion
-  col :mail_units                        # 'kilograms'
-  col :month, :type => :integer                             # month of flight
-  col :year, :type => :integer                              # year of flight
-  col :approximate_date, :type => :date                  # assumed 14th day of month
-  col :source                            # 'BTS T100' or 'ICAO TFS'
+  col :row_hash                            # auto-generated primary key
+  col :origin_airport_iata_code            # iata code
+  col :origin_airport_city                 # city
+  col :origin_country_iso_3166_code        # iso code
+  col :destination_airport_iata_code       # iata code
+  col :destination_airport_city            # city
+  col :destination_country_iso_3166_code   # iso code
+  col :airline_bts_code                    # bts code
+  col :airline_icao_code                   # icao code
+  col :airline_name                        # text description derived from bts or icao code
+  col :aircraft_bts_code                   # bts code
+  col :aircraft_description                # text description derived from BTS T100 or ICAO TFS
+  col :flights,          :type => :integer # number of flights over month or year
+  col :passengers,       :type => :integer # total passengers on all flights
+  col :seats,            :type => :integer # total seats on all flights
+  col :seats_per_flight, :type => :float   # average seats per flight; make this a float
+  col :load_factor,      :type => :float   # passengers / seats
+  col :freight_share,    :type => :float   # (freight + mail) / (freight + mail + (passengers * average passenger weight))
+  col :distance,         :type => :float   # flight distance
+  col :distance_units                      # 'kilometres'
+  col :payload_capacity, :type => :float   # aircraft maximum payload capacity rating; float b/c unit conversion
+  col :payload_capacity_units              # 'kilograms'
+  col :freight,          :type => :float   # total freight on all flights performed; float b/c unit conversion
+  col :freight_units                       # 'kilograms'
+  col :mail,             :type => :float   # total mail on all flights performed; float b/c unit conversion
+  col :mail_units                          # 'kilograms'
+  col :month,            :type => :integer # month of flight
+  col :year,             :type => :integer # year of flight
+  col :source                              # 'BTS T100' or 'ICAO TFS'
   add_index :origin_airport_iata_code
   add_index :origin_airport_city
   add_index :destination_airport_iata_code
@@ -59,26 +75,4 @@ class FlightSegment < ActiveRecord::Base
   add_index :aircraft_bts_code
   add_index :aircraft_description
   add_index :year
-  
-  def self.distances_between(floor, ceiling)
-    where arel_table[:distance].gteq(floor).and(arel_table[:distance].lt(ceiling))
-  end
-  
-  def self.update_averages!
-    # Derive load factor, which is passengers divided by available seats
-    update_all 'load_factor = 1.0 * passengers / seats', 'seats > 0'
-
-    # Assume a load factor of 1 where passengers > available seats
-    update_all 'load_factor = 1', 'passengers > seats AND seats > 0'
-  
-    # TODO: what is 90.718474
-    # Derive freight share as a fraction of the total weight carried
-    update_all 'freight_share = 1.0 * (freight + mail) / (freight + mail + (passengers * 90.718474))', '(freight + mail + passengers) > 0'
-  
-    # Derive average seats per flight
-    update_all 'seats_per_flight = 1.0 * seats / flights', 'flights > 0'
-    
-    # Add a useful date field
-    update_all %{approximate_date = DATE(year || '-' || month || '-' || '14')}, 'month IS NOT NULL'
-  end
-end
+end

data/lib/earth/air/flight_segment/data_miner.rb

@@ -222,7 +222,7 @@ FlightSegment.class_eval do
     process "Look up airline name based on BTS code" do
       connection.select_values("SELECT DISTINCT airline_bts_code FROM flight_segments WHERE airline_bts_code IS NOT NULL").each do |bts_code|
         if airline = Airline.find_by_bts_code(bts_code)
-          update_all({ :airline_name => airline.name }, :airline_bts_code => bts_code)
+          where(:airline_bts_code => bts_code).update_all :airline_name => airline.name
         end
       end
     end
@@ -230,7 +230,7 @@ FlightSegment.class_eval do
     process "Look up aircraft description based on BTS code" do
       connection.select_values("SELECT DISTINCT aircraft_bts_code FROM flight_segments WHERE aircraft_bts_code IS NOT NULL").each do |bts_code|
         if aircraft = BtsAircraft.find_by_bts_code(bts_code)
-          update_all({ :aircraft_description => aircraft.description.downcase }, :aircraft_bts_code => bts_code)
+          where(:aircraft_bts_code => bts_code).update_all :aircraft_description => aircraft.description.downcase
         end
       end
     end