earth 0.4.12 → 0.5.0

data/lib/earth/air/aircraft_class.rb

@@ -1,7 +1,15 @@
 class AircraftClass < ActiveRecord::Base
   set_primary_key :code
   
-  # has_many :aircraft, :foreign_key => 'class_code', :primary_key => 'code'
+  has_many :aircraft, :foreign_key => 'class_code', :primary_key => 'code'
+  
+  def fuel_use_coefficients
+    [m3, m2, m1, b]
+  end
+  
+  def valid_fuel_use_equation?
+    fuel_use_coefficients.all?(&:present?) and fuel_use_coefficients.any?(&:nonzero?)
+  end
   
   data_miner do
     tap "Brighter Planet's aircraft class data", Earth.taps_server

data/lib/earth/air/aircraft_class/data_miner.rb

@@ -1,30 +1,67 @@
 AircraftClass.class_eval do
   data_miner do
     schema Earth.database_options do
-      string  'code'
-      string  'name'
-      float   'm1'
-      float   'm2'
-      float   'm3'
-      float   'endpoint_fuel'
-      float   'seats'
+      string 'code'
+      float  'm3'
+      string 'm3_units'
+      float  'm2'
+      string 'm2_units'
+      float  'm1'
+      string 'm1_units'
+      float  'b'
+      string 'b_units'
+      float  'seats'
     end
     
-    import "a list of Brighter Planet-defined aircraft classes",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGNBbHFibmxJUFprQkUwZHp6VU51Smc&hl=en&gid=0&output=csv' do
-      key 'code', :field_name => 'aircraft_class_code'
-      store 'name'
+    process "Ensure Aircraft is populated" do
+      Aircraft.run_data_miner!
     end
     
-    process "Derive some average aircraft chraracteristics from aircraft" do
-      Aircraft.run_data_miner!
-      aircraft = Aircraft.arel_table
-      aircraft_classes = AircraftClass.arel_table
-      conditional_relation = aircraft_classes[:code].eq(aircraft[:class_code])
-      
-      %w{ m1 m2 m3 endpoint_fuel seats }.each do |column|
-        relation = Aircraft.weighted_average_relation(column).where(conditional_relation)
-        update_all "#{column} = (#{relation.to_sql})"
+    process "Derive aircraft classes from Aircraft" do
+      connection.select_values("SELECT DISTINCT class_code FROM aircraft WHERE aircraft.class_code IS NOT NULL").each do |class_code|
+        AircraftClass.find_or_create_by_code(class_code)
+      end
+    end
+    
+    process "Derive some average characteristics from Aircraft" do
+      AircraftClass.find_each do |aircraft_class|
+        %w{ m3 m2 m1 b }.each do |coefficient|
+          value = AircraftFuelUseEquation.where("aircraft.class_code = '#{aircraft_class.code}'").
+            weighted_average(:"#{coefficient}", :weighted_by => [:aircraft, :passengers])
+          aircraft_class.send("#{coefficient}=", value)
+        end
+#           # do this in sql because we want to preserve nils and weighted_average returns 0 when it gets NULL
+#           connection.execute %{
+# UPDATE aircraft_classes
+# SET aircraft_classes.#{coefficient} = (
+#   SELECT sum(aircraft_fuel_use_equations.#{coefficient} * aircraft.passengers) / sum(aircraft.passengers)
+#   FROM aircraft_fuel_use_equations
+#   INNER JOIN aircraft
+#   ON aircraft.fuel_use_code = aircraft_fuel_use_equations.code
+#   WHERE aircraft.class_code = '#{aircraft_class.code}'
+#   AND aircraft_fuel_use_equations.#{coefficient} IS NOT NULL
+# )
+# WHERE aircraft_classes.code = '#{aircraft_class.code}'
+#           }
+#        end
+        
+        aircraft_class.seats = aircraft_class.aircraft.weighted_average(:seats, :weighted_by => :passengers)
+#         # do this in sql because we want to preserve nils and weighted_average returns 0 when it gets NULL
+#         connection.execute %{
+# UPDATE aircraft_classes
+# SET aircraft_classes.seats = (
+#   SELECT sum(aircraft.seats * aircraft.passengers) / sum(aircraft.passengers)
+#   FROM aircraft
+#   WHERE aircraft.class_code = '#{aircraft_class.code}'
+# )
+# WHERE aircraft_classes.code = '#{aircraft_class.code}'
+#         }
+        
+        aircraft_class.m3_units = 'kilograms_per_cubic_nautical_mile'
+        aircraft_class.m2_units = 'kilograms_per_square_nautical_mile'
+        aircraft_class.m1_units = 'kilograms_per_nautical_mile'
+        aircraft_class.b_units  = 'kilograms'
+        aircraft_class.save
       end
     end
     

data/lib/earth/air/aircraft_fuel_use_equation.rb

@@ -0,0 +1,26 @@
+class AircraftFuelUseEquation < ActiveRecord::Base
+  set_primary_key :code
+  
+  has_many :aircraft, :foreign_key => 'fuel_use_code', :primary_key => 'code'
+  
+  falls_back_on :m3 => lambda { weighted_average(:m3, :weighted_by => [:aircraft, :passengers]) }, # 9.73423082858437e-08   r7110: 8.6540464368905e-8      r6972: 8.37e-8
+                :m2 => lambda { weighted_average(:m2, :weighted_by => [:aircraft, :passengers]) }, # -0.000134350543484608  r7110: -0.00015337661447817    r6972: -4.09e-5
+                :m1 => lambda { weighted_average(:m1, :weighted_by => [:aircraft, :passengers]) }, # 6.7728101555467        r7110: 4.7781966869412         r6972: 7.85
+                :b  => lambda { weighted_average(:b,  :weighted_by => [:aircraft, :passengers]) }, # 1527.81790006167       r7110: 1065.3476555284         r6972: 1.72e3
+                :m3_units => 'kilograms_per_cubic_nautical_mile',
+                :m2_units => 'kilograms_per_square_nautical_mile',
+                :m1_units => 'kilograms_per_nautical_mile',
+                :b_units  => 'kilograms'
+  
+  def fuel_use_coefficients
+    [m3, m2, m1, b]
+  end
+  
+  def valid_fuel_use_equation?
+    fuel_use_coefficients.all?(&:present?) and fuel_use_coefficients.any?(&:nonzero?)
+  end
+  
+  data_miner do
+    tap "Brighter Planet's sanitized aircraft fuel use data", Earth.taps_server
+  end
+end

data/lib/earth/air/aircraft_fuel_use_equation/data_miner.rb

@@ -0,0 +1,26 @@
+AircraftFuelUseEquation.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string 'code'
+      string 'aircraft_description'
+      float  'm3'
+      string 'm3_units'
+      float  'm2'
+      string 'm2_units'
+      float  'm1'
+      string 'm1_units'
+      float  'b'
+      string 'b_units'
+    end
+    
+    import "aircraft fuel use equations derived from EMEP/EEA and ICAO",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDltUVZVekVobEJPYlpFNUpWNkwyYXc&output=csv' do
+      key 'code'
+      store 'aircraft_description'
+      store 'm3', :units_field_name => 'm3_units'
+      store 'm2', :units_field_name => 'm2_units'
+      store 'm1', :units_field_name => 'm1_units'
+      store 'b',  :units_field_name => 'b_units'
+    end
+  end
+end

data/lib/earth/air/aircraft_manufacturer.rb

@@ -1,8 +1,6 @@
 class AircraftManufacturer < ActiveRecord::Base
   set_primary_key :name
   
-  # has_many :aircraft, :foreign_key => 'name', :primary_key => 'bp_code'
-  
   data_miner do
     tap "Brighter Planet's aircraft manufacturer data", Earth.taps_server
   end

data/lib/earth/air/aircraft_manufacturer/data_miner.rb

@@ -8,8 +8,11 @@ AircraftManufacturer.class_eval do
       string 'name'
     end
     
-    process "Derive a list of aircraft manufacturers from aircraft" do
+    process "Ensure Aircraft is populated" do
       Aircraft.run_data_miner!
+    end
+    
+    process "Derive a list of aircraft manufacturers from aircraft" do
       INSERT_IGNORE %{INTO aircraft_manufacturers(name)
         SELECT aircraft.manufacturer_name FROM aircraft WHERE aircraft.manufacturer_name IS NOT NULL
       }

data/lib/earth/air/airline.rb

@@ -1,16 +1,7 @@
 class Airline < ActiveRecord::Base
-  set_primary_key :iata_code
-
-#  has_many :airline_aircraft, :class_name => 'AirlineAircraft'
-#  has_many :seat_classes, :class_name => 'AirlineSeatClass'
-#  has_many :segments, :foreign_key => 'airline_iata_code', :primary_key => 'iata_code', :class_name => "FlightSegment"
-#  has_many :airline_aircraft_seat_classes, :class_name => 'AirlineAircraftSeatClass'
-
+  set_primary_key :name
+  
   data_miner do
     tap "Brighter Planet's sanitized airlines data", Earth.taps_server
   end
-  
-  def all_flights_domestic?
-    !international?
-  end
 end

data/lib/earth/air/airline/data_miner.rb

@@ -1,59 +1,18 @@
-require 'errata'
-
 Airline.class_eval do
-  class Airline::Guru
-    # needed by errata
-    def is_not_coral_air?(row); row['Code'].to_i != 19155; end                # 19155 Coral Air Inc.: COR
-    def is_not_aviacion_y_comercio?(row); row['Code'].to_i != 19452; end      # 19452 Aviacion Y Comercio S.A.: AO
-    def is_not_air_china?(row); row['Code'].to_i != 19543; end                # 19543 Air China: CA
-    def is_not_south_african_airways?(row); row['Code'].to_i != 19570; end    # 19570 South African Airways: SA
-    def is_not_continental_airlines?(row); row['Code'].to_i != 19704; end     # 19704 Continental Air Lines Inc.: CO
-    def is_not_sallee_s_aviation?(row); row['Code'].to_i != 19740; end        # 19740 Sallee's Aviation: SAL
-    def is_not_air_berlin?(row); row['Code'].to_i != 21361; end               # "21361","Air Berlin PLC and CO: AB"
-  end
-
   data_miner do
     schema Earth.database_options do
-      string   'iata_code'
-      string   'name'
-      string   'dot_airline_id_code'
-      boolean  'international'
-      float    'seats'
-      float    'distance'
-      string   'distance_units'
-      float    'load_factor'
-      float    'freight_share'
-      float    'payload'
-      string   'payload_units'
-    end
-    
-    import "the T100 AIRLINE_ID lookup table, which also includes IATA codes",
-           :url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_AIRLINE_ID',
-           :errata => { :url => 'http://static.brighterplanet.com/science/data/transport/air/airlines/errata.csv', :responder => Airline::Guru.new } do
-      key 'iata_code', :field_name => 'Description', :split => { :pattern => /:/, :keep => 1 }
-      store 'dot_airline_id_code', :field_name => 'Code'
-      store 'name', :field_name => 'Description', :split => { :pattern => /:/, :keep => 0 }
-    end
-    
-    process "Determine whether airlines fly internationally by looking at flight segments" do
-      FlightSegment.run_data_miner!
-      update_all 'international = 1', '(SELECT COUNT(*) FROM flight_segments WHERE flight_segments.airline_iata_code = airlines.iata_code AND flight_segments.origin_country_iso_3166_code != flight_segments.destination_country_iso_3166_code AND flight_segments.origin_country_iso_3166_code IS NOT NULL AND flight_segments.destination_country_iso_3166_code IS NOT NULL) > 0'
+      string 'name'
+      string 'bts_code'
+      string 'iata_code'
+      string 'icao_code'
     end
     
-    process "Derive some average flight characteristics from flight segments" do
-      FlightSegment.run_data_miner!
-      airlines = Airline.arel_table
-      segments = FlightSegment.arel_table
-      
-      conditional_relation = airlines[:iata_code].eq(segments[:airline_iata_code])
-
-      # sabshere 2/1/11 these need units
-      update_all "seats                    = (#{FlightSegment.weighted_average_relation(:seats,               :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
-      update_all "distance                 = (#{FlightSegment.weighted_average_relation(:distance,            :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
-      update_all "load_factor              = (#{FlightSegment.weighted_average_relation(:load_factor,         :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
-      update_all "freight_share            = (#{FlightSegment.weighted_average_relation(:freight_share,       :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
-      update_all "payload                  = (#{FlightSegment.weighted_average_relation(:payload,             :weighted_by => :passengers, :disaggregate_by => :departures_performed).where(conditional_relation).to_sql})"
+    import "a Brighter Planet-curated list of airlines",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDhoVHZmSTlZcHBsRUtPR0dPd0prMkE&output=csv' do
+      key 'name'
+      store 'bts_code',  :nullify => true
+      store 'iata_code',  :nullify => true
+      store 'icao_code', :nullify => true
     end
   end
 end
-

data/lib/earth/air/airport.rb

@@ -1,27 +1,12 @@
 class Airport < ActiveRecord::Base
   set_primary_key :iata_code
   
-  # --------------------------------
-  # virtual has_many association
-  # has_many :segments won't work because there's no general way to specify the correct conditions
-  # even if you get clever with it, like
-  # has_many  :segments,
-  #           :class_name => 'FlightSegment',
-  #           :foreign_key => 'origin_airport_id',
-  #           :conditions => 'flight_segments.destination_airport_id = #{id}'
-  # you get queries like "`flight_segments`.origin_airport_id = 3654 AND (flight_segments.destination_airport_id = 3654))"
-  # in which you notice the AND which must be an OR
-  # and you can't just do finder_sql, because that breaks any other :select
-  def segments
-    FlightSegment.scoped :conditions => ['origin_airport_id = ? OR destination_airport_id = ?', id, id]
-  end
-  # --------------------------------
-  
   belongs_to :country, :foreign_key => 'country_iso_3166_code', :primary_key => 'iso_3166_code'
+  
   acts_as_mappable :default_units => :nms,
                    :lat_column_name => :latitude,
                    :lng_column_name => :longitude
-
+  
   data_miner do
     tap "Brighter Planet's sanitized airports data", Earth.taps_server
     
@@ -29,16 +14,4 @@ class Airport < ActiveRecord::Base
       run_data_miner_on_belongs_to_associations
     end
   end
-  
-  def all_flights_from_here_domestic?
-    !international_origin?
-  end
-
-  def all_flights_to_here_domestic?
-    !international_destination?
-  end
-  
-  def united_states?
-    country == Country.united_states
-  end
 end

data/lib/earth/air/airport/data_miner.rb

@@ -1,4 +1,19 @@
 Airport.class_eval do
+  class Airport::Guru
+    def method_missing(method_id, *args, &block)
+      if method_id.to_s =~ /\A(id|iata)_is_([a-z]{3}|\d{1,4})\?$/
+        regexp = Regexp.new($2, Regexp::IGNORECASE)
+        if $1 == "iata"
+          args.first['iata_code'] =~ regexp # row['iata_code'] =~ /meh/i
+        else
+          args.first[$1] =~ regexp # row['id'] =~ /1234/i
+        end
+      else
+        super
+      end
+    end
+  end
+  
   data_miner do
     schema Earth.database_options do
       string   'iata_code'
@@ -8,74 +23,64 @@ Airport.class_eval do
       string   'country_iso_3166_code'
       float    'latitude'
       float    'longitude'
-      float    'seats'
-      float    'distance'
-      string   'distance_units'
-      float    'load_factor'
-      float    'freight_share'
-      float    'payload'
-      string   'payload_units'
-      boolean  'international_origin'
-      boolean  'international_destination'
     end
     
     import "the OpenFlights.org airports database",
            :url => 'https://openflights.svn.sourceforge.net/svnroot/openflights/openflights/data/airports.dat',
-           :headers => false,
-           :select => lambda { |row| row[4].present? } do
-      key   'iata_code', :field_number => 4
-      store 'name', :field_number => 1
-      store 'city', :field_number => 2
-      store 'country_name', :field_number => 3
-      store 'country_iso_3166_code', :field_number => 3, :upcase => true, :dictionary => { :input => 'name', :output => 'iso_3166_code', :url => 'http://data.brighterplanet.com/countries.csv' }
-      store 'latitude', :field_number => 6
-      store 'longitude', :field_number => 7
+           :headers => %w{ id name city country_name iata_code icao_code latitude longitude altitude timezone daylight_savings },
+           :errata => { :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdFc2UzhQYU5PWEQ0N21yWFZGNmc2a3c&gid=0&output=csv',
+                        :responder => Airport::Guru.new } do
+      key 'iata_code'
+      store 'name'
+      store 'city'
+      store 'country_name'
+      store 'latitude'
+      store 'longitude'
     end
     
-    # step.await :other_class => FlightSegment do |deferred|
-    #   deferred.derive :country # this uses a heuristic that depends on flight segments
-    # class << self
-    #   def derive_country
-    #     update_all('country_id = (SELECT      flight_segments.origin_country_id FROM flight_segments WHERE      flight_segments.origin_airport_id = airports.id AND      flight_segments.origin_country_id IS NOT NULL LIMIT 1)', 'airports.country_id IS NULL')
-    #     update_all('country_id = (SELECT flight_segments.destination_country_id FROM flight_segments WHERE flight_segments.destination_airport_id = airports.id AND flight_segments.destination_country_id IS NOT NULL LIMIT 1)', 'airports.country_id IS NULL')
-    #     Country.all.each do |c|
-    #       next if c.name.blank?
-    #       update_all("country_id = #{c.id}", ["airports.country_id IS NULL AND airports.country_name LIKE ?", "%#{c.name.upcase}%"])
-    #     end
-    #     Airport.all(:conditions => 'country_id IS NULL AND country_name IS NOT NULL').each do |a|
-    #       c = Country.find(:first, :conditions => ["name like ?", "%#{a.country_name}%"])
-    #       a.update_attributes(:country_id => c.id) if c
-    #     end
-    #   end
-    # end
+    import "airports missing from the OpenFlights.org database",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHpyR3NudEl5V21ZcEdXQXFDNU8zTWc&output=csv' do
+      key 'iata_code'
+      store 'name'
+      store 'city'
+      store 'country_name'
+      store 'latitude'
+      store 'longitude'
+    end
     
-    process "Determine whether each airport serves international flights" do
-      FlightSegment.run_data_miner!
-      update_all 'international_origin = 1','(SELECT COUNT(*) FROM flight_segments WHERE flight_segments.origin_airport_iata_code = airports.iata_code AND flight_segments.origin_country_iso_3166_code != flight_segments.destination_country_iso_3166_code AND flight_segments.origin_country_iso_3166_code IS NOT NULL AND flight_segments.destination_country_iso_3166_code IS NOT NULL LIMIT 1) > 0'
-      update_all 'international_destination = 1', '(SELECT COUNT(*) FROM flight_segments WHERE flight_segments.destination_airport_iata_code   = airports.iata_code AND flight_segments.origin_country_iso_3166_code != flight_segments.destination_country_iso_3166_code AND flight_segments.origin_country_iso_3166_code IS NOT NULL AND flight_segments.destination_country_iso_3166_code IS NOT NULL LIMIT 1) > 0'
+    process "Ensure Country is populated" do
+      Country.run_data_miner!
     end
     
-    # sabshere 5/24/10 using temporary tables because the WHERE clause has a very slow OR condition: iata_code = destination_iata_code OR iata_code = origin_iata_code
-    process "Derive some average flight characteristics from flight segments" do
-      FlightSegment.run_data_miner!
-      segments = FlightSegment.arel_table
-      airports = Airport.arel_table
-      
-      find_in_batches do |batch|
-        batch.each do |airport|
-          targeting_relation = airports[:iata_code].eq airport.iata_code
-          conditional_relation = segments[:origin_airport_iata_code].eq(airport.iata_code).or(segments[:destination_airport_iata_code].eq(airport.iata_code))
-          connection.execute "CREATE TEMPORARY TABLE tmp1 #{FlightSegment.where(conditional_relation).to_sql}"
-          update_all "seats                    = (#{FlightSegment.weighted_average_relation(:seats,         :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
-          update_all "distance                 = (#{FlightSegment.weighted_average_relation(:distance,      :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
-          update_all "load_factor              = (#{FlightSegment.weighted_average_relation(:load_factor,   :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
-          update_all "freight_share            = (#{FlightSegment.weighted_average_relation(:freight_share, :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
-          update_all "payload                  = (#{FlightSegment.weighted_average_relation(:payload,       :weighted_by => :passengers, :disaggregate_by => :departures_performed).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
-          connection.execute 'DROP TABLE tmp1'
-        end
+    process "Replace country names with country ISO 3166 codes" do
+      connection.select_values("SELECT DISTINCT country_name FROM airports WHERE country_name IS NOT NULL").each do |name|
+        code = Country.find_by_name(name).iso_3166_code
+        update_all %{country_iso_3166_code = "#{code}"}, %{country_name = "#{name}"}
       end
     end
     
+    # 10/14/2011 cutting this b/c we don't use it - Ian
+    # sabshere 5/24/10 using temporary tables because the WHERE clause has a very slow OR condition: iata_code = destination_iata_code OR iata_code = origin_iata_code
+    # process "Derive some average flight characteristics from flight segments" do
+    #   FlightSegment.run_data_miner!
+    #   segments = FlightSegment.arel_table
+    #   airports = Airport.arel_table
+    #   
+    #   find_in_batches do |batch|
+    #     batch.each do |airport|
+    #       targeting_relation = airports[:iata_code].eq airport.iata_code
+    #       conditional_relation = segments[:origin_airport_iata_code].eq(airport.iata_code).or(segments[:destination_airport_iata_code].eq(airport.iata_code))
+    #       connection.execute "CREATE TEMPORARY TABLE tmp1 #{FlightSegment.where(conditional_relation).to_sql}"
+    #       update_all "seats                    = (#{FlightSegment.weighted_average_relation(:seats,         :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+    #       update_all "distance                 = (#{FlightSegment.weighted_average_relation(:distance,      :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+    #       update_all "load_factor              = (#{FlightSegment.weighted_average_relation(:load_factor,   :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+    #       update_all "freight_share            = (#{FlightSegment.weighted_average_relation(:freight_share, :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+    #       update_all "payload                  = (#{FlightSegment.weighted_average_relation(:payload,       :weighted_by => :passengers, :disaggregate_by => :departures_performed).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+    #       connection.execute 'DROP TABLE tmp1'
+    #     end
+    #   end
+    # end
+    
     # FIXME TODO verify this
   end
 end