earth 0.0.18

data/lib/earth/air/aircraft_manufacturer/data_miner.rb

@@ -0,0 +1,20 @@
+AircraftManufacturer.class_eval do
+  data_miner do
+    process "Start from scratch" do
+      connection.drop_table table_name
+    end
+    
+    schema Earth.database_options do
+      string 'name'
+    end
+
+    process "Derive a list of aircraft manufacturers from aircraft" do
+      Aircraft.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO aircraft_manufacturers(name)
+        SELECT aircraft.manufacturer_name FROM aircraft WHERE LENGTH(aircraft.manufacturer_name) > 0
+      }
+    end
+  end
+end
+

data/lib/earth/air/airline.rb

@@ -0,0 +1,16 @@
+class Airline < ActiveRecord::Base
+  set_primary_key :iata_code
+
+#  has_many :airline_aircraft, :class_name => 'AirlineAircraft'
+#  has_many :seat_classes, :class_name => 'AirlineSeatClass'
+  has_many :segments, :class_name => "FlightSegment", :foreign_key => 'airline_iata_code'
+#  has_many :airline_aircraft_seat_classes, :class_name => 'AirlineAircraftSeatClass'
+
+  data_miner do
+    tap "Brighter Planet's sanitized airlines data", Earth.taps_server
+  end
+  
+  def all_flights_domestic?
+    !international?
+  end
+end

data/lib/earth/air/airline/data_miner.rb

@@ -0,0 +1,57 @@
+Airline.class_eval do
+  class Airline::Guru
+    # needed by errata
+    def is_not_coral_air?(row); row['Code'].to_i != 19155; end                # 19155 Coral Air Inc.: COR
+    def is_not_aviacion_y_comercio?(row); row['Code'].to_i != 19452; end      # 19452 Aviacion Y Comercio S.A.: AO
+    def is_not_air_china?(row); row['Code'].to_i != 19543; end                # 19543 Air China: CA
+    def is_not_south_african_airways?(row); row['Code'].to_i != 19570; end    # 19570 South African Airways: SA
+    def is_not_continental_airlines?(row); row['Code'].to_i != 19704; end     # 19704 Continental Air Lines Inc.: CO
+    def is_not_sallee_s_aviation?(row); row['Code'].to_i != 19740; end        # 19740 Sallee's Aviation: SAL
+    def is_not_air_berlin?(row); row['Code'].to_i != 21361; end               # "21361","Air Berlin PLC and CO: AB"
+  end
+
+  data_miner do
+    schema Earth.database_options do
+      string   'iata_code'
+      string   'name'
+      string   'dot_airline_id_code'
+      boolean  'international'
+      float    'seats'
+      float    'distance'
+      string   'distance_units'
+      float    'load_factor'
+      float    'freight_share'
+      float    'payload'
+      string   'payload_units'
+    end
+    
+    import "the T100 AIRLINE_ID lookup table, which also includes IATA codes",
+           :url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_AIRLINE_ID',
+           :errata => Errata.new(:url => 'http://static.brighterplanet.com/science/data/transport/air/airlines/errata.csv',
+                                 :responder => Airline::Guru.new) do
+      key 'iata_code', :field_name => 'Description', :split => { :pattern => /:/, :keep => 1 }
+      store 'dot_airline_id_code', :field_name => 'Code'
+      store 'name', :field_name => 'Description', :split => { :pattern => /:/, :keep => 0 }
+    end
+    
+    process "Determine whether airlines fly internationally by looking at flight segments" do
+      FlightSegment.run_data_miner!
+      update_all 'international = 1', '(SELECT COUNT(*) FROM flight_segments WHERE flight_segments.airline_iata_code = airlines.iata_code AND flight_segments.origin_country_iso_3166_code != flight_segments.dest_country_iso_3166_code AND flight_segments.origin_country_iso_3166_code IS NOT NULL AND flight_segments.dest_country_iso_3166_code IS NOT NULL) > 0'
+    end
+    
+    process "Derive some average flight characteristics from flight segments" do
+      FlightSegment.run_data_miner!
+      airlines = Airline.arel_table
+      segments = FlightSegment.arel_table
+      
+      conditional_relation = airlines[:iata_code].eq(segments[:airline_iata_code])
+
+      update_all "seats                    = (#{FlightSegment.weighted_average_relation(:seats,               :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+      update_all "distance                 = (#{FlightSegment.weighted_average_relation(:distance,            :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+      update_all "load_factor              = (#{FlightSegment.weighted_average_relation(:load_factor,         :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+      update_all "freight_share            = (#{FlightSegment.weighted_average_relation(:freight_share,       :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+      update_all "payload                  = (#{FlightSegment.weighted_average_relation(:payload,             :weighted_by => :passengers, :disaggregate_by => :departures_performed).where(conditional_relation).to_sql})"
+    end
+  end
+end
+

data/lib/earth/air/airport.rb

@@ -0,0 +1,44 @@
+class Airport < ActiveRecord::Base
+  set_primary_key :iata_code
+  
+  # --------------------------------
+  # virtual has_many association
+  # has_many :segments won't work because there's no general way to specify the correct conditions
+  # even if you get clever with it, like
+  # has_many  :segments,
+  #           :class_name => 'FlightSegment',
+  #           :foreign_key => 'origin_airport_id',
+  #           :conditions => 'flight_segments.destination_airport_id = #{id}'
+  # you get queries like "`flight_segments`.origin_airport_id = 3654 AND (flight_segments.destination_airport_id = 3654))"
+  # in which you notice the AND which must be an OR
+  # and you can't just do finder_sql, because that breaks any other :select
+  def segments
+    FlightSegment.scoped :conditions => ['origin_airport_id = ? OR destination_airport_id = ?', id, id]
+  end
+  # --------------------------------
+  
+  belongs_to :country, :foreign_key => 'country_iso_3166_code'
+  acts_as_mappable :default_units => :nms,
+                   :lat_column_name => :latitude,
+                   :lng_column_name => :longitude
+
+  data_miner do
+    tap "Brighter Planet's sanitized airports data", Earth.taps_server
+    
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+  
+  def all_flights_from_here_domestic?
+    !international_origin?
+  end
+
+  def all_flights_to_here_domestic?
+    !international_destination?
+  end
+  
+  def united_states?
+    country == Country.united_states
+  end
+end

data/lib/earth/air/airport/data_miner.rb

@@ -0,0 +1,80 @@
+Airport.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'iata_code'
+      string   'name'
+      string   'city'
+      string   'country_name'
+      string   'country_iso_3166_code'
+      float    'latitude'
+      float    'longitude'
+      float    'seats'
+      float    'distance'
+      string   'distance_units'
+      float    'load_factor'
+      float    'freight_share'
+      float    'payload'
+      string   'payload_units'
+      boolean  'international_origin'
+      boolean  'international_destination'
+    end
+    
+    import "the OpenFlights.org airports database",
+           :url => 'http://openflights.svn.sourceforge.net/viewvc/openflights/openflights/data/airports.dat',
+           :headers => false,
+           :select => lambda { |row| row[4].present? } do
+      key   'iata_code', :field_number => 4
+      store 'name', :field_number => 1
+      store 'city', :field_number => 2
+      store 'country_name', :field_number => 3
+      store 'country_iso_3166_code', :field_number => 3, :upcase => true, :dictionary => { :input => 'name', :output => 'iso_3166_code', :url => 'http://data.brighterplanet.com/countries.csv' }
+      store 'latitude', :field_number => 6
+      store 'longitude', :field_number => 7
+    end
+    
+    # step.await :other_class => FlightSegment do |deferred|
+    #   deferred.derive :country # this uses a heuristic that depends on flight segments
+    # class << self
+    #   def derive_country
+    #     update_all('country_id = (SELECT      flight_segments.origin_country_id FROM flight_segments WHERE      flight_segments.origin_airport_id = airports.id AND      flight_segments.origin_country_id IS NOT NULL LIMIT 1)', 'airports.country_id IS NULL')
+    #     update_all('country_id = (SELECT flight_segments.destination_country_id FROM flight_segments WHERE flight_segments.destination_airport_id = airports.id AND flight_segments.destination_country_id IS NOT NULL LIMIT 1)', 'airports.country_id IS NULL')
+    #     Country.all.each do |c|
+    #       next if c.name.blank?
+    #       update_all("country_id = #{c.id}", ["airports.country_id IS NULL AND airports.country_name LIKE ?", "%#{c.name.upcase}%"])
+    #     end
+    #     Airport.all(:conditions => 'country_id IS NULL AND country_name IS NOT NULL').each do |a|
+    #       c = Country.find(:first, :conditions => ["name like ?", "%#{a.country_name}%"])
+    #       a.update_attributes(:country_id => c.id) if c
+    #     end
+    #   end
+    # end
+
+    process "Determine whether each airport serves international flights" do
+      FlightSegment.run_data_miner!
+      update_all 'international_origin = 1',      '(SELECT COUNT(*) FROM flight_segments WHERE flight_segments.origin_airport_iata_code = airports.iata_code AND flight_segments.origin_country_iso_3166_code != flight_segments.dest_country_iso_3166_code AND flight_segments.origin_country_iso_3166_code IS NOT NULL AND flight_segments.dest_country_iso_3166_code IS NOT NULL LIMIT 1) > 0'
+      update_all 'international_destination = 1', '(SELECT COUNT(*) FROM flight_segments WHERE flight_segments.dest_airport_iata_code   = airports.iata_code AND flight_segments.origin_country_iso_3166_code != flight_segments.dest_country_iso_3166_code AND flight_segments.origin_country_iso_3166_code IS NOT NULL AND flight_segments.dest_country_iso_3166_code IS NOT NULL LIMIT 1) > 0'
+    end
+    
+    # sabshere 5/24/10 using temporary tables because the WHERE clause has a very slow OR condition: iata_code = dest_iata_code OR iata_code = origin_iata_code
+    process "Derive some average flight characteristics from flight segments" do
+      FlightSegment.run_data_miner!
+      segments = FlightSegment.arel_table
+      airports = Airport.arel_table
+
+      find_in_batches do |batch|
+        batch.each do |airport|
+          targeting_relation = airports[:iata_code].eq airport.iata_code
+          conditional_relation = segments[:origin_airport_iata_code].eq(airport.iata_code).or(segments[:dest_airport_iata_code].eq(airport.iata_code))
+          connection.execute "CREATE TEMPORARY TABLE tmp1 #{FlightSegment.where(conditional_relation).to_sql}"
+          update_all "seats                    = (#{FlightSegment.weighted_average_relation(:seats,         :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+          update_all "distance                 = (#{FlightSegment.weighted_average_relation(:distance,      :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+          update_all "load_factor              = (#{FlightSegment.weighted_average_relation(:load_factor,   :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+          update_all "freight_share            = (#{FlightSegment.weighted_average_relation(:freight_share, :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+          update_all "payload                  = (#{FlightSegment.weighted_average_relation(:payload,       :weighted_by => :passengers, :disaggregate_by => :departures_performed).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+          connection.execute 'DROP TABLE tmp1'
+        end
+      end
+    end
+  end
+end
+

data/lib/earth/air/data_miner.rb

@@ -0,0 +1,15 @@
+require 'earth/locality'
+require 'earth/locality/data_miner'
+require 'earth/air/aircraft/data_miner'
+require 'earth/air/aircraft_class/data_miner'
+require 'earth/air/aircraft_manufacturer/data_miner'
+require 'earth/air/airline/data_miner'
+require 'earth/air/airport/data_miner'
+require 'earth/air/flight_configuration/data_miner'
+require 'earth/air/flight_distance_class/data_miner'
+require 'earth/air/flight_domesticity/data_miner'
+require 'earth/air/flight_fuel_type/data_miner'
+require 'earth/air/flight_propulsion/data_miner'
+require 'earth/air/flight_seat_class/data_miner'
+require 'earth/air/flight_segment/data_miner'
+require 'earth/air/flight_service/data_miner'

data/lib/earth/air/flight_configuration.rb

@@ -0,0 +1,18 @@
+class FlightConfiguration < ActiveRecord::Base
+  set_primary_key :name
+  
+  data_miner do
+    schema do
+      string 'name'
+      string 'bts_aircraft_configuration_code'
+    end
+
+    process "derive from flight segments" do
+      FlightSegment.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO flight_configurations(name, bts_aircraft_configuration_code)
+        SELECT flight_segments.configuration_id, flight_segments.bts_aircraft_configuration_code FROM flight_segments WHERE LENGTH(flight_segments.configuration_id) > 0
+      }
+    end
+  end
+end

data/lib/earth/air/flight_configuration/data_miner.rb

@@ -0,0 +1,4 @@
+FlightConfiguration.class_eval do
+#  data_miner do
+#...
+end

data/lib/earth/air/flight_distance_class.rb

@@ -0,0 +1,7 @@
+class FlightDistanceClass < ActiveRecord::Base
+  set_primary_key :name
+  
+  data_miner do
+    tap "Brighter Planet's sanitized distance class data", Earth.taps_server
+  end
+end

data/lib/earth/air/flight_distance_class/data_miner.rb

@@ -0,0 +1,16 @@
+FlightDistanceClass.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string 'name'
+      float  'distance'
+      string 'distance_units'
+    end
+    
+    import "a list of Brighter Planet-defined distance classes",
+           :url => 'http://static.brighterplanet.com/science/data/transport/air/distance_classes/distance_classes.csv' do
+      key   'name'
+      store 'distance', :units_field_name => 'units', :to_units => :kilometres
+    end
+  end
+end
+

data/lib/earth/air/flight_domesticity.rb

@@ -0,0 +1,6 @@
+class FlightDomesticity < ActiveRecord::Base
+  set_primary_key :name
+  data_miner do
+    tap "Brighter Planet's flight domesticity info", Earth.taps_server
+  end
+end

data/lib/earth/air/flight_domesticity/data_miner.rb

@@ -0,0 +1,57 @@
+FlightDomesticity.class_eval do
+  data_miner do
+    process "Start from scratch" do
+      connection.drop_table table_name
+    end
+    
+    schema Earth.database_options do
+      string 'name'
+      string 'bts_data_source_code'
+      float  'distance'
+      string 'distance_units'
+      float  'freight_share'
+      float  'load_factor'
+      float  'seats'
+      float  'payload'
+      string 'payload_units'
+    end
+    
+    process "Derive flight domesticities from flight segments" do # FIXME TODO might make more sense to combine foreign and domestic carriers -> domestic flight (all carriers), international flight (all carriers)
+      FlightSegment.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO flight_domesticities(name, bts_data_source_code)
+        SELECT flight_segments.domesticity, flight_segments.bts_data_source_code FROM flight_segments WHERE LENGTH(flight_segments.domesticity) > 0
+      }
+    end
+    
+    process "Derive average flight characteristics from flight segments" do
+      FlightSegment.run_data_miner!
+      segments = FlightSegment.arel_table
+      flight_domesticities = FlightDomesticity.arel_table
+      ## slow, all-in-one method
+      # conditional_relation = flight_domesticities[:name].eq(segments[:domesticity])
+      # update_all "seats         = (#{FlightSegment.weighted_average_relation(:seats,         :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+      # update_all "distance      = (#{FlightSegment.weighted_average_relation(:distance,      :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+      # update_all "load_factor   = (#{FlightSegment.weighted_average_relation(:load_factor,   :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+      # update_all "freight_share = (#{FlightSegment.weighted_average_relation(:freight_share, :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+      # update_all "payload       = (#{FlightSegment.weighted_average_relation(:payload,       :weighted_by => :passengers, :disaggregate_by => :departures_performed).where(conditional_relation).to_sql})"
+      ## fast method using temp tables
+      find_in_batches do |batch|
+        batch.each do |flight_domesticity|
+          targeting_relation = flight_domesticities[:name].eq flight_domesticity.name
+          conditional_relation = segments[:domesticity].eq flight_domesticity.name
+          connection.execute "CREATE TEMPORARY TABLE tmp1 #{FlightSegment.where(conditional_relation).to_sql}"
+          update_all "seats                    = (#{FlightSegment.weighted_average_relation(:seats,         :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+          update_all "distance                 = (#{FlightSegment.weighted_average_relation(:distance,      :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+          update_all "load_factor              = (#{FlightSegment.weighted_average_relation(:load_factor,   :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+          update_all "freight_share            = (#{FlightSegment.weighted_average_relation(:freight_share, :weighted_by => :passengers                                           ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+          update_all "payload                  = (#{FlightSegment.weighted_average_relation(:payload,       :weighted_by => :passengers, :disaggregate_by => :departures_performed).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
+          connection.execute 'DROP TABLE tmp1'
+        end
+      end
+      update_all "distance_units = 'kilometres'"
+      update_all "payload_units  = 'kilograms'"
+    end
+  end
+end
+

data/lib/earth/air/flight_fuel_type.rb

@@ -0,0 +1,17 @@
+class FlightFuelType < ActiveRecord::Base
+  # this fallback is jet fuel
+  falls_back_on :emission_factor =>           (21.09.pounds.to(:kilograms) / 1.gallons.to(:litres)),  # in pounds CO2/gallon fuel: http://www.eia.doe.gov/oiaf/1605/excel/Fuel%20Emission%20Factors.xls
+                :radiative_forcing_index =>   2,                                                      # from Matt
+                :density =>                   3.057                                                   # kg / gal
+  
+  data_miner do
+    schema do
+      string 'name'
+      float  'emission_factor'
+      float  'radiative_forcing_index'
+      float  'density'
+    end
+    
+    # we just always use the fallback
+  end
+end

data/lib/earth/air/flight_fuel_type/data_miner.rb

@@ -0,0 +1,4 @@
+FlightFuelType.class_eval do
+#  data_miner do
+#...
+end

data/lib/earth/air/flight_propulsion.rb

@@ -0,0 +1,18 @@
+class FlightPropulsion < ActiveRecord::Base
+  set_primary_key :name
+  
+  data_miner do
+    schema do
+      string 'name'
+      string 'bts_aircraft_group_code'
+    end
+    
+    process "derive from flight segments" do
+      FlightSegment.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO flight_propulsions(name, bts_aircraft_group_code)
+        SELECT flight_segments.propulsion_id, flight_segments.bts_aircraft_group_code FROM flight_segments WHERE LENGTH(flight_segments.propulsion_id) > 0
+      }
+    end
+  end
+end

data/lib/earth/air/flight_propulsion/data_miner.rb

@@ -0,0 +1,4 @@
+FlightPropulsion.class_eval do
+#  data_miner do
+#...
+end

data/lib/earth/air/flight_seat_class.rb

@@ -0,0 +1,12 @@
+class FlightSeatClass < ActiveRecord::Base
+  set_primary_key :name
+#  has_many :airline_seat_classes, :class_name => 'AirlineSeatClass'
+#  has_many :aircraft_seat_classes, :class_name => 'AircraftSeatClass'
+#  has_many :airline_aircraft_seat_classes, :class_name => 'AirlineAircraftSeatClass'
+  
+  falls_back_on :multiplier => 1
+
+  data_miner do
+    tap "Brighter Planet's sanitized flight seat class data", Earth.taps_server
+  end
+end

data/lib/earth/air/flight_seat_class/data_miner.rb

@@ -0,0 +1,36 @@
+FlightSeatClass.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'name'
+      float    'multiplier'
+      integer  'seats'
+    end
+    
+    import "a list of Brighter Planet-defined seat classes and pre-calculated multipliers",
+           :url => 'http://static.brighterplanet.com/science/data/transport/air/seat_classes/seat_classes.csv' do
+      key   'name'
+      store 'multiplier'
+    end
+
+    # sabshere 5/21/10 in case we ever need this
+    # class << self
+    #   def refresh
+    #     update_all_weighted_averages(:seats,      :weighted_by => :airline_aircraft_seat_classes)
+    #     update_all_weighted_averages(:multiplier, :weighted_by => :airline_aircraft_seat_classes)
+    #   end
+    # 
+    #   def safe_find_by_name(name)
+    #     guess = case name.to_s.downcase.gsub(/[^a-z]/, '')
+    #     when /first/
+    #       'first'
+    #     when /busi/, /exec/
+    #       'business'
+    #     when /econ/, /coach/
+    #       'economy'
+    #     end
+    #     find_or_create_by_name(guess) unless guess.nil?
+    #   end
+    # end
+  end
+end
+