earth 0.0.18

data/lib/earth.rb

@@ -0,0 +1,169 @@
+require 'active_record'
+require 'cohort_scope'
+require 'earth/conversions_ext'
+require 'earth/inflectors'
+require 'data_miner'
+require 'falls_back_on'
+require 'falls_back_on/active_record_ext'
+
+# The earth module is an interface for establishing a taps server (used to fetch 
+# data) and for loading data models from various domains.
+module Earth
+  extend self
+
+  def taps_server
+    if defined?(@taps_server)
+      @taps_server
+    else
+      @taps_server = nil
+    end
+  end
+
+  # taps_server is a URL. See the data_miner gem docs
+  def taps_server=(val)
+    @taps_server = val
+  end
+
+  def classes
+    [
+      AirConditionerUse,
+      Aircraft,
+      AircraftClass,
+      AircraftManufacturer,
+      Airline,
+      Airport,
+      AutomobileFuelType,
+      AutomobileMake,
+      AutomobileMakeFleetYear,
+      AutomobileMakeYear,
+      AutomobileModel,
+      AutomobileModelYear,
+      AutomobileSizeClass,
+      AutomobileVariant,
+      Breed,
+      BreedGender,
+      BusClass,
+      CensusDivision,
+      CensusRegion,
+      ClimateDivision,
+      ClothesMachineUse,
+      Country,
+      DietClass,
+      DishwasherUse,
+      EgridRegion,
+      EgridSubregion,
+      FlightConfiguration,
+      FlightDistanceClass,
+      FlightDomesticity,
+      FlightFuelType,
+      FlightPropulsion,
+      FlightSeatClass,
+      FlightSegment,
+      FlightService,
+      FoodGroup,
+      Gender,
+      PetroleumAdministrationForDefenseDistrict,
+      RailClass,
+      ResidenceAppliance,
+      ResidenceClass,
+      ResidenceFuelPrice,
+      ResidenceFuelType,
+      ResidentialEnergyConsumptionSurveyResponse,
+      Species,
+      State,
+      Urbanity,
+      ZipCode
+    ]
+  end
+
+  def root 
+    File.join(File.dirname(__FILE__), '..')
+  end
+
+  def domains
+    %w{air automobile bus diet fuel locality pet rail residence}
+  end
+
+  # Earth.init will load any specified domains, any needed ActiveRecord plugins, 
+  # and will apply each domain model's schema to the database if the 
+  # :apply_schemas option is given. See Earth.domains for the list of allowable 
+  # domains.
+  #
+  # Earth.init should be performed after a connection is made to the database and 
+  # before any domain models are referenced.
+  def init(*args)
+    load_plugins
+
+    domains = []
+    options = {}
+    args.each do |arg|
+      if arg.is_a?(Hash)
+        options = arg
+      else
+        domains << arg
+      end
+    end
+
+    load_domains(domains, options)
+    load_schemas if options[:apply_schemas]
+  end
+
+  def database_options
+    if ActiveRecord::Base.connection.adapter_name.downcase == 'sqlite'
+      {}
+    else
+      { :options => 'ENGINE=InnoDB default charset=utf8' }
+    end
+  end
+
+private
+  def load_domains(domains, options)
+    if domains.empty? or domains.include?(:all)
+      require 'earth/all'
+      require 'earth/data_miner' if options[:apply_schemas] or options[:load_data_miner]
+    elsif !domains.include?(:none)
+      domains.each do |domain| 
+        require "earth/#{domain}"
+        require "earth/#{domain}/data_miner" if options[:apply_schemas] or options[:load_data_miner]
+      end
+    end
+  end
+
+  def load_plugins
+    Dir[File.join(Earth.root, 'vendor', '**', 'init.rb')].each do |pluginit|
+      $:.unshift File.join(File.dirname(pluginit), 'lib')
+      load pluginit
+    end
+  end
+
+  def load_schemas
+    load_ar_schema
+    load_data_miner_schemas
+  end
+
+  def load_ar_schema
+    orig_std_out = STDOUT.clone
+    STDOUT.reopen File.open(File.join('/tmp', 'schema_output'), 'w') 
+
+    load File.join(File.dirname(__FILE__), 'earth', 'schema.rb')
+  ensure
+    STDOUT.reopen(orig_std_out)
+  end
+
+  def load_data_miner_schemas
+    models = Module.constants.select do |k|
+      const = Object.const_get(k)
+      if const.instance_of?(Class)
+        const.superclass == ActiveRecord::Base
+      else
+        false
+      end
+    end
+    models.sort.each do |model|
+      klass = Object.const_get(model)
+      if klass.respond_to?(:execute_schema) and !klass.table_exists?
+        klass.execute_schema 
+      end
+    end
+  end
+end

data/lib/earth/air.rb

@@ -0,0 +1,13 @@
+require 'earth/air/aircraft'
+require 'earth/air/aircraft_class'
+require 'earth/air/aircraft_manufacturer'
+require 'earth/air/airline'
+require 'earth/air/airport'
+require 'earth/air/flight_configuration'
+require 'earth/air/flight_distance_class'
+require 'earth/air/flight_domesticity'
+require 'earth/air/flight_fuel_type'
+require 'earth/air/flight_propulsion'
+require 'earth/air/flight_seat_class'
+require 'earth/air/flight_segment'
+require 'earth/air/flight_service'

data/lib/earth/air/aircraft.rb

@@ -0,0 +1,32 @@
+class Aircraft < ActiveRecord::Base
+  set_primary_key :icao_code
+  
+  belongs_to :aircraft_class, :foreign_key => 'brighter_planet_aircraft_class_code'
+  belongs_to :manufacturer,   :foreign_key => 'manufacturer_name',                  :class_name => 'AircraftManufacturer'
+  has_many   :segments,       :foreign_key => 'bts_aircraft_type_code',             :class_name => "FlightSegment", :primary_key => 'bts_aircraft_type_code'
+
+
+  falls_back_on :m3 =>            lambda {
+    weighted_average(:m3,            :weighted_by => [:segments, :passengers])
+  }, # 9.73423082858437e-08   r7110: 8.6540464368905e-8      r6972: 8.37e-8
+                :m2 =>            lambda {
+    weighted_average(:m2,            :weighted_by => [:segments, :passengers])
+  }, # -0.000134350543484608  r7110: -0.00015337661447817    r6972: -4.09e-5
+                :m1 =>            lambda {
+    weighted_average(:m1,            :weighted_by => [:segments, :passengers])
+  }, # 6.7728101555467        r7110: 4.7781966869412         r6972: 7.85
+                :endpoint_fuel => lambda {
+    weighted_average(:endpoint_fuel, :weighted_by => [:segments, :passengers])
+  }, # 1527.81790006167       r7110: 1065.3476555284         r6972: 1.72e3
+                :seats =>         lambda {
+    weighted_average(:seats,         :weighted_by => [:segments, :passengers])
+  } # 62.1741
+
+  data_miner do
+    tap "Brighter Planet's sanitized aircraft data", Earth.taps_server
+    
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+end

data/lib/earth/air/aircraft/data_miner.rb

@@ -0,0 +1,171 @@
+Aircraft.class_eval do
+  class Aircraft::BtsMatcher
+    attr_reader :wants
+    def initialize(wants)
+      @wants = wants
+    end
+    def match(raw_faa_icao_record)
+      @_match ||= Hash.new
+      return @_match[raw_faa_icao_record] if @_match.has_key?(raw_faa_icao_record)
+      faa_icao_record = [ raw_faa_icao_record['Manufacturer'] + ' ' + raw_faa_icao_record['Model'] ]
+      bts_record = Aircraft.bts_name_dictionary.left_to_right faa_icao_record
+      retval = case wants
+      when :bts_aircraft_type_code
+        bts_record['Code']
+      when :bts_name
+        bts_record['Description']
+      end if bts_record
+      @_match[raw_faa_icao_record] = retval
+    end
+  end
+
+  class Aircraft::FuelUseMatcher
+    def match(raw_fuel_use_record)
+      @_match ||= Hash.new
+      return @_match[raw_fuel_use_record] if @_match.has_key?(raw_fuel_use_record)
+
+      aircraft_record = if raw_fuel_use_record['ICAO'] =~ /\A[0-9A-Z]+\z/
+        Aircraft.find_by_icao_code raw_fuel_use_record['ICAO']
+      end
+      
+      aircraft_record ||= if raw_fuel_use_record['Aircraft Name'].present?
+        Aircraft.icao_name_dictionary.left_to_right [ raw_fuel_use_record['Aircraft Name'] ]
+      end
+      
+      if aircraft_record
+        @_match[raw_fuel_use_record] = aircraft_record.icao_code
+      else
+        raise "Didn't find a match for #{raw_fuel_use_record['Aircraft Name']} (#{raw_fuel_use_record['ICAO']}), which we found in the fuel use spreadsheet"
+      end
+    end
+  end
+
+  class Aircraft::Guru
+    # for errata
+    def is_a_dc_plane?(row)
+      row['Designator'] =~ /^DC\d/i
+    end
+    
+    # def is_a_crj_900?(row)
+    #   row['Designator'].downcase == 'crj9'
+    # end
+    
+    def is_a_g159?(row)
+      row['Designator'] =~ /^G159$/
+    end
+
+    def is_a_galx?(row)
+      row['Designator'] =~ /^GALX$/
+    end
+    
+    def method_missing(method_id, *args, &block)
+      if method_id.to_s =~ /\Ais_n?o?t?_?attributed_to_([^\?]+)/
+        manufacturer_name = $1
+        manufacturer_regexp = Regexp.new(manufacturer_name.gsub('_', ' ?'), Regexp::IGNORECASE)
+        matches = manufacturer_regexp.match(args.first['Manufacturer']) # row['Manufacturer'] =~ /mcdonnell douglas/i
+        method_id.to_s.include?('not_attributed') ? matches.nil? : !matches.nil?
+      else
+        super
+      end
+    end
+  end
+
+  data_miner do
+    process "Don't re-import too often" do
+      raise DataMiner::Skip unless DataMiner::Run.allowed? Aircraft
+    end
+    
+    schema Earth.database_options do
+      string   'icao_code'
+      string   'manufacturer_name'
+      string   'name'
+      string   'bts_name'
+      string   'bts_aircraft_type_code'
+      string   'brighter_planet_aircraft_class_code'
+      string   'fuel_use_aircraft_name'
+      float    'm3'
+      string   'm3_units'
+      float    'm2'
+      string   'm2_units'
+      float    'm1'
+      string   'm1_units'
+      float    'endpoint_fuel'
+      string   'endpoint_fuel_units'
+      float    'seats'
+      float    'distance'
+      string   'distance_units'
+      float    'load_factor'
+      float    'freight_share'
+      float    'payload'
+      float    'weighting'
+      index    'bts_aircraft_type_code'
+    end
+    
+    ('A'..'Z').each do |letter|
+    # ('Z'..'Z').each do |letter|
+      import( "ICAO aircraft codes starting with the letter #{letter} used by the FAA",
+              :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
+              :errata => Errata.new(:url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
+                                    :responder => Aircraft::Guru.new),
+              :encoding => 'windows-1252',
+              :row_xpath => '//table/tr[2]/td/table/tr',
+              :column_xpath => 'td' ) do
+        key 'icao_code', :field_name => 'Designator'
+        store 'bts_aircraft_type_code', :matcher => Aircraft::BtsMatcher.new(:bts_aircraft_type_code)
+        store 'bts_name', :matcher => Aircraft::BtsMatcher.new(:bts_name)
+        store 'manufacturer_name', :field_name => 'Manufacturer'
+        store 'name', :field_name => 'Model'
+      end
+    end
+    
+    # TODO fixme need to remake aircraft classes dictionary based on ICAO codes
+    # sabshere 5/17/10 or maybe we can replace this with typ/weight class from FAA (?)
+    import "Brighter Planet's aircraft class codes",
+           :url => 'http://static.brighterplanet.com/science/data/transport/air/bts_aircraft_type/bts_aircraft_types-brighter_planet_aircraft_classes.csv' do
+      key   'bts_aircraft_type_code', :field_name => 'bts_aircraft_type'
+      store 'brighter_planet_aircraft_class_code'
+    end
+
+    # EEA
+    import "pre-calculated fuel use equation coefficients",
+           :url => 'http://static.brighterplanet.com/science/data/transport/air/fuel_use/aircraft_fuel_use_formulae.ods',
+           :select => lambda { |row| row['ICAO'].present? or row['Aircraft Name'].present? } do
+      key   'icao_code', :matcher => Aircraft::FuelUseMatcher.new
+      store 'fuel_use_aircraft_name', :field_name => 'Aircraft Name'
+      store 'm3'
+      store 'm2'
+      store 'm1'
+      store 'endpoint_fuel', :field_name => 'b'
+    end
+    
+    process "Derive some average flight characteristics from flight segments" do
+      FlightSegment.run_data_miner!
+      aircraft = Aircraft.arel_table
+      segments = FlightSegment.arel_table
+
+      # non-working joins method
+      # update_all "aircraft.distance_1            = (SELECT * FROM (#{FlightSegment.joins(:aircraft).weighted_average_relation(:distance,            :weighted_by => :passengers                                           ).to_sql}) AS anonymous_1)"
+      # update_all "aircraft.load_factor_1         = (SELECT * FROM (#{FlightSegment.joins(:aircraft).weighted_average_relation(:load_factor,         :weighted_by => :passengers                                           ).to_sql}) AS anonymous_1)"
+      # execute %{
+      #   update aircraft as t1
+      #   set t1.distance_1 = (SELECT * FROM (#{FlightSegment.joins(:aircraft).weighted_average_relation(:distance,            :weighted_by => :passengers                                           ).where('t1.bts_aircraft_type_code = flight_segments.bts_aircraft_type_code').to_sql}) AS anonymous_1)
+      # }
+
+      conditional_relation = aircraft[:bts_aircraft_type_code].eq(segments[:bts_aircraft_type_code])
+      update_all "seats         = (#{FlightSegment.weighted_average_relation(:seats,         :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+      update_all "distance      = (#{FlightSegment.weighted_average_relation(:distance,      :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+      update_all "load_factor   = (#{FlightSegment.weighted_average_relation(:load_factor,   :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+      update_all "freight_share = (#{FlightSegment.weighted_average_relation(:freight_share, :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
+      update_all "payload       = (#{FlightSegment.weighted_average_relation(:payload,       :weighted_by => :passengers, :disaggregate_by => :departures_performed).where(conditional_relation).to_sql})"
+      
+      update_all "weighting = (#{segments.project(segments[:passengers].sum).where(aircraft[:bts_aircraft_type_code].eq(segments[:bts_aircraft_type_code])).to_sql})"
+    end
+    
+    [ AircraftManufacturer ].each do |synthetic_resource|
+      process "Synthesize #{synthetic_resource}" do
+        synthetic_resource.run_data_miner!
+      end
+    end
+    
+  end
+end

data/lib/earth/air/aircraft_class.rb

@@ -0,0 +1,10 @@
+class AircraftClass < ActiveRecord::Base
+  set_primary_key :brighter_planet_aircraft_class_code
+  
+  has_many :aircraft, :foreign_key => 'brighter_planet_aircraft_class_code'
+#  has_many :airline_aircraft_seat_classes, :through => :aircraft
+
+  data_miner do
+    tap "Brighter Planet's aircraft class data", Earth.taps_server
+  end
+end

data/lib/earth/air/aircraft_class/data_miner.rb

@@ -0,0 +1,42 @@
+AircraftClass.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string  'brighter_planet_aircraft_class_code'
+      string  'name'
+      float   'm1'
+      float   'm2'
+      float   'm3'
+      float   'endpoint_fuel'
+      integer 'seats'
+    end
+    
+    import "Brighter Planet's aircraft classes", :url => 'http://static.brighterplanet.com/science/data/transport/air/brighter_planet_aircraft_classes.csv' do
+      key 'brighter_planet_aircraft_class_code'
+      store 'name', :field_name => 'description'
+    end
+    
+    process "Derive some average aircraft chraracteristics from aircraft" do
+      Aircraft.run_data_miner!
+      aircraft = Aircraft.arel_table
+      aircraft_classes = AircraftClass.arel_table
+      conditional_relation = aircraft_classes[:brighter_planet_aircraft_class_code].eq(aircraft[:brighter_planet_aircraft_class_code])
+      %w{ m1 m2 m3 endpoint_fuel }.each do |column|
+        relation = Aircraft.weighted_average_relation(column).
+                            where(conditional_relation)
+        update_all "#{column} = (#{relation.to_sql})"
+      end
+    end
+    
+    process "Derive some average aircraft characteristics from flight segments" do # FIXME TODO why not derive this from aircraft?
+      FlightSegment.run_data_miner!
+      aircraft = Aircraft.arel_table
+      aircraft_classes = AircraftClass.arel_table
+      segments = FlightSegment.arel_table
+      relation = FlightSegment.joins(:aircraft). # this requires associations
+                               weighted_average_relation(:seats, :weighted_by => :passengers).
+                               where(aircraft_classes[:brighter_planet_aircraft_class_code].eq(aircraft[:brighter_planet_aircraft_class_code]))
+      update_all "seats = (#{relation.to_sql})"
+    end
+  end
+end
+

data/lib/earth/air/aircraft_manufacturer.rb

@@ -0,0 +1,9 @@
+class AircraftManufacturer < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :aircraft, :foreign_key => 'manufacturer_name'
+  
+  data_miner do
+    tap "Brighter Planet's aircraft manufacturer data", Earth.taps_server
+  end
+end