earth 0.4.12 → 0.5.0

data/.gitignore

@@ -18,6 +18,7 @@ coverage
 rdoc
 pkg
 doc
+log
 
 ## PROJECT::SPECIFIC
 Gemfile.lock

data/Gemfile

@@ -6,6 +6,6 @@ gemspec
 gem 'data_miner', :path => ENV['LOCAL_DATA_MINER'] if ENV['LOCAL_DATA_MINER']
 gem 'sniff', :path => ENV['LOCAL_SNIFF'] if ENV['LOCAL_SNIFF']
 
-if RUBY_VERSION =~ /^1\.8/
+if RUBY_VERSION < "1.9"
   gem 'fastercsv'
 end

data/earth.gemspec

@@ -5,7 +5,7 @@ require "earth/version"
 Gem::Specification.new do |s|
   s.name        = "earth"
   s.version     = Earth::VERSION
-  s.date = "2011-05-21"
+  s.date = "2011-06-06"
   s.platform    = Gem::Platform::RUBY
   s.authors = ["Seamus Abshere", "Derek Kastner", "Andy Rossmeissl"]
   s.email = %q{andy@rossmeissl.net}
@@ -26,12 +26,15 @@ Gem::Specification.new do |s|
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.3.7}
-
-  s.add_runtime_dependency 'cohort_scope', '>=0.2.0'
+  
   s.add_runtime_dependency 'data_miner'
+  s.add_runtime_dependency 'to_regexp'
+  s.add_runtime_dependency 'cohort_scope'
   s.add_runtime_dependency 'remote_table', '>=1.2.3'
   s.add_runtime_dependency 'falls_back_on'
+  s.add_runtime_dependency 'fixed_width-multibyte'
   s.add_runtime_dependency 'geokit-rails'
+  s.add_runtime_dependency 'loose_tight_dictionary', '>=0.2.3'
   s.add_runtime_dependency 'weighted_average'
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'bueller'
@@ -40,5 +43,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rdoc'
   s.add_development_dependency 'rdoc'
   s.add_development_dependency 'rspec'
+  s.add_development_dependency 'ruby-debug19'
   s.add_development_dependency 'sqlite3-ruby'
 end

data/features/automobile_make_model_year_variant.feature

@@ -8,11 +8,11 @@ Feature: Data import for AutomobileMakeModelYearVariant
     When a data import verifies "Year should be from 1985 to 2011"
     Then the verification should be successful
 
-  # FIXME TODO
-  # Scenario: Successfully verifying that fuel type code is found in AutomobileFuel
-  #   Given a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_good"
-  #   When a data import verifies "Fuel type code should be found in AutomobileFuel"
-  #   Then the verification should be successful
+  Scenario: Successfully verifying that fuel type code is found in AutomobileFuel
+    Given a "AutomobileFuel" data import fetches results listed in "automobile_fuel_good"
+    And a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_good"
+    When a data import verifies "Fuel code should appear in AutomobileFuel"
+    Then the verification should be successful
 
   Scenario: Successfully verifying that fuel efficiencies are greater than zero
     Given a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_good"
@@ -29,11 +29,11 @@ Feature: Data import for AutomobileMakeModelYearVariant
     When a data import verifies "Year should be from 1985 to 2011"
     Then the verification should not be successful
 
-  # FIXME TODO
-  # Scenario: Failing to verify that fuel type code is found in AutomobileFuel
-  #   Given a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_bad"
-  #   When a data import verifies "Fuel type code should be found in AutomobileFuel"
-  #   Then the verification should not be successful
+  Scenario: Failing to verifying that fuel type code is found in AutomobileFuel
+    Given a "AutomobileFuel" data import fetches results listed in "automobile_fuel_good"
+    And a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_bad"
+    When a data import verifies "Fuel code should appear in AutomobileFuel"
+    Then the verification should not be successful
 
   Scenario: Failing to verify that fuel efficiencies are greater than zero
     Given a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_bad"

data/features/step_definitions/data_steps.rb

@@ -1,7 +1,7 @@
 require 'active_record'
 require 'active_record/fixtures'
 
-Given /^a "([^\"]*)" data import fetches results listed in "(.*)"$/ do |model, file_name|
+Given /^an? "([^\"]*)" data import fetches results listed in "(.*)"$/ do |model, file_name|
   @data_import_model = model.constantize
   table_name = @data_import_model.table_name
 

data/features/support/env.rb

@@ -6,10 +6,10 @@ Bundler.setup
 require 'earth'
 
 require 'data_miner'
-DataMiner.logger = Logger.new nil
+DataMiner.logger = Logger.new 'log/test.log'
 
 require 'active_record'
 require 'sqlite3'
-ActiveRecord::Base.logger = Logger.new nil
+ActiveRecord::Base.logger = Logger.new 'log/test.log'
 ActiveRecord::Base.establish_connection :adapter => 'sqlite3', :database => ':memory:'
 Earth.init :all, :apply_schemas => true

data/features/support/imports/automobile_fuel_good.csv

@@ -1,3 +1,3 @@
-name,co2_emission_factor,co2_emission_factor_units,co2_biogenic_emission_factor,co2_biogenic_emission_factor_units,ch4_emission_factor,ch4_emission_factor_units,n2o_emission_factor,n2o_emission_factor_units,hfc_emission_factor,hfc_emission_factor_units,base_fuel_name,distance_key,ef_key,blend_portion
-regular gasoline,1,kilograms_per_litre,0,kilograms_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,Motor Gasoline,gasoline,gasoline
-b20,0,kilograms_per_litre,1,kilograms_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,Distillate Fuel Oil No. 2,diesel,diesel,0.2
+name,code,co2_emission_factor,co2_emission_factor_units,co2_biogenic_emission_factor,co2_biogenic_emission_factor_units,ch4_emission_factor,ch4_emission_factor_units,n2o_emission_factor,n2o_emission_factor_units,hfc_emission_factor,hfc_emission_factor_units,base_fuel_name,distance_key,ef_key,blend_portion
+regular gasoline,R,1,kilograms_per_litre,0,kilograms_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,Motor Gasoline,gasoline,gasoline
+b20,BP-B20,0,kilograms_per_litre,1,kilograms_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,Distillate Fuel Oil No. 2,diesel,diesel,0.2

data/features/support/imports/automobile_make_model_year_variant_bad.csv

@@ -1,3 +1,3 @@
 row_hash,year,fuel_code,fuel_efficiency,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_units,fuel_efficiency_city_units,fuel_efficiency_highway_units
-a1,2000,D,12.0,10.0,15.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
-b2,2012,P,0,-10,0,foo,bar
+a1,2000,R,12.0,10.0,15.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
+b2,2012,X,0,-10,0,foo,bar

data/features/support/imports/automobile_make_model_year_variant_good.csv

@@ -1,3 +1,3 @@
 row_hash,year,fuel_code,fuel_efficiency,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_units,fuel_efficiency_city_units,fuel_efficiency_highway_units
-a1,2000,D,12.0,10.0,15.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
-b2,2011,X,20.5,20.0,21.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
+a1,2000,R,12.0,10.0,15.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
+b2,2011,BP-B20,20.5,20.0,21.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre

data/lib/earth/air.rb

@@ -1,9 +1,10 @@
 require 'earth/air/aircraft'
 require 'earth/air/aircraft_class'
+require 'earth/air/aircraft_fuel_use_equation'
 require 'earth/air/aircraft_manufacturer'
 require 'earth/air/airline'
 require 'earth/air/airport'
+require 'earth/air/bts_aircraft'
 require 'earth/air/flight_distance_class'
-require 'earth/air/flight_fuel_type'
 require 'earth/air/flight_seat_class'
 require 'earth/air/flight_segment'

data/lib/earth/air/aircraft.rb

@@ -1,14 +1,27 @@
+# need this for association with FlightSegment through loose_tight_dictionary_cached_results
+require 'loose_tight_dictionary/cached_result'
+
 class Aircraft < ActiveRecord::Base
-  set_primary_key :bp_code
+  set_primary_key :icao_code
   
-  belongs_to :manufacturer,   :foreign_key => 'manufacturer_name', :primary_key => 'name',     :class_name => 'AircraftManufacturer'
-  belongs_to :aircraft_class, :foreign_key => 'class_code',        :primary_key => 'code'
-  has_many   :segments,       :foreign_key => 'aircraft_bts_code', :primary_key => 'bts_code', :class_name => 'FlightSegment'
+  belongs_to :aircraft_class,    :foreign_key => 'class_code',    :primary_key => 'code'
+  belongs_to :fuel_use_equation, :foreign_key => 'fuel_use_code', :primary_key => 'code', :class_name => 'AircraftFuelUseEquation'
   
-  falls_back_on :m3            => lambda { weighted_average(:m3,            :weighted_by => [:segments, :passengers]) }, # 9.73423082858437e-08   r7110: 8.6540464368905e-8      r6972: 8.37e-8
-                :m2            => lambda { weighted_average(:m2,            :weighted_by => [:segments, :passengers]) }, # -0.000134350543484608  r7110: -0.00015337661447817    r6972: -4.09e-5
-                :m1            => lambda { weighted_average(:m1,            :weighted_by => [:segments, :passengers]) }, # 6.7728101555467        r7110: 4.7781966869412         r6972: 7.85
-                :endpoint_fuel => lambda { weighted_average(:endpoint_fuel, :weighted_by => [:segments, :passengers]) }  # 1527.81790006167       r7110: 1065.3476555284         r6972: 1.72e3
+  # set up a loose_tight_dictionary for matching Aircraft description with FlightSegment aircraft_description
+  class << self
+    def loose_tight_dictionary
+      @loose_tight_dictionary ||= LooseTightDictionary.new(Aircraft.all,
+          :haystack_reader => lambda { |record| record.description },
+          :blockings  => RemoteTable.new(:url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDlRR2NmdzE2ZjZwTy1ucjh4cWFYOFE&gid=0&output=csv').map { |record| record['blocking'] },
+          :identities => RemoteTable.new(:url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDlRR2NmdzE2ZjZwTy1ucjh4cWFYOFE&gid=1&output=csv').map { |record| record['identity'] },
+          :tighteners => RemoteTable.new(:url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDlRR2NmdzE2ZjZwTy1ucjh4cWFYOFE&gid=2&output=csv').map { |record| record['tightener'] },
+          :must_match_blocking => true,
+          :first_blocking_decides => true)
+    end
+  end
+  
+  # Enable aircraft.flight_segments
+  cache_loose_tight_dictionary_matches_with :flight_segments, :primary_key => :description, :foreign_key => :aircraft_description
   
   data_miner do
     tap "Brighter Planet's sanitized aircraft data", Earth.taps_server

data/lib/earth/air/aircraft/data_miner.rb

@@ -1,203 +1,108 @@
 Aircraft.class_eval do
-  # def self.bts_name_dictionary
-  #   @_bts_dictionary ||= LooseTightDictionary.new RemoteTable.new(:url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_AIRCRAFT_TYPE', :select => lambda { |record| record['Code'].to_i.between?(1, 998) }),
-  #                                                 :tightenings  => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false),
-  #                                                 :identities   => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false),
-  #                                                 :blockings    => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false),
-  #                                                 :blocking_only => true,
-  #                                                 :right_reader => lambda { |record| record['Description'] }
-  # end
-  # 
-  # # warning: self-referential, assumes it will be used once first import step is done
-  # def self.icao_name_dictionary
-  #   @_icao_dictionary ||= LooseTightDictionary.new Aircraft.all,
-  #                                                  :tightenings  => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false),
-  #                                                  :identities   => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false),
-  #                                                  :blockings    => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false),
-  #                                                  :right_reader => lambda { |record| record.manufacturer_name.to_s + ' ' + record.name.to_s }
-  # end
-  # 
-  # class Aircraft::BtsMatcher
-  #   attr_reader :wants
-  #   def initialize(wants)
-  #     @wants = wants
-  #   end
-  #   def match(raw_faa_icao_record)
-  #     @_match ||= Hash.new
-  #     return @_match[raw_faa_icao_record] if @_match.has_key?(raw_faa_icao_record)
-  #     faa_icao_record = [ raw_faa_icao_record['Manufacturer'] + ' ' + raw_faa_icao_record['Model'] ]
-  #     bts_record = Aircraft.bts_name_dictionary.left_to_right faa_icao_record
-  #     retval = case wants
-  #     when :bts_aircraft_type_code
-  #       bts_record['Code']
-  #     when :bts_name
-  #       bts_record['Description']
-  #     end if bts_record
-  #     @_match[raw_faa_icao_record] = retval
-  #   end
-  # end
-  # 
-  # class Aircraft::FuelUseMatcher
-  #   def match(raw_fuel_use_record)
-  #     @_match ||= Hash.new
-  #     return @_match[raw_fuel_use_record] if @_match.has_key?(raw_fuel_use_record)
-  #     
-  #     aircraft_record = if raw_fuel_use_record['ICAO'] =~ /\A[0-9A-Z]+\z/
-  #       Aircraft.find_by_icao_code raw_fuel_use_record['ICAO']
-  #     end
-  #     
-  #     aircraft_record ||= if raw_fuel_use_record['Aircraft Name'].present?
-  #       Aircraft.icao_name_dictionary.left_to_right [ raw_fuel_use_record['Aircraft Name'] ]
-  #     end
-  #     
-  #     if aircraft_record
-  #       @_match[raw_fuel_use_record] = aircraft_record.icao_code
-  #     else
-  #       raise "Didn't find a match for #{raw_fuel_use_record['Aircraft Name']} (#{raw_fuel_use_record['ICAO']}), which we found in the fuel use spreadsheet"
-  #     end
-  #   end
-  # end
-  
-  # for errata
+  # For errata
   class Aircraft::Guru
-    def is_a_dc_plane?(row)
-      row['Designator'] =~ /^DC\d/i
-    end
-    
-    def is_a_g159?(row)
-      row['Designator'] =~ /^G159$/
-    end
-    
-    def is_a_galx?(row)
-      row['Designator'] =~ /^GALX$/
-    end
-    
     def method_missing(method_id, *args, &block)
-      if method_id.to_s =~ /\Ais_n?o?t?_?attributed_to_([^\?]+)/
-        manufacturer_name = $1
-        manufacturer_regexp = Regexp.new(manufacturer_name.gsub('_', ' ?'), Regexp::IGNORECASE)
-        matches = manufacturer_regexp.match(args.first['Manufacturer']) # row['Manufacturer'] =~ /mcdonnell douglas/i
-        method_id.to_s.include?('not_attributed') ? matches.nil? : !matches.nil?
+      if method_id.to_s =~ /\A([a-z]+)_is_(?:not_)?([^\?]+)/
+        column_name = $1
+        value = $2
+        value_regexp = /^#{value.gsub('_',' ')}$/i
+        # row['Manufacturer'] =~ /mcdonnell douglas/i
+        matches = value_regexp.match(args.first[column_name.titleize])
+        method_id.to_s.include?('_not_') ? matches.nil? : !matches.nil?
       else
         super
       end
     end
   end
   
+  # We're only interested in aircraft from certain manufacturers
+  def self.manufacturer_whitelist?(candidate)
+    @manufacturer_whitelist ||= RemoteTable.new(:url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdFRFalpOdlg1cnF6amlSM1dDc1lya2c&output=csv').map { |record| record['Manufacturer'].to_regexp }
+    @manufacturer_whitelist.any? { |manufacturer_regexp| manufacturer_regexp.match candidate }
+  end
+  
   data_miner do
     schema Earth.database_options do
-      string 'bp_code'
-      string 'icao_code'
-      string 'bts_code'
-      string 'class_code'
-      string 'fuel_use_code'
-      string 'manufacturer_name'
-      string 'name'
-      string 'fuel_use_aircraft_name'
-      float  'm3'
-      string 'm3_units'
-      float  'm2'
-      string 'm2_units'
-      float  'm1'
-      string 'm1_units'
-      float  'endpoint_fuel'
-      string 'endpoint_fuel_units'
-      float  'seats'
-      float  'weighting'
-      index  'bts_code'
+      string  'icao_code'
+      string  'manufacturer_name'
+      string  'model_name'
+      string  'description'
+      string  'aircraft_type'
+      string  'engine_type'
+      integer 'engines'
+      string  'weight_class'
+      string  'class_code'
+      string  'fuel_use_code'
+      float   'seats'
+      float   'passengers'
     end
     
-    import "a curated list of aircraft",
-            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDJFblR4MDE1RGtnLVM1S2JHRGZpT3c&hl=en&gid=0&output=csv' do
-      key 'bp_code'
-      store 'icao_code',              :nullify => true
-      store 'bts_code',               :nullify => true
-      store 'class_code',             :nullify => true
-      store 'fuel_use_code',          :nullify => true
-      store 'manufacturer_name',      :nullify => true
-      store 'name',                   :nullify => true
-      store 'fuel_use_aircraft_name', :nullify => true
-      store 'm3',                     :nullify => true, :units => :kilograms_per_cubic_nautical_mile
-      store 'm2',                     :nullify => true, :units => :kilograms_per_square_nautical_mile
-      store 'm1',                     :nullify => true, :units => :kilograms_per_nautical_mile
-      store 'endpoint_fuel',          :nullify => true, :units => :kilograms
+    ('A'..'Z').each do |letter|
+      import("aircraft made by whitelisted manufacturers whose ICAO code starts with '#{letter}' from the FAA",
+             :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
+             :encoding => 'windows-1252',
+             :row_xpath => '//table/tr[2]/td/table/tr',
+             :column_xpath => 'td',
+             :errata => { :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGVBRnhkRGhSaVptSDJ5bXJGbkpUSWc&output=csv', :responder => Aircraft::Guru.new },
+             :select => lambda { |record| manufacturer_whitelist? record['Manufacturer'] }) do
+        key 'icao_code',           :field_name => 'Designator'
+        store 'manufacturer_name', :field_name => 'Manufacturer'
+        store 'model_name',        :field_name => 'Model'
+        store 'aircraft_type',     :field_name => 'Type/Wt Class', :chars => 0, :dictionary => {:input => 'code', :output => 'description', :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDJtU0stS0duR1NhQVVIU29valYwREE&output=csv'}
+        store 'engine_type',       :field_name => 'Type/Wt Class', :chars => 2, :dictionary => {:input => 'code', :output => 'description', :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdHgtS0xRTERpbjAxSzBBNFc4LUJEMXc&output=csv'}
+        store 'engines',           :field_name => 'Type/Wt Class', :chars => 1
+        store 'weight_class',      :field_name => 'Type/Wt Class', :split => { :pattern => %r{/}, :keep => 1 }, :dictionary => {:input => 'code', :output => 'description', :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGhrWElDZ25oV2NPREg0eUhjRVRYUHc&output=csv'}
+      end
     end
     
-    # ('A'..'Z').each do |letter|
-    #   import( "ICAO manufacturers and names for aircraft with an ICAO code starting with the letter #{letter} from the FAA",
-    #           :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
-    #           :errata => { :url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
-    #                        :responder => Aircraft::Guru.new },
-    #           :encoding => 'windows-1252',
-    #           :row_xpath => '//table/tr[2]/td/table/tr',
-    #           :column_xpath => 'td' ) do
-    #     key 'icao_code',       :field_name => 'Designator'
-    #     store 'icao_manufacturer_name', :field_name => 'Manufacturer'
-    #     store 'icao_name',     :field_name => 'Model'
-    #   end
-    # end
+    import "aircraft not included in the FAA database",
+           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHRNaVpSUWw2Z2VhN3RUV25yYWdQX2c&output=csv' do
+      key 'icao_code'
+      store 'manufacturer_name'
+      store 'model_name'
+      store 'aircraft_type'
+      store 'engine_type'
+      store 'engines'
+      store 'weight_class'
+    end
     
-    # import "some hand-picked ICAO manufacturers and names, including some for ICAO codes not used by the FAA",
-    #        :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHRNaVpSUWw2Z2VhN3RUV25yYWdQX2c&hl=en&single=true&gid=0&output=csv' do
-    #   key 'icao_code',       :field_name => 'icao_code'
-    #   store 'icao_manufacturer_name', :field_name => 'manufacturer_name'
-    #   store 'icao_name',     :field_name => 'name'
-    # end
+    import "a curated list of aircraft fuel use codes",
+           :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGxqRjFJdDlQWVVLYS11NnJVcDZsYWc&output=csv' do
+      key 'icao_code'
+      store 'fuel_use_code'
+    end
     
-    # import "aircraft BTS names",
-    #        :url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_AIRCRAFT_TYPE',
-    #        :errata => { :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEZ2d3JQMzV5T1o1T3JmVlFyNUZxdEE&hl=en&single=true&gid=0&output=csv'} do
-    #   key 'bts_code',   :field_name => 'Code'
-    #   store 'bts_name', :field_name => 'Description'
-    # end
+    process "Synthesize description from manufacturer name and model name" do
+      Aircraft.find_each do |aircraft|
+        aircraft.update_attribute :description, [aircraft.manufacturer_name, aircraft.model_name].join(' ').downcase
+      end
+    end
     
-    # import "the fuel use equations associated with each fuel use code",
-    #        :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdG9tSC1RczJOdjliWTdjT2ZpdV9RTnc&hl=en&single=true&gid=0&output=csv' do
-    #   key   'fuel_use_code', :field_name => 'fuel_use_code'
-    #   store 'fuel_use_aircraft_name', :field_name => 'aircraft_name'
-    #   store 'm3', :units => :kilograms_per_cubic_nautical_mile
-    #   store 'm2', :units => :kilograms_per_square_nautical_mile
-    #   store 'm1', :units => :kilograms_per_nautical_mile
-    #   store 'endpoint_fuel', :field_name => 'b', :units => :kilograms
-    # end
+    process "Synthesize class code from engine type and weight class" do
+      Aircraft.find_each do |aircraft|
+        size = case aircraft.weight_class
+        when 'Small', 'Small+', 'Light'
+          'Light'
+        when 'Large', 'Medium'
+          'Medium'
+        else
+          'Heavy'
+        end
+        aircraft.update_attribute :class_code, [size, aircraft.engines.to_s, 'engine', aircraft.engine_type].join(' ')
+      end
+    end
     
-    process "Derive some average flight characteristics from flight segments" do
+    process "Ensure FlightSegment is populated" do
       FlightSegment.run_data_miner!
-      
-      aircraft = Aircraft.arel_table
-      segments = FlightSegment.arel_table
-      
-      # non-working joins method
-      # update_all "aircraft.distance_1            = (SELECT * FROM (#{FlightSegment.joins(:aircraft).weighted_average_relation(:distance,            :weighted_by => :passengers                                           ).to_sql}) AS anonymous_1)"
-      # update_all "aircraft.load_factor_1         = (SELECT * FROM (#{FlightSegment.joins(:aircraft).weighted_average_relation(:load_factor,         :weighted_by => :passengers                                           ).to_sql}) AS anonymous_1)"
-      # execute %{
-      #   update aircraft as t1
-      #   set t1.distance_1 = (SELECT * FROM (#{FlightSegment.joins(:aircraft).weighted_average_relation(:distance,            :weighted_by => :passengers                                           ).where('t1.bts_aircraft_type_code = flight_segments.bts_aircraft_type_code').to_sql}) AS anonymous_1)
-      # }
-      
-      # FIXME TODO
-      # This should calculate weighted averages from flight segments
-      # For example, to calculate seats:
-      # SELECT aircraft.icao_code, sum(flight_segments.seats * flight_segments.passengers) / sum(flight_segments.passengers)
-      # FROM flight_segments
-      # INNER JOIN aircraft_aircraft_types ON flight_segments.aircraft_type_code = aircraft_aircraft_types.aircraft_type_code
-      # INNER JOIN aircraft ON aircraft_aircraft_types.icao_code = aircraft.icao_code
-      # GROUP BY aircraft.icao_code
-      
-      conditional_relation = segments[:aircraft_bts_code].eq(aircraft[:bts_code])
-      update_all "seats = (#{FlightSegment.weighted_average_relation(:seats, :weighted_by => :passengers).where(conditional_relation).to_sql})"
-      update_all "weighting = (#{segments.project(segments[:passengers].sum).where(conditional_relation).to_sql})"
-      
-      # conditional_relation = aircraft[:aircraft_type_code].eq(segments[:aircraft_type_code])
-      
-      # update_all "seats         = (#{FlightSegment.weighted_average_relation(:seats,         :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
-      
-      # update_all "distance      = (#{FlightSegment.weighted_average_relation(:distance,      :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
-      # update_all "load_factor   = (#{FlightSegment.weighted_average_relation(:load_factor,   :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
-      # update_all "freight_share = (#{FlightSegment.weighted_average_relation(:freight_share, :weighted_by => :passengers                                           ).where(conditional_relation).to_sql})"
-      # update_all "payload       = (#{FlightSegment.weighted_average_relation(:payload,       :weighted_by => :passengers, :disaggregate_by => :departures_performed).where(conditional_relation).to_sql})"
-      
-      # update_all "weighting = (#{segments.project(segments[:passengers].sum).where(aircraft[:aircraft_type_code].eq(segments[:aircraft_type_code])).to_sql})"
+    end
+    
+    # FIXME TODO do we want to restrict this to certain years?
+    process "Derive some average characteristics from flight segments" do
+      Aircraft.find_each do |aircraft|
+        aircraft.seats = aircraft.flight_segments.weighted_average :seats_per_flight, :weighted_by => :passengers
+        aircraft.passengers = aircraft.flight_segments.sum :passengers
+        aircraft.save
+      end
     end
     
     # FIXME TODO verify this