earth 0.4.12 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. data/.gitignore +1 -0
  2. data/Gemfile +1 -1
  3. data/earth.gemspec +7 -3
  4. data/features/automobile_make_model_year_variant.feature +10 -10
  5. data/features/step_definitions/data_steps.rb +1 -1
  6. data/features/support/env.rb +2 -2
  7. data/features/support/imports/automobile_fuel_good.csv +3 -3
  8. data/features/support/imports/automobile_make_model_year_variant_bad.csv +2 -2
  9. data/features/support/imports/automobile_make_model_year_variant_good.csv +2 -2
  10. data/lib/earth/air.rb +2 -1
  11. data/lib/earth/air/aircraft.rb +21 -8
  12. data/lib/earth/air/aircraft/data_miner.rb +85 -180
  13. data/lib/earth/air/aircraft_class.rb +9 -1
  14. data/lib/earth/air/aircraft_class/data_miner.rb +57 -20
  15. data/lib/earth/air/aircraft_fuel_use_equation.rb +26 -0
  16. data/lib/earth/air/aircraft_fuel_use_equation/data_miner.rb +26 -0
  17. data/lib/earth/air/aircraft_manufacturer.rb +0 -2
  18. data/lib/earth/air/aircraft_manufacturer/data_miner.rb +4 -1
  19. data/lib/earth/air/airline.rb +2 -11
  20. data/lib/earth/air/airline/data_miner.rb +10 -51
  21. data/lib/earth/air/airport.rb +2 -29
  22. data/lib/earth/air/airport/data_miner.rb +61 -56
  23. data/lib/earth/air/bts_aircraft.rb +7 -0
  24. data/lib/earth/air/bts_aircraft/data_miner.rb +15 -0
  25. data/lib/earth/air/data_miner.rb +2 -1
  26. data/lib/earth/air/flight_seat_class.rb +1 -4
  27. data/lib/earth/air/flight_segment.rb +16 -10
  28. data/lib/earth/air/flight_segment/data_miner.rb +170 -129
  29. data/lib/earth/automobile/automobile_make/data_miner.rb +12 -3
  30. data/lib/earth/automobile/automobile_make_model/data_miner.rb +4 -1
  31. data/lib/earth/automobile/automobile_make_model_year/data_miner.rb +4 -1
  32. data/lib/earth/automobile/automobile_make_model_year_variant/data_miner.rb +29 -16
  33. data/lib/earth/automobile/automobile_make_year/data_miner.rb +8 -2
  34. data/lib/earth/automobile/automobile_size_class/data_miner.rb +4 -1
  35. data/lib/earth/automobile/automobile_type_fuel_age/data_miner.rb +4 -1
  36. data/lib/earth/automobile/automobile_type_fuel_year/data_miner.rb +4 -2
  37. data/lib/earth/automobile/automobile_type_fuel_year_age/data_miner.rb +4 -1
  38. data/lib/earth/automobile/automobile_type_year/data_miner.rb +4 -1
  39. data/lib/earth/fuel/fuel/data_miner.rb +4 -1
  40. data/lib/earth/hospitality/lodging_class.rb +0 -16
  41. data/lib/earth/hospitality/lodging_class/data_miner.rb +11 -1
  42. data/lib/earth/inflectors.rb +1 -1
  43. data/lib/earth/locality/country/data_miner.rb +14 -14
  44. data/lib/earth/locality/urbanity/data_miner.rb +4 -1
  45. data/lib/earth/residence/air_conditioner_use/data_miner.rb +4 -1
  46. data/lib/earth/residence/clothes_machine_use/data_miner.rb +4 -1
  47. data/lib/earth/residence/dishwasher_use/data_miner.rb +4 -1
  48. data/lib/earth/residence/residence_appliance/data_miner.rb +5 -2
  49. data/lib/earth/residence/residence_class/data_miner.rb +4 -1
  50. data/lib/earth/version.rb +1 -1
  51. data/spec/earth_spec.rb +2 -2
  52. data/test/test_aircraft_match.rb +732 -0
  53. metadata +87 -51
  54. data/features/flight_segment.feature +0 -14
  55. data/features/support/imports/flight_segment_bad.csv +0 -8
  56. data/features/support/imports/flight_segment_good.csv +0 -8
  57. data/lib/earth/air/flight_fuel_type.rb +0 -12
  58. data/lib/earth/air/flight_fuel_type/data_miner.rb +0 -12
data/.gitignore CHANGED
@@ -18,6 +18,7 @@ coverage
18
18
  rdoc
19
19
  pkg
20
20
  doc
21
+ log
21
22
 
22
23
  ## PROJECT::SPECIFIC
23
24
  Gemfile.lock
data/Gemfile CHANGED
@@ -6,6 +6,6 @@ gemspec
6
6
  gem 'data_miner', :path => ENV['LOCAL_DATA_MINER'] if ENV['LOCAL_DATA_MINER']
7
7
  gem 'sniff', :path => ENV['LOCAL_SNIFF'] if ENV['LOCAL_SNIFF']
8
8
 
9
- if RUBY_VERSION =~ /^1\.8/
9
+ if RUBY_VERSION < "1.9"
10
10
  gem 'fastercsv'
11
11
  end
@@ -5,7 +5,7 @@ require "earth/version"
5
5
  Gem::Specification.new do |s|
6
6
  s.name = "earth"
7
7
  s.version = Earth::VERSION
8
- s.date = "2011-05-21"
8
+ s.date = "2011-06-06"
9
9
  s.platform = Gem::Platform::RUBY
10
10
  s.authors = ["Seamus Abshere", "Derek Kastner", "Andy Rossmeissl"]
11
11
  s.email = %q{andy@rossmeissl.net}
@@ -26,12 +26,15 @@ Gem::Specification.new do |s|
26
26
  s.rdoc_options = ["--charset=UTF-8"]
27
27
  s.require_paths = ["lib"]
28
28
  s.rubygems_version = %q{1.3.7}
29
-
30
- s.add_runtime_dependency 'cohort_scope', '>=0.2.0'
29
+
31
30
  s.add_runtime_dependency 'data_miner'
31
+ s.add_runtime_dependency 'to_regexp'
32
+ s.add_runtime_dependency 'cohort_scope'
32
33
  s.add_runtime_dependency 'remote_table', '>=1.2.3'
33
34
  s.add_runtime_dependency 'falls_back_on'
35
+ s.add_runtime_dependency 'fixed_width-multibyte'
34
36
  s.add_runtime_dependency 'geokit-rails'
37
+ s.add_runtime_dependency 'loose_tight_dictionary', '>=0.2.3'
35
38
  s.add_runtime_dependency 'weighted_average'
36
39
  s.add_development_dependency 'bundler'
37
40
  s.add_development_dependency 'bueller'
@@ -40,5 +43,6 @@ Gem::Specification.new do |s|
40
43
  s.add_development_dependency 'rdoc'
41
44
  s.add_development_dependency 'rdoc'
42
45
  s.add_development_dependency 'rspec'
46
+ s.add_development_dependency 'ruby-debug19'
43
47
  s.add_development_dependency 'sqlite3-ruby'
44
48
  end
@@ -8,11 +8,11 @@ Feature: Data import for AutomobileMakeModelYearVariant
8
8
  When a data import verifies "Year should be from 1985 to 2011"
9
9
  Then the verification should be successful
10
10
 
11
- # FIXME TODO
12
- # Scenario: Successfully verifying that fuel type code is found in AutomobileFuel
13
- # Given a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_good"
14
- # When a data import verifies "Fuel type code should be found in AutomobileFuel"
15
- # Then the verification should be successful
11
+ Scenario: Successfully verifying that fuel type code is found in AutomobileFuel
12
+ Given a "AutomobileFuel" data import fetches results listed in "automobile_fuel_good"
13
+ And a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_good"
14
+ When a data import verifies "Fuel code should appear in AutomobileFuel"
15
+ Then the verification should be successful
16
16
 
17
17
  Scenario: Successfully verifying that fuel efficiencies are greater than zero
18
18
  Given a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_good"
@@ -29,11 +29,11 @@ Feature: Data import for AutomobileMakeModelYearVariant
29
29
  When a data import verifies "Year should be from 1985 to 2011"
30
30
  Then the verification should not be successful
31
31
 
32
- # FIXME TODO
33
- # Scenario: Failing to verify that fuel type code is found in AutomobileFuel
34
- # Given a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_bad"
35
- # When a data import verifies "Fuel type code should be found in AutomobileFuel"
36
- # Then the verification should not be successful
32
+ Scenario: Failing to verifying that fuel type code is found in AutomobileFuel
33
+ Given a "AutomobileFuel" data import fetches results listed in "automobile_fuel_good"
34
+ And a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_bad"
35
+ When a data import verifies "Fuel code should appear in AutomobileFuel"
36
+ Then the verification should not be successful
37
37
 
38
38
  Scenario: Failing to verify that fuel efficiencies are greater than zero
39
39
  Given a "AutomobileMakeModelYearVariant" data import fetches results listed in "automobile_make_model_year_variant_bad"
@@ -1,7 +1,7 @@
1
1
  require 'active_record'
2
2
  require 'active_record/fixtures'
3
3
 
4
- Given /^a "([^\"]*)" data import fetches results listed in "(.*)"$/ do |model, file_name|
4
+ Given /^an? "([^\"]*)" data import fetches results listed in "(.*)"$/ do |model, file_name|
5
5
  @data_import_model = model.constantize
6
6
  table_name = @data_import_model.table_name
7
7
 
@@ -6,10 +6,10 @@ Bundler.setup
6
6
  require 'earth'
7
7
 
8
8
  require 'data_miner'
9
- DataMiner.logger = Logger.new nil
9
+ DataMiner.logger = Logger.new 'log/test.log'
10
10
 
11
11
  require 'active_record'
12
12
  require 'sqlite3'
13
- ActiveRecord::Base.logger = Logger.new nil
13
+ ActiveRecord::Base.logger = Logger.new 'log/test.log'
14
14
  ActiveRecord::Base.establish_connection :adapter => 'sqlite3', :database => ':memory:'
15
15
  Earth.init :all, :apply_schemas => true
@@ -1,3 +1,3 @@
1
- name,co2_emission_factor,co2_emission_factor_units,co2_biogenic_emission_factor,co2_biogenic_emission_factor_units,ch4_emission_factor,ch4_emission_factor_units,n2o_emission_factor,n2o_emission_factor_units,hfc_emission_factor,hfc_emission_factor_units,base_fuel_name,distance_key,ef_key,blend_portion
2
- regular gasoline,1,kilograms_per_litre,0,kilograms_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,Motor Gasoline,gasoline,gasoline
3
- b20,0,kilograms_per_litre,1,kilograms_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,Distillate Fuel Oil No. 2,diesel,diesel,0.2
1
+ name,code,co2_emission_factor,co2_emission_factor_units,co2_biogenic_emission_factor,co2_biogenic_emission_factor_units,ch4_emission_factor,ch4_emission_factor_units,n2o_emission_factor,n2o_emission_factor_units,hfc_emission_factor,hfc_emission_factor_units,base_fuel_name,distance_key,ef_key,blend_portion
2
+ regular gasoline,R,1,kilograms_per_litre,0,kilograms_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,Motor Gasoline,gasoline,gasoline
3
+ b20,BP-B20,0,kilograms_per_litre,1,kilograms_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,0.1,kilograms_co2e_per_litre,Distillate Fuel Oil No. 2,diesel,diesel,0.2
@@ -1,3 +1,3 @@
1
1
  row_hash,year,fuel_code,fuel_efficiency,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_units,fuel_efficiency_city_units,fuel_efficiency_highway_units
2
- a1,2000,D,12.0,10.0,15.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
3
- b2,2012,P,0,-10,0,foo,bar
2
+ a1,2000,R,12.0,10.0,15.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
3
+ b2,2012,X,0,-10,0,foo,bar
@@ -1,3 +1,3 @@
1
1
  row_hash,year,fuel_code,fuel_efficiency,fuel_efficiency_city,fuel_efficiency_highway,fuel_efficiency_units,fuel_efficiency_city_units,fuel_efficiency_highway_units
2
- a1,2000,D,12.0,10.0,15.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
3
- b2,2011,X,20.5,20.0,21.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
2
+ a1,2000,R,12.0,10.0,15.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
3
+ b2,2011,BP-B20,20.5,20.0,21.0,kilometres_per_litre,kilometres_per_litre,kilometres_per_litre
@@ -1,9 +1,10 @@
1
1
  require 'earth/air/aircraft'
2
2
  require 'earth/air/aircraft_class'
3
+ require 'earth/air/aircraft_fuel_use_equation'
3
4
  require 'earth/air/aircraft_manufacturer'
4
5
  require 'earth/air/airline'
5
6
  require 'earth/air/airport'
7
+ require 'earth/air/bts_aircraft'
6
8
  require 'earth/air/flight_distance_class'
7
- require 'earth/air/flight_fuel_type'
8
9
  require 'earth/air/flight_seat_class'
9
10
  require 'earth/air/flight_segment'
@@ -1,14 +1,27 @@
1
+ # need this for association with FlightSegment through loose_tight_dictionary_cached_results
2
+ require 'loose_tight_dictionary/cached_result'
3
+
1
4
  class Aircraft < ActiveRecord::Base
2
- set_primary_key :bp_code
5
+ set_primary_key :icao_code
3
6
 
4
- belongs_to :manufacturer, :foreign_key => 'manufacturer_name', :primary_key => 'name', :class_name => 'AircraftManufacturer'
5
- belongs_to :aircraft_class, :foreign_key => 'class_code', :primary_key => 'code'
6
- has_many :segments, :foreign_key => 'aircraft_bts_code', :primary_key => 'bts_code', :class_name => 'FlightSegment'
7
+ belongs_to :aircraft_class, :foreign_key => 'class_code', :primary_key => 'code'
8
+ belongs_to :fuel_use_equation, :foreign_key => 'fuel_use_code', :primary_key => 'code', :class_name => 'AircraftFuelUseEquation'
7
9
 
8
- falls_back_on :m3 => lambda { weighted_average(:m3, :weighted_by => [:segments, :passengers]) }, # 9.73423082858437e-08 r7110: 8.6540464368905e-8 r6972: 8.37e-8
9
- :m2 => lambda { weighted_average(:m2, :weighted_by => [:segments, :passengers]) }, # -0.000134350543484608 r7110: -0.00015337661447817 r6972: -4.09e-5
10
- :m1 => lambda { weighted_average(:m1, :weighted_by => [:segments, :passengers]) }, # 6.7728101555467 r7110: 4.7781966869412 r6972: 7.85
11
- :endpoint_fuel => lambda { weighted_average(:endpoint_fuel, :weighted_by => [:segments, :passengers]) } # 1527.81790006167 r7110: 1065.3476555284 r6972: 1.72e3
10
+ # set up a loose_tight_dictionary for matching Aircraft description with FlightSegment aircraft_description
11
+ class << self
12
+ def loose_tight_dictionary
13
+ @loose_tight_dictionary ||= LooseTightDictionary.new(Aircraft.all,
14
+ :haystack_reader => lambda { |record| record.description },
15
+ :blockings => RemoteTable.new(:url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDlRR2NmdzE2ZjZwTy1ucjh4cWFYOFE&gid=0&output=csv').map { |record| record['blocking'] },
16
+ :identities => RemoteTable.new(:url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDlRR2NmdzE2ZjZwTy1ucjh4cWFYOFE&gid=1&output=csv').map { |record| record['identity'] },
17
+ :tighteners => RemoteTable.new(:url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDlRR2NmdzE2ZjZwTy1ucjh4cWFYOFE&gid=2&output=csv').map { |record| record['tightener'] },
18
+ :must_match_blocking => true,
19
+ :first_blocking_decides => true)
20
+ end
21
+ end
22
+
23
+ # Enable aircraft.flight_segments
24
+ cache_loose_tight_dictionary_matches_with :flight_segments, :primary_key => :description, :foreign_key => :aircraft_description
12
25
 
13
26
  data_miner do
14
27
  tap "Brighter Planet's sanitized aircraft data", Earth.taps_server
@@ -1,203 +1,108 @@
1
1
  Aircraft.class_eval do
2
- # def self.bts_name_dictionary
3
- # @_bts_dictionary ||= LooseTightDictionary.new RemoteTable.new(:url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_AIRCRAFT_TYPE', :select => lambda { |record| record['Code'].to_i.between?(1, 998) }),
4
- # :tightenings => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false),
5
- # :identities => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false),
6
- # :blockings => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false),
7
- # :blocking_only => true,
8
- # :right_reader => lambda { |record| record['Description'] }
9
- # end
10
- #
11
- # # warning: self-referential, assumes it will be used once first import step is done
12
- # def self.icao_name_dictionary
13
- # @_icao_dictionary ||= LooseTightDictionary.new Aircraft.all,
14
- # :tightenings => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false),
15
- # :identities => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false),
16
- # :blockings => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false),
17
- # :right_reader => lambda { |record| record.manufacturer_name.to_s + ' ' + record.name.to_s }
18
- # end
19
- #
20
- # class Aircraft::BtsMatcher
21
- # attr_reader :wants
22
- # def initialize(wants)
23
- # @wants = wants
24
- # end
25
- # def match(raw_faa_icao_record)
26
- # @_match ||= Hash.new
27
- # return @_match[raw_faa_icao_record] if @_match.has_key?(raw_faa_icao_record)
28
- # faa_icao_record = [ raw_faa_icao_record['Manufacturer'] + ' ' + raw_faa_icao_record['Model'] ]
29
- # bts_record = Aircraft.bts_name_dictionary.left_to_right faa_icao_record
30
- # retval = case wants
31
- # when :bts_aircraft_type_code
32
- # bts_record['Code']
33
- # when :bts_name
34
- # bts_record['Description']
35
- # end if bts_record
36
- # @_match[raw_faa_icao_record] = retval
37
- # end
38
- # end
39
- #
40
- # class Aircraft::FuelUseMatcher
41
- # def match(raw_fuel_use_record)
42
- # @_match ||= Hash.new
43
- # return @_match[raw_fuel_use_record] if @_match.has_key?(raw_fuel_use_record)
44
- #
45
- # aircraft_record = if raw_fuel_use_record['ICAO'] =~ /\A[0-9A-Z]+\z/
46
- # Aircraft.find_by_icao_code raw_fuel_use_record['ICAO']
47
- # end
48
- #
49
- # aircraft_record ||= if raw_fuel_use_record['Aircraft Name'].present?
50
- # Aircraft.icao_name_dictionary.left_to_right [ raw_fuel_use_record['Aircraft Name'] ]
51
- # end
52
- #
53
- # if aircraft_record
54
- # @_match[raw_fuel_use_record] = aircraft_record.icao_code
55
- # else
56
- # raise "Didn't find a match for #{raw_fuel_use_record['Aircraft Name']} (#{raw_fuel_use_record['ICAO']}), which we found in the fuel use spreadsheet"
57
- # end
58
- # end
59
- # end
60
-
61
- # for errata
2
+ # For errata
62
3
  class Aircraft::Guru
63
- def is_a_dc_plane?(row)
64
- row['Designator'] =~ /^DC\d/i
65
- end
66
-
67
- def is_a_g159?(row)
68
- row['Designator'] =~ /^G159$/
69
- end
70
-
71
- def is_a_galx?(row)
72
- row['Designator'] =~ /^GALX$/
73
- end
74
-
75
4
  def method_missing(method_id, *args, &block)
76
- if method_id.to_s =~ /\Ais_n?o?t?_?attributed_to_([^\?]+)/
77
- manufacturer_name = $1
78
- manufacturer_regexp = Regexp.new(manufacturer_name.gsub('_', ' ?'), Regexp::IGNORECASE)
79
- matches = manufacturer_regexp.match(args.first['Manufacturer']) # row['Manufacturer'] =~ /mcdonnell douglas/i
80
- method_id.to_s.include?('not_attributed') ? matches.nil? : !matches.nil?
5
+ if method_id.to_s =~ /\A([a-z]+)_is_(?:not_)?([^\?]+)/
6
+ column_name = $1
7
+ value = $2
8
+ value_regexp = /^#{value.gsub('_',' ')}$/i
9
+ # row['Manufacturer'] =~ /mcdonnell douglas/i
10
+ matches = value_regexp.match(args.first[column_name.titleize])
11
+ method_id.to_s.include?('_not_') ? matches.nil? : !matches.nil?
81
12
  else
82
13
  super
83
14
  end
84
15
  end
85
16
  end
86
17
 
18
+ # We're only interested in aircraft from certain manufacturers
19
+ def self.manufacturer_whitelist?(candidate)
20
+ @manufacturer_whitelist ||= RemoteTable.new(:url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdFRFalpOdlg1cnF6amlSM1dDc1lya2c&output=csv').map { |record| record['Manufacturer'].to_regexp }
21
+ @manufacturer_whitelist.any? { |manufacturer_regexp| manufacturer_regexp.match candidate }
22
+ end
23
+
87
24
  data_miner do
88
25
  schema Earth.database_options do
89
- string 'bp_code'
90
- string 'icao_code'
91
- string 'bts_code'
92
- string 'class_code'
93
- string 'fuel_use_code'
94
- string 'manufacturer_name'
95
- string 'name'
96
- string 'fuel_use_aircraft_name'
97
- float 'm3'
98
- string 'm3_units'
99
- float 'm2'
100
- string 'm2_units'
101
- float 'm1'
102
- string 'm1_units'
103
- float 'endpoint_fuel'
104
- string 'endpoint_fuel_units'
105
- float 'seats'
106
- float 'weighting'
107
- index 'bts_code'
26
+ string 'icao_code'
27
+ string 'manufacturer_name'
28
+ string 'model_name'
29
+ string 'description'
30
+ string 'aircraft_type'
31
+ string 'engine_type'
32
+ integer 'engines'
33
+ string 'weight_class'
34
+ string 'class_code'
35
+ string 'fuel_use_code'
36
+ float 'seats'
37
+ float 'passengers'
108
38
  end
109
39
 
110
- import "a curated list of aircraft",
111
- :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDJFblR4MDE1RGtnLVM1S2JHRGZpT3c&hl=en&gid=0&output=csv' do
112
- key 'bp_code'
113
- store 'icao_code', :nullify => true
114
- store 'bts_code', :nullify => true
115
- store 'class_code', :nullify => true
116
- store 'fuel_use_code', :nullify => true
117
- store 'manufacturer_name', :nullify => true
118
- store 'name', :nullify => true
119
- store 'fuel_use_aircraft_name', :nullify => true
120
- store 'm3', :nullify => true, :units => :kilograms_per_cubic_nautical_mile
121
- store 'm2', :nullify => true, :units => :kilograms_per_square_nautical_mile
122
- store 'm1', :nullify => true, :units => :kilograms_per_nautical_mile
123
- store 'endpoint_fuel', :nullify => true, :units => :kilograms
40
+ ('A'..'Z').each do |letter|
41
+ import("aircraft made by whitelisted manufacturers whose ICAO code starts with '#{letter}' from the FAA",
42
+ :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
43
+ :encoding => 'windows-1252',
44
+ :row_xpath => '//table/tr[2]/td/table/tr',
45
+ :column_xpath => 'td',
46
+ :errata => { :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGVBRnhkRGhSaVptSDJ5bXJGbkpUSWc&output=csv', :responder => Aircraft::Guru.new },
47
+ :select => lambda { |record| manufacturer_whitelist? record['Manufacturer'] }) do
48
+ key 'icao_code', :field_name => 'Designator'
49
+ store 'manufacturer_name', :field_name => 'Manufacturer'
50
+ store 'model_name', :field_name => 'Model'
51
+ store 'aircraft_type', :field_name => 'Type/Wt Class', :chars => 0, :dictionary => {:input => 'code', :output => 'description', :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDJtU0stS0duR1NhQVVIU29valYwREE&output=csv'}
52
+ store 'engine_type', :field_name => 'Type/Wt Class', :chars => 2, :dictionary => {:input => 'code', :output => 'description', :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdHgtS0xRTERpbjAxSzBBNFc4LUJEMXc&output=csv'}
53
+ store 'engines', :field_name => 'Type/Wt Class', :chars => 1
54
+ store 'weight_class', :field_name => 'Type/Wt Class', :split => { :pattern => %r{/}, :keep => 1 }, :dictionary => {:input => 'code', :output => 'description', :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGhrWElDZ25oV2NPREg0eUhjRVRYUHc&output=csv'}
55
+ end
124
56
  end
125
57
 
126
- # ('A'..'Z').each do |letter|
127
- # import( "ICAO manufacturers and names for aircraft with an ICAO code starting with the letter #{letter} from the FAA",
128
- # :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
129
- # :errata => { :url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
130
- # :responder => Aircraft::Guru.new },
131
- # :encoding => 'windows-1252',
132
- # :row_xpath => '//table/tr[2]/td/table/tr',
133
- # :column_xpath => 'td' ) do
134
- # key 'icao_code', :field_name => 'Designator'
135
- # store 'icao_manufacturer_name', :field_name => 'Manufacturer'
136
- # store 'icao_name', :field_name => 'Model'
137
- # end
138
- # end
58
+ import "aircraft not included in the FAA database",
59
+ :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHRNaVpSUWw2Z2VhN3RUV25yYWdQX2c&output=csv' do
60
+ key 'icao_code'
61
+ store 'manufacturer_name'
62
+ store 'model_name'
63
+ store 'aircraft_type'
64
+ store 'engine_type'
65
+ store 'engines'
66
+ store 'weight_class'
67
+ end
139
68
 
140
- # import "some hand-picked ICAO manufacturers and names, including some for ICAO codes not used by the FAA",
141
- # :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHRNaVpSUWw2Z2VhN3RUV25yYWdQX2c&hl=en&single=true&gid=0&output=csv' do
142
- # key 'icao_code', :field_name => 'icao_code'
143
- # store 'icao_manufacturer_name', :field_name => 'manufacturer_name'
144
- # store 'icao_name', :field_name => 'name'
145
- # end
69
+ import "a curated list of aircraft fuel use codes",
70
+ :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGxqRjFJdDlQWVVLYS11NnJVcDZsYWc&output=csv' do
71
+ key 'icao_code'
72
+ store 'fuel_use_code'
73
+ end
146
74
 
147
- # import "aircraft BTS names",
148
- # :url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_AIRCRAFT_TYPE',
149
- # :errata => { :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdEZ2d3JQMzV5T1o1T3JmVlFyNUZxdEE&hl=en&single=true&gid=0&output=csv'} do
150
- # key 'bts_code', :field_name => 'Code'
151
- # store 'bts_name', :field_name => 'Description'
152
- # end
75
+ process "Synthesize description from manufacturer name and model name" do
76
+ Aircraft.find_each do |aircraft|
77
+ aircraft.update_attribute :description, [aircraft.manufacturer_name, aircraft.model_name].join(' ').downcase
78
+ end
79
+ end
153
80
 
154
- # import "the fuel use equations associated with each fuel use code",
155
- # :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdG9tSC1RczJOdjliWTdjT2ZpdV9RTnc&hl=en&single=true&gid=0&output=csv' do
156
- # key 'fuel_use_code', :field_name => 'fuel_use_code'
157
- # store 'fuel_use_aircraft_name', :field_name => 'aircraft_name'
158
- # store 'm3', :units => :kilograms_per_cubic_nautical_mile
159
- # store 'm2', :units => :kilograms_per_square_nautical_mile
160
- # store 'm1', :units => :kilograms_per_nautical_mile
161
- # store 'endpoint_fuel', :field_name => 'b', :units => :kilograms
162
- # end
81
+ process "Synthesize class code from engine type and weight class" do
82
+ Aircraft.find_each do |aircraft|
83
+ size = case aircraft.weight_class
84
+ when 'Small', 'Small+', 'Light'
85
+ 'Light'
86
+ when 'Large', 'Medium'
87
+ 'Medium'
88
+ else
89
+ 'Heavy'
90
+ end
91
+ aircraft.update_attribute :class_code, [size, aircraft.engines.to_s, 'engine', aircraft.engine_type].join(' ')
92
+ end
93
+ end
163
94
 
164
- process "Derive some average flight characteristics from flight segments" do
95
+ process "Ensure FlightSegment is populated" do
165
96
  FlightSegment.run_data_miner!
166
-
167
- aircraft = Aircraft.arel_table
168
- segments = FlightSegment.arel_table
169
-
170
- # non-working joins method
171
- # update_all "aircraft.distance_1 = (SELECT * FROM (#{FlightSegment.joins(:aircraft).weighted_average_relation(:distance, :weighted_by => :passengers ).to_sql}) AS anonymous_1)"
172
- # update_all "aircraft.load_factor_1 = (SELECT * FROM (#{FlightSegment.joins(:aircraft).weighted_average_relation(:load_factor, :weighted_by => :passengers ).to_sql}) AS anonymous_1)"
173
- # execute %{
174
- # update aircraft as t1
175
- # set t1.distance_1 = (SELECT * FROM (#{FlightSegment.joins(:aircraft).weighted_average_relation(:distance, :weighted_by => :passengers ).where('t1.bts_aircraft_type_code = flight_segments.bts_aircraft_type_code').to_sql}) AS anonymous_1)
176
- # }
177
-
178
- # FIXME TODO
179
- # This should calculate weighted averages from flight segments
180
- # For example, to calculate seats:
181
- # SELECT aircraft.icao_code, sum(flight_segments.seats * flight_segments.passengers) / sum(flight_segments.passengers)
182
- # FROM flight_segments
183
- # INNER JOIN aircraft_aircraft_types ON flight_segments.aircraft_type_code = aircraft_aircraft_types.aircraft_type_code
184
- # INNER JOIN aircraft ON aircraft_aircraft_types.icao_code = aircraft.icao_code
185
- # GROUP BY aircraft.icao_code
186
-
187
- conditional_relation = segments[:aircraft_bts_code].eq(aircraft[:bts_code])
188
- update_all "seats = (#{FlightSegment.weighted_average_relation(:seats, :weighted_by => :passengers).where(conditional_relation).to_sql})"
189
- update_all "weighting = (#{segments.project(segments[:passengers].sum).where(conditional_relation).to_sql})"
190
-
191
- # conditional_relation = aircraft[:aircraft_type_code].eq(segments[:aircraft_type_code])
192
-
193
- # update_all "seats = (#{FlightSegment.weighted_average_relation(:seats, :weighted_by => :passengers ).where(conditional_relation).to_sql})"
194
-
195
- # update_all "distance = (#{FlightSegment.weighted_average_relation(:distance, :weighted_by => :passengers ).where(conditional_relation).to_sql})"
196
- # update_all "load_factor = (#{FlightSegment.weighted_average_relation(:load_factor, :weighted_by => :passengers ).where(conditional_relation).to_sql})"
197
- # update_all "freight_share = (#{FlightSegment.weighted_average_relation(:freight_share, :weighted_by => :passengers ).where(conditional_relation).to_sql})"
198
- # update_all "payload = (#{FlightSegment.weighted_average_relation(:payload, :weighted_by => :passengers, :disaggregate_by => :departures_performed).where(conditional_relation).to_sql})"
199
-
200
- # update_all "weighting = (#{segments.project(segments[:passengers].sum).where(aircraft[:aircraft_type_code].eq(segments[:aircraft_type_code])).to_sql})"
97
+ end
98
+
99
+ # FIXME TODO do we want to restrict this to certain years?
100
+ process "Derive some average characteristics from flight segments" do
101
+ Aircraft.find_each do |aircraft|
102
+ aircraft.seats = aircraft.flight_segments.weighted_average :seats_per_flight, :weighted_by => :passengers
103
+ aircraft.passengers = aircraft.flight_segments.sum :passengers
104
+ aircraft.save
105
+ end
201
106
  end
202
107
 
203
108
  # FIXME TODO verify this