earth 0.0.18

Sign up to get free protection for your applications and to get access to all the features.
Files changed (189) hide show
  1. data/.document +5 -0
  2. data/.gitignore +22 -0
  3. data/.rvmrc +8 -0
  4. data/Gemfile +5 -0
  5. data/LICENSE +20 -0
  6. data/README.markdown +38 -0
  7. data/Rakefile +71 -0
  8. data/VERSION +1 -0
  9. data/earth.gemspec +265 -0
  10. data/lib/earth.rb +169 -0
  11. data/lib/earth/air.rb +13 -0
  12. data/lib/earth/air/aircraft.rb +32 -0
  13. data/lib/earth/air/aircraft/data_miner.rb +171 -0
  14. data/lib/earth/air/aircraft_class.rb +10 -0
  15. data/lib/earth/air/aircraft_class/data_miner.rb +42 -0
  16. data/lib/earth/air/aircraft_manufacturer.rb +9 -0
  17. data/lib/earth/air/aircraft_manufacturer/data_miner.rb +20 -0
  18. data/lib/earth/air/airline.rb +16 -0
  19. data/lib/earth/air/airline/data_miner.rb +57 -0
  20. data/lib/earth/air/airport.rb +44 -0
  21. data/lib/earth/air/airport/data_miner.rb +80 -0
  22. data/lib/earth/air/data_miner.rb +15 -0
  23. data/lib/earth/air/flight_configuration.rb +18 -0
  24. data/lib/earth/air/flight_configuration/data_miner.rb +4 -0
  25. data/lib/earth/air/flight_distance_class.rb +7 -0
  26. data/lib/earth/air/flight_distance_class/data_miner.rb +16 -0
  27. data/lib/earth/air/flight_domesticity.rb +6 -0
  28. data/lib/earth/air/flight_domesticity/data_miner.rb +57 -0
  29. data/lib/earth/air/flight_fuel_type.rb +17 -0
  30. data/lib/earth/air/flight_fuel_type/data_miner.rb +4 -0
  31. data/lib/earth/air/flight_propulsion.rb +18 -0
  32. data/lib/earth/air/flight_propulsion/data_miner.rb +4 -0
  33. data/lib/earth/air/flight_seat_class.rb +12 -0
  34. data/lib/earth/air/flight_seat_class/data_miner.rb +36 -0
  35. data/lib/earth/air/flight_segment.rb +38 -0
  36. data/lib/earth/air/flight_segment/data_miner.rb +334 -0
  37. data/lib/earth/air/flight_service.rb +18 -0
  38. data/lib/earth/air/flight_service/data_miner.rb +4 -0
  39. data/lib/earth/all.rb +10 -0
  40. data/lib/earth/automobile.rb +8 -0
  41. data/lib/earth/automobile/automobile_fuel_type.rb +18 -0
  42. data/lib/earth/automobile/automobile_fuel_type/data_miner.rb +45 -0
  43. data/lib/earth/automobile/automobile_make.rb +14 -0
  44. data/lib/earth/automobile/automobile_make/data_miner.rb +68 -0
  45. data/lib/earth/automobile/automobile_make_fleet_year.rb +15 -0
  46. data/lib/earth/automobile/automobile_make_fleet_year/data_miner.rb +29 -0
  47. data/lib/earth/automobile/automobile_make_year.rb +14 -0
  48. data/lib/earth/automobile/automobile_make_year/data_miner.rb +45 -0
  49. data/lib/earth/automobile/automobile_model.rb +14 -0
  50. data/lib/earth/automobile/automobile_model/data_miner.rb +38 -0
  51. data/lib/earth/automobile/automobile_model_year.rb +15 -0
  52. data/lib/earth/automobile/automobile_model_year/data_miner.rb +51 -0
  53. data/lib/earth/automobile/automobile_size_class.rb +14 -0
  54. data/lib/earth/automobile/automobile_size_class/data_miner.rb +43 -0
  55. data/lib/earth/automobile/automobile_variant.rb +17 -0
  56. data/lib/earth/automobile/automobile_variant/data_miner.rb +464 -0
  57. data/lib/earth/automobile/data_miner.rb +8 -0
  58. data/lib/earth/bus.rb +1 -0
  59. data/lib/earth/bus/bus_class.rb +19 -0
  60. data/lib/earth/bus/bus_class/data_miner.rb +41 -0
  61. data/lib/earth/bus/data_miner.rb +1 -0
  62. data/lib/earth/conversions_ext.rb +45 -0
  63. data/lib/earth/data_miner.rb +10 -0
  64. data/lib/earth/diet.rb +2 -0
  65. data/lib/earth/diet/data_miner.rb +2 -0
  66. data/lib/earth/diet/diet_class.rb +15 -0
  67. data/lib/earth/diet/diet_class/data_miner.rb +36 -0
  68. data/lib/earth/diet/food_group.rb +17 -0
  69. data/lib/earth/diet/food_group/data_miner.rb +26 -0
  70. data/lib/earth/fuel.rb +2 -0
  71. data/lib/earth/fuel/data_miner.rb +2 -0
  72. data/lib/earth/fuel/fuel_price.rb +13 -0
  73. data/lib/earth/fuel/fuel_price/data_miner.rb +20 -0
  74. data/lib/earth/fuel/fuel_type.rb +18 -0
  75. data/lib/earth/fuel/fuel_type/data_miner.rb +37 -0
  76. data/lib/earth/industry.rb +10 -0
  77. data/lib/earth/industry/data_miner.rb +2 -0
  78. data/lib/earth/industry/industries_product_lines.rb +6 -0
  79. data/lib/earth/industry/industries_sectors.rb +6 -0
  80. data/lib/earth/industry/industry.rb +12 -0
  81. data/lib/earth/industry/industry/data_miner.rb +21 -0
  82. data/lib/earth/industry/merchant.rb +5 -0
  83. data/lib/earth/industry/merchant_categories_industries.rb +8 -0
  84. data/lib/earth/industry/merchant_category.rb +6 -0
  85. data/lib/earth/industry/product_line.rb +10 -0
  86. data/lib/earth/industry/product_line/data_miner.rb +18 -0
  87. data/lib/earth/industry/product_lines_sectors.rb +6 -0
  88. data/lib/earth/industry/sector.rb +3 -0
  89. data/lib/earth/inflectors.rb +9 -0
  90. data/lib/earth/locality.rb +10 -0
  91. data/lib/earth/locality/census_division.rb +17 -0
  92. data/lib/earth/locality/census_division/data_miner.rb +21 -0
  93. data/lib/earth/locality/census_region.rb +13 -0
  94. data/lib/earth/locality/census_region/data_miner.rb +17 -0
  95. data/lib/earth/locality/climate_division.rb +17 -0
  96. data/lib/earth/locality/climate_division/data_miner.rb +20 -0
  97. data/lib/earth/locality/country.rb +13 -0
  98. data/lib/earth/locality/country/data_miner.rb +19 -0
  99. data/lib/earth/locality/data_miner.rb +10 -0
  100. data/lib/earth/locality/egrid_region.rb +15 -0
  101. data/lib/earth/locality/egrid_region/data_miner.rb +19 -0
  102. data/lib/earth/locality/egrid_subregion.rb +16 -0
  103. data/lib/earth/locality/egrid_subregion/data_miner.rb +26 -0
  104. data/lib/earth/locality/petroleum_administration_for_defense_district.rb +13 -0
  105. data/lib/earth/locality/petroleum_administration_for_defense_district/data_miner.rb +21 -0
  106. data/lib/earth/locality/state.rb +22 -0
  107. data/lib/earth/locality/state/data_miner.rb +37 -0
  108. data/lib/earth/locality/urbanity.rb +20 -0
  109. data/lib/earth/locality/urbanity/data_miner.rb +4 -0
  110. data/lib/earth/locality/zip_code.rb +29 -0
  111. data/lib/earth/locality/zip_code/data_miner.rb +36 -0
  112. data/lib/earth/pet.rb +4 -0
  113. data/lib/earth/pet/breed.rb +15 -0
  114. data/lib/earth/pet/breed/data_miner.rb +25 -0
  115. data/lib/earth/pet/breed_gender.rb +14 -0
  116. data/lib/earth/pet/breed_gender/data_miner.rb +21 -0
  117. data/lib/earth/pet/data_miner.rb +4 -0
  118. data/lib/earth/pet/gender.rb +10 -0
  119. data/lib/earth/pet/gender/data_miner.rb +13 -0
  120. data/lib/earth/pet/species.rb +40 -0
  121. data/lib/earth/pet/species/data_miner.rb +42 -0
  122. data/lib/earth/rail.rb +1 -0
  123. data/lib/earth/rail/data_miner.rb +1 -0
  124. data/lib/earth/rail/rail_class.rb +16 -0
  125. data/lib/earth/rail/rail_class/data_miner.rb +36 -0
  126. data/lib/earth/residence.rb +8 -0
  127. data/lib/earth/residence/air_conditioner_use.rb +30 -0
  128. data/lib/earth/residence/air_conditioner_use/data_miner.rb +4 -0
  129. data/lib/earth/residence/clothes_machine_use.rb +33 -0
  130. data/lib/earth/residence/clothes_machine_use/data_miner.rb +4 -0
  131. data/lib/earth/residence/data_miner.rb +8 -0
  132. data/lib/earth/residence/dishwasher_use.rb +33 -0
  133. data/lib/earth/residence/dishwasher_use/data_miner.rb +4 -0
  134. data/lib/earth/residence/residence_appliance.rb +7 -0
  135. data/lib/earth/residence/residence_appliance/data_miner.rb +19 -0
  136. data/lib/earth/residence/residence_class.rb +26 -0
  137. data/lib/earth/residence/residence_class/data_miner.rb +4 -0
  138. data/lib/earth/residence/residence_fuel_price.rb +18 -0
  139. data/lib/earth/residence/residence_fuel_price/data_miner.rb +200 -0
  140. data/lib/earth/residence/residence_fuel_type.rb +33 -0
  141. data/lib/earth/residence/residence_fuel_type/data_miner.rb +18 -0
  142. data/lib/earth/residence/residential_energy_consumption_survey_response.rb +67 -0
  143. data/lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb +264 -0
  144. data/lib/earth/schema.rb +8 -0
  145. data/spec/lib/earth_spec.rb +25 -0
  146. data/spec/spec_helper.rb +11 -0
  147. data/vendor/geokit-rails/CHANGELOG.rdoc +46 -0
  148. data/vendor/geokit-rails/MIT-LICENSE +20 -0
  149. data/vendor/geokit-rails/README.markdown +561 -0
  150. data/vendor/geokit-rails/Rakefile +18 -0
  151. data/vendor/geokit-rails/about.yml +9 -0
  152. data/vendor/geokit-rails/assets/api_keys_template +61 -0
  153. data/vendor/geokit-rails/init.rb +1 -0
  154. data/vendor/geokit-rails/install.rb +14 -0
  155. data/vendor/geokit-rails/lib/geokit-rails.rb +24 -0
  156. data/vendor/geokit-rails/lib/geokit-rails/acts_as_mappable.rb +456 -0
  157. data/vendor/geokit-rails/lib/geokit-rails/adapters/abstract.rb +31 -0
  158. data/vendor/geokit-rails/lib/geokit-rails/adapters/mysql.rb +22 -0
  159. data/vendor/geokit-rails/lib/geokit-rails/adapters/postgresql.rb +22 -0
  160. data/vendor/geokit-rails/lib/geokit-rails/adapters/sqlserver.rb +43 -0
  161. data/vendor/geokit-rails/lib/geokit-rails/defaults.rb +22 -0
  162. data/vendor/geokit-rails/lib/geokit-rails/geocoder_control.rb +16 -0
  163. data/vendor/geokit-rails/lib/geokit-rails/ip_geocode_lookup.rb +46 -0
  164. data/vendor/geokit-rails/test/acts_as_mappable_test.rb +474 -0
  165. data/vendor/geokit-rails/test/boot.rb +25 -0
  166. data/vendor/geokit-rails/test/database.yml +20 -0
  167. data/vendor/geokit-rails/test/fixtures/companies.yml +7 -0
  168. data/vendor/geokit-rails/test/fixtures/custom_locations.yml +54 -0
  169. data/vendor/geokit-rails/test/fixtures/locations.yml +54 -0
  170. data/vendor/geokit-rails/test/fixtures/mock_addresses.yml +17 -0
  171. data/vendor/geokit-rails/test/fixtures/mock_families.yml +2 -0
  172. data/vendor/geokit-rails/test/fixtures/mock_houses.yml +9 -0
  173. data/vendor/geokit-rails/test/fixtures/mock_organizations.yml +5 -0
  174. data/vendor/geokit-rails/test/fixtures/mock_people.yml +5 -0
  175. data/vendor/geokit-rails/test/fixtures/stores.yml +0 -0
  176. data/vendor/geokit-rails/test/ip_geocode_lookup_test.rb +77 -0
  177. data/vendor/geokit-rails/test/models/company.rb +3 -0
  178. data/vendor/geokit-rails/test/models/custom_location.rb +12 -0
  179. data/vendor/geokit-rails/test/models/location.rb +4 -0
  180. data/vendor/geokit-rails/test/models/mock_address.rb +4 -0
  181. data/vendor/geokit-rails/test/models/mock_family.rb +3 -0
  182. data/vendor/geokit-rails/test/models/mock_house.rb +3 -0
  183. data/vendor/geokit-rails/test/models/mock_organization.rb +4 -0
  184. data/vendor/geokit-rails/test/models/mock_person.rb +4 -0
  185. data/vendor/geokit-rails/test/models/store.rb +3 -0
  186. data/vendor/geokit-rails/test/schema.rb +60 -0
  187. data/vendor/geokit-rails/test/tasks.rake +31 -0
  188. data/vendor/geokit-rails/test/test_helper.rb +23 -0
  189. metadata +444 -0
@@ -0,0 +1,20 @@
1
+ AircraftManufacturer.class_eval do
2
+ data_miner do
3
+ process "Start from scratch" do
4
+ connection.drop_table table_name
5
+ end
6
+
7
+ schema Earth.database_options do
8
+ string 'name'
9
+ end
10
+
11
+ process "Derive a list of aircraft manufacturers from aircraft" do
12
+ Aircraft.run_data_miner!
13
+ connection.execute %{
14
+ INSERT IGNORE INTO aircraft_manufacturers(name)
15
+ SELECT aircraft.manufacturer_name FROM aircraft WHERE LENGTH(aircraft.manufacturer_name) > 0
16
+ }
17
+ end
18
+ end
19
+ end
20
+
@@ -0,0 +1,16 @@
1
+ class Airline < ActiveRecord::Base
2
+ set_primary_key :iata_code
3
+
4
+ # has_many :airline_aircraft, :class_name => 'AirlineAircraft'
5
+ # has_many :seat_classes, :class_name => 'AirlineSeatClass'
6
+ has_many :segments, :class_name => "FlightSegment", :foreign_key => 'airline_iata_code'
7
+ # has_many :airline_aircraft_seat_classes, :class_name => 'AirlineAircraftSeatClass'
8
+
9
+ data_miner do
10
+ tap "Brighter Planet's sanitized airlines data", Earth.taps_server
11
+ end
12
+
13
+ def all_flights_domestic?
14
+ !international?
15
+ end
16
+ end
@@ -0,0 +1,57 @@
1
+ Airline.class_eval do
2
+ class Airline::Guru
3
+ # needed by errata
4
+ def is_not_coral_air?(row); row['Code'].to_i != 19155; end # 19155 Coral Air Inc.: COR
5
+ def is_not_aviacion_y_comercio?(row); row['Code'].to_i != 19452; end # 19452 Aviacion Y Comercio S.A.: AO
6
+ def is_not_air_china?(row); row['Code'].to_i != 19543; end # 19543 Air China: CA
7
+ def is_not_south_african_airways?(row); row['Code'].to_i != 19570; end # 19570 South African Airways: SA
8
+ def is_not_continental_airlines?(row); row['Code'].to_i != 19704; end # 19704 Continental Air Lines Inc.: CO
9
+ def is_not_sallee_s_aviation?(row); row['Code'].to_i != 19740; end # 19740 Sallee's Aviation: SAL
10
+ def is_not_air_berlin?(row); row['Code'].to_i != 21361; end # "21361","Air Berlin PLC and CO: AB"
11
+ end
12
+
13
+ data_miner do
14
+ schema Earth.database_options do
15
+ string 'iata_code'
16
+ string 'name'
17
+ string 'dot_airline_id_code'
18
+ boolean 'international'
19
+ float 'seats'
20
+ float 'distance'
21
+ string 'distance_units'
22
+ float 'load_factor'
23
+ float 'freight_share'
24
+ float 'payload'
25
+ string 'payload_units'
26
+ end
27
+
28
+ import "the T100 AIRLINE_ID lookup table, which also includes IATA codes",
29
+ :url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_AIRLINE_ID',
30
+ :errata => Errata.new(:url => 'http://static.brighterplanet.com/science/data/transport/air/airlines/errata.csv',
31
+ :responder => Airline::Guru.new) do
32
+ key 'iata_code', :field_name => 'Description', :split => { :pattern => /:/, :keep => 1 }
33
+ store 'dot_airline_id_code', :field_name => 'Code'
34
+ store 'name', :field_name => 'Description', :split => { :pattern => /:/, :keep => 0 }
35
+ end
36
+
37
+ process "Determine whether airlines fly internationally by looking at flight segments" do
38
+ FlightSegment.run_data_miner!
39
+ update_all 'international = 1', '(SELECT COUNT(*) FROM flight_segments WHERE flight_segments.airline_iata_code = airlines.iata_code AND flight_segments.origin_country_iso_3166_code != flight_segments.dest_country_iso_3166_code AND flight_segments.origin_country_iso_3166_code IS NOT NULL AND flight_segments.dest_country_iso_3166_code IS NOT NULL) > 0'
40
+ end
41
+
42
+ process "Derive some average flight characteristics from flight segments" do
43
+ FlightSegment.run_data_miner!
44
+ airlines = Airline.arel_table
45
+ segments = FlightSegment.arel_table
46
+
47
+ conditional_relation = airlines[:iata_code].eq(segments[:airline_iata_code])
48
+
49
+ update_all "seats = (#{FlightSegment.weighted_average_relation(:seats, :weighted_by => :passengers ).where(conditional_relation).to_sql})"
50
+ update_all "distance = (#{FlightSegment.weighted_average_relation(:distance, :weighted_by => :passengers ).where(conditional_relation).to_sql})"
51
+ update_all "load_factor = (#{FlightSegment.weighted_average_relation(:load_factor, :weighted_by => :passengers ).where(conditional_relation).to_sql})"
52
+ update_all "freight_share = (#{FlightSegment.weighted_average_relation(:freight_share, :weighted_by => :passengers ).where(conditional_relation).to_sql})"
53
+ update_all "payload = (#{FlightSegment.weighted_average_relation(:payload, :weighted_by => :passengers, :disaggregate_by => :departures_performed).where(conditional_relation).to_sql})"
54
+ end
55
+ end
56
+ end
57
+
@@ -0,0 +1,44 @@
1
+ class Airport < ActiveRecord::Base
2
+ set_primary_key :iata_code
3
+
4
+ # --------------------------------
5
+ # virtual has_many association
6
+ # has_many :segments won't work because there's no general way to specify the correct conditions
7
+ # even if you get clever with it, like
8
+ # has_many :segments,
9
+ # :class_name => 'FlightSegment',
10
+ # :foreign_key => 'origin_airport_id',
11
+ # :conditions => 'flight_segments.destination_airport_id = #{id}'
12
+ # you get queries like "`flight_segments`.origin_airport_id = 3654 AND (flight_segments.destination_airport_id = 3654))"
13
+ # in which you notice the AND which must be an OR
14
+ # and you can't just do finder_sql, because that breaks any other :select
15
+ def segments
16
+ FlightSegment.scoped :conditions => ['origin_airport_id = ? OR destination_airport_id = ?', id, id]
17
+ end
18
+ # --------------------------------
19
+
20
+ belongs_to :country, :foreign_key => 'country_iso_3166_code'
21
+ acts_as_mappable :default_units => :nms,
22
+ :lat_column_name => :latitude,
23
+ :lng_column_name => :longitude
24
+
25
+ data_miner do
26
+ tap "Brighter Planet's sanitized airports data", Earth.taps_server
27
+
28
+ process "pull dependencies" do
29
+ run_data_miner_on_belongs_to_associations
30
+ end
31
+ end
32
+
33
+ def all_flights_from_here_domestic?
34
+ !international_origin?
35
+ end
36
+
37
+ def all_flights_to_here_domestic?
38
+ !international_destination?
39
+ end
40
+
41
+ def united_states?
42
+ country == Country.united_states
43
+ end
44
+ end
@@ -0,0 +1,80 @@
1
+ Airport.class_eval do
2
+ data_miner do
3
+ schema Earth.database_options do
4
+ string 'iata_code'
5
+ string 'name'
6
+ string 'city'
7
+ string 'country_name'
8
+ string 'country_iso_3166_code'
9
+ float 'latitude'
10
+ float 'longitude'
11
+ float 'seats'
12
+ float 'distance'
13
+ string 'distance_units'
14
+ float 'load_factor'
15
+ float 'freight_share'
16
+ float 'payload'
17
+ string 'payload_units'
18
+ boolean 'international_origin'
19
+ boolean 'international_destination'
20
+ end
21
+
22
+ import "the OpenFlights.org airports database",
23
+ :url => 'http://openflights.svn.sourceforge.net/viewvc/openflights/openflights/data/airports.dat',
24
+ :headers => false,
25
+ :select => lambda { |row| row[4].present? } do
26
+ key 'iata_code', :field_number => 4
27
+ store 'name', :field_number => 1
28
+ store 'city', :field_number => 2
29
+ store 'country_name', :field_number => 3
30
+ store 'country_iso_3166_code', :field_number => 3, :upcase => true, :dictionary => { :input => 'name', :output => 'iso_3166_code', :url => 'http://data.brighterplanet.com/countries.csv' }
31
+ store 'latitude', :field_number => 6
32
+ store 'longitude', :field_number => 7
33
+ end
34
+
35
+ # step.await :other_class => FlightSegment do |deferred|
36
+ # deferred.derive :country # this uses a heuristic that depends on flight segments
37
+ # class << self
38
+ # def derive_country
39
+ # update_all('country_id = (SELECT flight_segments.origin_country_id FROM flight_segments WHERE flight_segments.origin_airport_id = airports.id AND flight_segments.origin_country_id IS NOT NULL LIMIT 1)', 'airports.country_id IS NULL')
40
+ # update_all('country_id = (SELECT flight_segments.destination_country_id FROM flight_segments WHERE flight_segments.destination_airport_id = airports.id AND flight_segments.destination_country_id IS NOT NULL LIMIT 1)', 'airports.country_id IS NULL')
41
+ # Country.all.each do |c|
42
+ # next if c.name.blank?
43
+ # update_all("country_id = #{c.id}", ["airports.country_id IS NULL AND airports.country_name LIKE ?", "%#{c.name.upcase}%"])
44
+ # end
45
+ # Airport.all(:conditions => 'country_id IS NULL AND country_name IS NOT NULL').each do |a|
46
+ # c = Country.find(:first, :conditions => ["name like ?", "%#{a.country_name}%"])
47
+ # a.update_attributes(:country_id => c.id) if c
48
+ # end
49
+ # end
50
+ # end
51
+
52
+ process "Determine whether each airport serves international flights" do
53
+ FlightSegment.run_data_miner!
54
+ update_all 'international_origin = 1', '(SELECT COUNT(*) FROM flight_segments WHERE flight_segments.origin_airport_iata_code = airports.iata_code AND flight_segments.origin_country_iso_3166_code != flight_segments.dest_country_iso_3166_code AND flight_segments.origin_country_iso_3166_code IS NOT NULL AND flight_segments.dest_country_iso_3166_code IS NOT NULL LIMIT 1) > 0'
55
+ update_all 'international_destination = 1', '(SELECT COUNT(*) FROM flight_segments WHERE flight_segments.dest_airport_iata_code = airports.iata_code AND flight_segments.origin_country_iso_3166_code != flight_segments.dest_country_iso_3166_code AND flight_segments.origin_country_iso_3166_code IS NOT NULL AND flight_segments.dest_country_iso_3166_code IS NOT NULL LIMIT 1) > 0'
56
+ end
57
+
58
+ # sabshere 5/24/10 using temporary tables because the WHERE clause has a very slow OR condition: iata_code = dest_iata_code OR iata_code = origin_iata_code
59
+ process "Derive some average flight characteristics from flight segments" do
60
+ FlightSegment.run_data_miner!
61
+ segments = FlightSegment.arel_table
62
+ airports = Airport.arel_table
63
+
64
+ find_in_batches do |batch|
65
+ batch.each do |airport|
66
+ targeting_relation = airports[:iata_code].eq airport.iata_code
67
+ conditional_relation = segments[:origin_airport_iata_code].eq(airport.iata_code).or(segments[:dest_airport_iata_code].eq(airport.iata_code))
68
+ connection.execute "CREATE TEMPORARY TABLE tmp1 #{FlightSegment.where(conditional_relation).to_sql}"
69
+ update_all "seats = (#{FlightSegment.weighted_average_relation(:seats, :weighted_by => :passengers ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
70
+ update_all "distance = (#{FlightSegment.weighted_average_relation(:distance, :weighted_by => :passengers ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
71
+ update_all "load_factor = (#{FlightSegment.weighted_average_relation(:load_factor, :weighted_by => :passengers ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
72
+ update_all "freight_share = (#{FlightSegment.weighted_average_relation(:freight_share, :weighted_by => :passengers ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
73
+ update_all "payload = (#{FlightSegment.weighted_average_relation(:payload, :weighted_by => :passengers, :disaggregate_by => :departures_performed).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
74
+ connection.execute 'DROP TABLE tmp1'
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
80
+
@@ -0,0 +1,15 @@
1
+ require 'earth/locality'
2
+ require 'earth/locality/data_miner'
3
+ require 'earth/air/aircraft/data_miner'
4
+ require 'earth/air/aircraft_class/data_miner'
5
+ require 'earth/air/aircraft_manufacturer/data_miner'
6
+ require 'earth/air/airline/data_miner'
7
+ require 'earth/air/airport/data_miner'
8
+ require 'earth/air/flight_configuration/data_miner'
9
+ require 'earth/air/flight_distance_class/data_miner'
10
+ require 'earth/air/flight_domesticity/data_miner'
11
+ require 'earth/air/flight_fuel_type/data_miner'
12
+ require 'earth/air/flight_propulsion/data_miner'
13
+ require 'earth/air/flight_seat_class/data_miner'
14
+ require 'earth/air/flight_segment/data_miner'
15
+ require 'earth/air/flight_service/data_miner'
@@ -0,0 +1,18 @@
1
+ class FlightConfiguration < ActiveRecord::Base
2
+ set_primary_key :name
3
+
4
+ data_miner do
5
+ schema do
6
+ string 'name'
7
+ string 'bts_aircraft_configuration_code'
8
+ end
9
+
10
+ process "derive from flight segments" do
11
+ FlightSegment.run_data_miner!
12
+ connection.execute %{
13
+ INSERT IGNORE INTO flight_configurations(name, bts_aircraft_configuration_code)
14
+ SELECT flight_segments.configuration_id, flight_segments.bts_aircraft_configuration_code FROM flight_segments WHERE LENGTH(flight_segments.configuration_id) > 0
15
+ }
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,4 @@
1
+ FlightConfiguration.class_eval do
2
+ # data_miner do
3
+ #...
4
+ end
@@ -0,0 +1,7 @@
1
+ class FlightDistanceClass < ActiveRecord::Base
2
+ set_primary_key :name
3
+
4
+ data_miner do
5
+ tap "Brighter Planet's sanitized distance class data", Earth.taps_server
6
+ end
7
+ end
@@ -0,0 +1,16 @@
1
+ FlightDistanceClass.class_eval do
2
+ data_miner do
3
+ schema Earth.database_options do
4
+ string 'name'
5
+ float 'distance'
6
+ string 'distance_units'
7
+ end
8
+
9
+ import "a list of Brighter Planet-defined distance classes",
10
+ :url => 'http://static.brighterplanet.com/science/data/transport/air/distance_classes/distance_classes.csv' do
11
+ key 'name'
12
+ store 'distance', :units_field_name => 'units', :to_units => :kilometres
13
+ end
14
+ end
15
+ end
16
+
@@ -0,0 +1,6 @@
1
+ class FlightDomesticity < ActiveRecord::Base
2
+ set_primary_key :name
3
+ data_miner do
4
+ tap "Brighter Planet's flight domesticity info", Earth.taps_server
5
+ end
6
+ end
@@ -0,0 +1,57 @@
1
+ FlightDomesticity.class_eval do
2
+ data_miner do
3
+ process "Start from scratch" do
4
+ connection.drop_table table_name
5
+ end
6
+
7
+ schema Earth.database_options do
8
+ string 'name'
9
+ string 'bts_data_source_code'
10
+ float 'distance'
11
+ string 'distance_units'
12
+ float 'freight_share'
13
+ float 'load_factor'
14
+ float 'seats'
15
+ float 'payload'
16
+ string 'payload_units'
17
+ end
18
+
19
+ process "Derive flight domesticities from flight segments" do # FIXME TODO might make more sense to combine foreign and domestic carriers -> domestic flight (all carriers), international flight (all carriers)
20
+ FlightSegment.run_data_miner!
21
+ connection.execute %{
22
+ INSERT IGNORE INTO flight_domesticities(name, bts_data_source_code)
23
+ SELECT flight_segments.domesticity, flight_segments.bts_data_source_code FROM flight_segments WHERE LENGTH(flight_segments.domesticity) > 0
24
+ }
25
+ end
26
+
27
+ process "Derive average flight characteristics from flight segments" do
28
+ FlightSegment.run_data_miner!
29
+ segments = FlightSegment.arel_table
30
+ flight_domesticities = FlightDomesticity.arel_table
31
+ ## slow, all-in-one method
32
+ # conditional_relation = flight_domesticities[:name].eq(segments[:domesticity])
33
+ # update_all "seats = (#{FlightSegment.weighted_average_relation(:seats, :weighted_by => :passengers ).where(conditional_relation).to_sql})"
34
+ # update_all "distance = (#{FlightSegment.weighted_average_relation(:distance, :weighted_by => :passengers ).where(conditional_relation).to_sql})"
35
+ # update_all "load_factor = (#{FlightSegment.weighted_average_relation(:load_factor, :weighted_by => :passengers ).where(conditional_relation).to_sql})"
36
+ # update_all "freight_share = (#{FlightSegment.weighted_average_relation(:freight_share, :weighted_by => :passengers ).where(conditional_relation).to_sql})"
37
+ # update_all "payload = (#{FlightSegment.weighted_average_relation(:payload, :weighted_by => :passengers, :disaggregate_by => :departures_performed).where(conditional_relation).to_sql})"
38
+ ## fast method using temp tables
39
+ find_in_batches do |batch|
40
+ batch.each do |flight_domesticity|
41
+ targeting_relation = flight_domesticities[:name].eq flight_domesticity.name
42
+ conditional_relation = segments[:domesticity].eq flight_domesticity.name
43
+ connection.execute "CREATE TEMPORARY TABLE tmp1 #{FlightSegment.where(conditional_relation).to_sql}"
44
+ update_all "seats = (#{FlightSegment.weighted_average_relation(:seats, :weighted_by => :passengers ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
45
+ update_all "distance = (#{FlightSegment.weighted_average_relation(:distance, :weighted_by => :passengers ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
46
+ update_all "load_factor = (#{FlightSegment.weighted_average_relation(:load_factor, :weighted_by => :passengers ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
47
+ update_all "freight_share = (#{FlightSegment.weighted_average_relation(:freight_share, :weighted_by => :passengers ).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
48
+ update_all "payload = (#{FlightSegment.weighted_average_relation(:payload, :weighted_by => :passengers, :disaggregate_by => :departures_performed).to_sql.gsub('flight_segments', 'tmp1')})", targeting_relation.to_sql
49
+ connection.execute 'DROP TABLE tmp1'
50
+ end
51
+ end
52
+ update_all "distance_units = 'kilometres'"
53
+ update_all "payload_units = 'kilograms'"
54
+ end
55
+ end
56
+ end
57
+
@@ -0,0 +1,17 @@
1
+ class FlightFuelType < ActiveRecord::Base
2
+ # this fallback is jet fuel
3
+ falls_back_on :emission_factor => (21.09.pounds.to(:kilograms) / 1.gallons.to(:litres)), # in pounds CO2/gallon fuel: http://www.eia.doe.gov/oiaf/1605/excel/Fuel%20Emission%20Factors.xls
4
+ :radiative_forcing_index => 2, # from Matt
5
+ :density => 3.057 # kg / gal
6
+
7
+ data_miner do
8
+ schema do
9
+ string 'name'
10
+ float 'emission_factor'
11
+ float 'radiative_forcing_index'
12
+ float 'density'
13
+ end
14
+
15
+ # we just always use the fallback
16
+ end
17
+ end
@@ -0,0 +1,4 @@
1
+ FlightFuelType.class_eval do
2
+ # data_miner do
3
+ #...
4
+ end
@@ -0,0 +1,18 @@
1
+ class FlightPropulsion < ActiveRecord::Base
2
+ set_primary_key :name
3
+
4
+ data_miner do
5
+ schema do
6
+ string 'name'
7
+ string 'bts_aircraft_group_code'
8
+ end
9
+
10
+ process "derive from flight segments" do
11
+ FlightSegment.run_data_miner!
12
+ connection.execute %{
13
+ INSERT IGNORE INTO flight_propulsions(name, bts_aircraft_group_code)
14
+ SELECT flight_segments.propulsion_id, flight_segments.bts_aircraft_group_code FROM flight_segments WHERE LENGTH(flight_segments.propulsion_id) > 0
15
+ }
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,4 @@
1
+ FlightPropulsion.class_eval do
2
+ # data_miner do
3
+ #...
4
+ end
@@ -0,0 +1,12 @@
1
+ class FlightSeatClass < ActiveRecord::Base
2
+ set_primary_key :name
3
+ # has_many :airline_seat_classes, :class_name => 'AirlineSeatClass'
4
+ # has_many :aircraft_seat_classes, :class_name => 'AircraftSeatClass'
5
+ # has_many :airline_aircraft_seat_classes, :class_name => 'AirlineAircraftSeatClass'
6
+
7
+ falls_back_on :multiplier => 1
8
+
9
+ data_miner do
10
+ tap "Brighter Planet's sanitized flight seat class data", Earth.taps_server
11
+ end
12
+ end
@@ -0,0 +1,36 @@
1
+ FlightSeatClass.class_eval do
2
+ data_miner do
3
+ schema Earth.database_options do
4
+ string 'name'
5
+ float 'multiplier'
6
+ integer 'seats'
7
+ end
8
+
9
+ import "a list of Brighter Planet-defined seat classes and pre-calculated multipliers",
10
+ :url => 'http://static.brighterplanet.com/science/data/transport/air/seat_classes/seat_classes.csv' do
11
+ key 'name'
12
+ store 'multiplier'
13
+ end
14
+
15
+ # sabshere 5/21/10 in case we ever need this
16
+ # class << self
17
+ # def refresh
18
+ # update_all_weighted_averages(:seats, :weighted_by => :airline_aircraft_seat_classes)
19
+ # update_all_weighted_averages(:multiplier, :weighted_by => :airline_aircraft_seat_classes)
20
+ # end
21
+ #
22
+ # def safe_find_by_name(name)
23
+ # guess = case name.to_s.downcase.gsub(/[^a-z]/, '')
24
+ # when /first/
25
+ # 'first'
26
+ # when /busi/, /exec/
27
+ # 'business'
28
+ # when /econ/, /coach/
29
+ # 'economy'
30
+ # end
31
+ # find_or_create_by_name(guess) unless guess.nil?
32
+ # end
33
+ # end
34
+ end
35
+ end
36
+