earth 0.0.18

data/lib/earth/air/flight_segment.rb

@@ -0,0 +1,38 @@
+class FlightSegment < ActiveRecord::Base
+  set_primary_key :row_hash
+  
+  extend CohortScope
+  self.minimum_cohort_size = 5 #FIXME ??
+  
+  belongs_to :airline,             :foreign_key => 'airline_iata_code'
+  belongs_to :origin_airport,      :foreign_key => "origin_airport_iata_code",     :class_name => 'Airport'
+  belongs_to :destination_airport, :foreign_key => "dest_airport_iata_code",       :class_name => 'Airport'
+  belongs_to :origin_country,      :foreign_key => 'origin_country_iso_3166_code', :class_name => 'Country'
+  belongs_to :destination_country, :foreign_key => 'dest_country_iso_3166_code',   :class_name => 'Country'
+  belongs_to :aircraft,            :foreign_key => 'bts_aircraft_type_code',                                              :primary_key => 'bts_aircraft_type_code'
+  belongs_to :propulsion,                                                          :class_name => 'FlightPropulsion'
+  belongs_to :configuration,                                                       :class_name => 'FlightConfiguration'
+  belongs_to :service_class,                                                       :class_name => 'FlightService'
+  belongs_to :domesticity,                                                         :class_name => 'FlightDomesticity'
+  
+  falls_back_on :load_factor =>   lambda { weighted_average(:load_factor,   :weighted_by => :passengers) }, # 0.78222911236768
+                :freight_share => lambda { weighted_average(:freight_share, :weighted_by => :passengers) },  # 0.024017329363736
+                :seats => lambda { weighted_average :seats, :weighted_by => :passengers }
+
+  data_miner do
+    tap "Brighter Planet's sanitized T100 data", Earth.taps_server
+    
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+
+  INPUT_CHARACTERISTICS = [
+    :origin_airport,
+    :destination_airport,
+    :aircraft,
+    :airline,
+    :propulsion,
+    :domesticity
+  ]
+end

data/lib/earth/air/flight_segment/data_miner.rb

@@ -0,0 +1,334 @@
+FlightSegment.class_eval do
+  URL = 'http://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=293&Has_Group=3&Is_Zipped=0'
+  FORM_DATA = %{
+    UserTableName=T_100_Segment__All_Carriers&
+    DBShortName=Air_Carriers&
+    RawDataTable=T_T100_SEGMENT_ALL_CARRIER&
+    sqlstr=+SELECT+DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE+FROM++T_T100_SEGMENT_ALL_CARRIER+WHERE+Month+%3D__MONTH_NUMBER__+AND+YEAR%3D__YEAR__&
+    varlist=DEPARTURES_SCHEDULED%2CDEPARTURES_PERFORMED%2CPAYLOAD%2CSEATS%2CPASSENGERS%2CFREIGHT%2CMAIL%2CDISTANCE%2CRAMP_TO_RAMP%2CAIR_TIME%2CUNIQUE_CARRIER%2CAIRLINE_ID%2CUNIQUE_CARRIER_NAME%2CUNIQUE_CARRIER_ENTITY%2CREGION%2CCARRIER%2CCARRIER_NAME%2CCARRIER_GROUP%2CCARRIER_GROUP_NEW%2CORIGIN%2CORIGIN_CITY_NAME%2CORIGIN_CITY_NUM%2CORIGIN_STATE_ABR%2CORIGIN_STATE_FIPS%2CORIGIN_STATE_NM%2CORIGIN_COUNTRY%2CORIGIN_COUNTRY_NAME%2CORIGIN_WAC%2CDEST%2CDEST_CITY_NAME%2CDEST_CITY_NUM%2CDEST_STATE_ABR%2CDEST_STATE_FIPS%2CDEST_STATE_NM%2CDEST_COUNTRY%2CDEST_COUNTRY_NAME%2CDEST_WAC%2CAIRCRAFT_GROUP%2CAIRCRAFT_TYPE%2CAIRCRAFT_CONFIG%2CYEAR%2CQUARTER%2CMONTH%2CDISTANCE_GROUP%2CCLASS%2CDATA_SOURCE&
+    grouplist=&
+    suml=&
+    sumRegion=&
+    filter1=title%3D&
+    filter2=title%3D&
+    geo=All%A0&
+    time=__MONTH_NAME__&
+    timename=Month&
+    GEOGRAPHY=All&
+    XYEAR=__YEAR__&
+    FREQUENCY=__MONTH_NUMBER__&
+    AllVars=All&
+    VarName=DEPARTURES_SCHEDULED&
+    VarDesc=DepScheduled&
+    VarType=Num&
+    VarName=DEPARTURES_PERFORMED&
+    VarDesc=DepPerformed&
+    VarType=Num&
+    VarName=PAYLOAD&
+    VarDesc=Payload&
+    VarType=Num&
+    VarName=SEATS&
+    VarDesc=Seats&
+    VarType=Num&
+    VarName=PASSENGERS&
+    VarDesc=Passengers&
+    VarType=Num&
+    VarName=FREIGHT&
+    VarDesc=Freight&
+    VarType=Num&
+    VarName=MAIL&
+    VarDesc=Mail&
+    VarType=Num&
+    VarName=DISTANCE&
+    VarDesc=Distance&
+    VarType=Num&
+    VarName=RAMP_TO_RAMP&
+    VarDesc=RampToRamp&
+    VarType=Num&
+    VarName=AIR_TIME&
+    VarDesc=AirTime&
+    VarType=Num&
+    VarName=UNIQUE_CARRIER&
+    VarDesc=UniqueCarrier&
+    VarType=Char&
+    VarName=AIRLINE_ID&
+    VarDesc=AirlineID&
+    VarType=Num&
+    VarName=UNIQUE_CARRIER_NAME&
+    VarDesc=UniqueCarrierName&
+    VarType=Char&
+    VarName=UNIQUE_CARRIER_ENTITY&
+    VarDesc=UniqCarrierEntity&
+    VarType=Char&
+    VarName=REGION&
+    VarDesc=CarrierRegion&
+    VarType=Char&
+    VarName=CARRIER&
+    VarDesc=Carrier&
+    VarType=Char&
+    VarName=CARRIER_NAME&
+    VarDesc=CarrierName&
+    VarType=Char&
+    VarName=CARRIER_GROUP&
+    VarDesc=CarrierGroup&
+    VarType=Num&
+    VarName=CARRIER_GROUP_NEW&
+    VarDesc=CarrierGroupNew&
+    VarType=Num&
+    VarName=ORIGIN&
+    VarDesc=Origin&
+    VarType=Char&
+    VarName=ORIGIN_CITY_NAME&
+    VarDesc=OriginCityName&
+    VarType=Char&
+    VarName=ORIGIN_CITY_NUM&
+    VarDesc=OriginCityNum&
+    VarType=Num&
+    VarName=ORIGIN_STATE_ABR&
+    VarDesc=OriginState&
+    VarType=Char&
+    VarName=ORIGIN_STATE_FIPS&
+    VarDesc=OriginStateFips&
+    VarType=Char&
+    VarName=ORIGIN_STATE_NM&
+    VarDesc=OriginStateName&
+    VarType=Char&
+    VarName=ORIGIN_COUNTRY&
+    VarDesc=OriginCountry&
+    VarType=Char&
+    VarName=ORIGIN_COUNTRY_NAME&
+    VarDesc=OriginCountryName&
+    VarType=Char&
+    VarName=ORIGIN_WAC&
+    VarDesc=OriginWac&
+    VarType=Num&
+    VarName=DEST&
+    VarDesc=Dest&
+    VarType=Char&
+    VarName=DEST_CITY_NAME&
+    VarDesc=DestCityName&
+    VarType=Char&
+    VarName=DEST_CITY_NUM&
+    VarDesc=DestCityNum&
+    VarType=Num&
+    VarName=DEST_STATE_ABR&
+    VarDesc=DestState&
+    VarType=Char&
+    VarName=DEST_STATE_FIPS&
+    VarDesc=DestStateFips&
+    VarType=Char&
+    VarName=DEST_STATE_NM&
+    VarDesc=DestStateName&
+    VarType=Char&
+    VarName=DEST_COUNTRY&
+    VarDesc=DestCountry&
+    VarType=Char&
+    VarName=DEST_COUNTRY_NAME&
+    VarDesc=DestCountryName&
+    VarType=Char&
+    VarName=DEST_WAC&
+    VarDesc=DestWac&
+    VarType=Num&
+    VarName=AIRCRAFT_GROUP&
+    VarDesc=AircraftGroup&
+    VarType=Num&
+    VarName=AIRCRAFT_TYPE&
+    VarDesc=AircraftType&
+    VarType=Char&
+    VarName=AIRCRAFT_CONFIG&
+    VarDesc=AircraftConfig&
+    VarType=Num&
+    VarName=YEAR&
+    VarDesc=Year&
+    VarType=Num&
+    VarName=QUARTER&
+    VarDesc=Quarter&
+    VarType=Num&
+    VarName=MONTH&
+    VarDesc=Month&
+    VarType=Num&
+    VarName=DISTANCE_GROUP&
+    VarDesc=DistanceGroup&
+    VarType=Num&
+    VarName=CLASS&
+    VarDesc=Class&
+    VarType=Char&
+    VarName=DATA_SOURCE&
+    VarDesc=DataSource&
+    VarType=Char
+  }.gsub /[\s]+/,''
+
+  data_miner do
+    process "Don't re-import too often" do
+      raise DataMiner::Skip unless DataMiner::Run.allowed? FlightSegment
+    end
+    
+    schema Earth.database_options do
+      string   'row_hash'
+      string   'propulsion_id'
+      integer  'bts_aircraft_group_code'
+      string   'configuration_id'
+      integer  'bts_aircraft_configuration_code'
+      string   'distance_group'
+      integer  'bts_distance_group_code'
+      string   'service_class_id'
+      string   'bts_service_class_code'
+      string   'domesticity_id'
+      string   'bts_data_source_code'
+      integer  'departures_performed'
+      integer  'payload'
+      integer  'total_seats'
+      integer  'passengers'
+      integer  'freight'
+      integer  'mail'
+      integer  'ramp_to_ramp'
+      integer  'air_time'
+      float    'load_factor'
+      float    'freight_share'
+      integer  'distance'
+      integer  'departures_scheduled'
+      string   'airline_iata_code'
+      string   'dot_airline_id_code'
+      string   'unique_carrier_name'
+      string   'unique_carrier_entity'
+      string   'region'
+      string   'current_airline_iata_code'
+      string   'carrier_name'
+      integer  'carrier_group'
+      integer  'carrier_group_new'
+      string   'origin_airport_iata_code'
+      string   'origin_city_name'
+      integer  'origin_city_num'
+      string   'origin_state_abr'
+      string   'origin_state_fips'
+      string   'origin_state_nm'
+      string   'origin_country_iso_3166_code'
+      string   'origin_country_name'
+      integer  'origin_wac'
+      string   'dest_airport_iata_code'
+      string   'dest_city_name'
+      integer  'dest_city_num'
+      string   'dest_state_abr'
+      string   'dest_state_fips'
+      string   'dest_state_nm'
+      string   'dest_country_iso_3166_code'
+      string   'dest_country_name'
+      integer  'dest_wac'
+      integer  'bts_aircraft_type_code'
+      integer  'year'
+      integer  'quarter'
+      integer  'month'
+      float    'seats'
+      string   'payload_units'
+      string   'freight_units'
+      string   'mail_units'
+      string   'distance_units'
+      index    'airline_iata_code'
+      index    'bts_aircraft_type_code'
+      index    'origin_airport_iata_code'
+      index    'dest_airport_iata_code'
+      index    'domesticity_id'
+      # add_index "flight_segments", ["flight_airline_id", "origin_airport_id", "destination_airport_id", "flight_configuration_id", "flight_aircraft_id", "flight_propulsion_id", "flight_service_id", "origin_country_id", "destination_country_id"], :name => "super_4_index"
+    end
+    
+    months = Hash.new
+    (2008..2009).each do |year|
+    # (2008..2008).each do |year| # DEBUG MODE!
+      (1..12).each do |month|
+      # (1..1).each do |month| # DEBUG MODE!
+        time = Time.gm year, month
+        form_data = FORM_DATA.dup
+        form_data.gsub! '__YEAR__', time.year.to_s
+        form_data.gsub! '__MONTH_NUMBER__', time.month.to_s
+        form_data.gsub! '__MONTH_NAME__', time.strftime('%B')
+        months[time] = form_data
+      end
+    end
+    # creating dictionaries by hand so that a new one doesn't get created for every month
+    propulsion_dictionary = DataMiner::Dictionary.new :input => 'Code', :output => 'Description', :url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_AIRCRAFT_GROUP'
+    configuration_dictionary = DataMiner::Dictionary.new :input => 'Code', :output => 'Description', :url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_AIRCRAFT_CONFIG'
+    distance_group_dictionary = DataMiner::Dictionary.new :input => 'Code', :output => 'Description', :url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_DISTANCE_GROUP_500'
+    service_class_dictionary = DataMiner::Dictionary.new :input => 'Code', :output => 'Description', :url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_SERVICE_CLASS'
+    domesticity_dictionary = DataMiner::Dictionary.new :input => 'Code', :output => 'Description', :url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_DATA_SOURCE'
+    months.each do |month, form_data|
+      import "T100 flight segment data from #{month.strftime('%B %Y')}",
+             :url => URL,
+             :form_data => form_data,
+             :compression => :zip,
+             :glob => '/*.csv' do
+        
+        key 'row_hash'
+        
+        store 'propulsion_id', :field_name => 'AIRCRAFT_GROUP', :dictionary => propulsion_dictionary
+        store 'bts_aircraft_group_code', :field_name => 'AIRCRAFT_GROUP'
+        
+        store 'configuration_id', :field_name => 'AIRCRAFT_CONFIG', :dictionary => configuration_dictionary
+        store 'bts_aircraft_configuration_code', :field_name => 'AIRCRAFT_CONFIG'
+        
+        store 'distance_group', :field_name => 'DISTANCE_GROUP', :dictionary => distance_group_dictionary
+        store 'bts_distance_group_code', :field_name => 'DISTANCE_GROUP'
+        
+        store 'service_class_id', :field_name => 'CLASS', :dictionary => service_class_dictionary
+        store 'bts_service_class_code', :field_name => 'CLASS'
+        
+        store 'domesticity_id', :field_name => 'DATA_SOURCE', :dictionary => domesticity_dictionary
+        store 'bts_data_source_code', :field_name => 'DATA_SOURCE'
+        
+        store 'departures_scheduled', :field_name => 'DEPARTURES_SCHEDULED'
+        store 'departures_performed', :field_name => 'DEPARTURES_PERFORMED'
+        store 'payload', :field_name => 'PAYLOAD', :from_units => :pounds, :to_units => :kilograms
+        store 'total_seats', :field_name => 'SEATS'
+        store 'passengers', :field_name => 'PASSENGERS'
+        store 'freight', :field_name => 'FREIGHT', :from_units => :pounds, :to_units => :kilograms
+        store 'mail', :field_name => 'MAIL', :from_units => :pounds, :to_units => :kilograms
+        store 'distance', :field_name => 'DISTANCE', :from_units => :miles, :to_units => :kilometres
+        store 'ramp_to_ramp', :field_name => 'RAMP_TO_RAMP'
+        store 'air_time', :field_name => 'AIR_TIME'
+        store 'airline_iata_code', :field_name => 'UNIQUE_CARRIER' # adjusted for uniqueness
+        store 'dot_airline_id_code', :field_name => 'AIRLINE_ID'
+        store 'unique_carrier_name', :field_name => 'UNIQUE_CARRIER_NAME'
+        store 'unique_carrier_entity', :field_name => 'UNIQUE_CARRIER_ENTITY'
+        store 'region', :field_name => 'REGION'
+        store 'current_airline_iata_code', :field_name => 'CARRIER'
+        store 'carrier_name', :field_name => 'CARRIER_NAME'
+        store 'carrier_group', :field_name => 'CARRIER_GROUP'
+        store 'carrier_group_new', :field_name => 'CARRIER_GROUP_NEW'
+        store 'origin_airport_iata_code', :field_name => 'ORIGIN'
+        store 'origin_city_name', :field_name => 'ORIGIN_CITY_NAME'
+        store 'origin_city_num', :field_name => 'ORIGIN_CITY_NUM'
+        store 'origin_state_abr', :field_name => 'ORIGIN_STATE_ABR'
+        store 'origin_state_fips', :field_name => 'ORIGIN_STATE_FIPS'
+        store 'origin_state_nm', :field_name => 'ORIGIN_STATE_NM'
+        store 'origin_country_iso_3166_code', :field_name => 'ORIGIN_COUNTRY'
+        store 'origin_country_name', :field_name => 'ORIGIN_COUNTRY_NAME'
+        store 'origin_wac', :field_name => 'ORIGIN_WAC'
+        store 'dest_airport_iata_code', :field_name => 'DEST'
+        store 'dest_city_name', :field_name => 'DEST_CITY_NAME'
+        store 'dest_city_num', :field_name => 'DEST_CITY_NUM'
+        store 'dest_state_abr', :field_name => 'DEST_STATE_ABR'
+        store 'dest_state_fips', :field_name => 'DEST_STATE_FIPS'
+        store 'dest_state_nm', :field_name => 'DEST_STATE_NM'
+        store 'dest_country_iso_3166_code', :field_name => 'DEST_COUNTRY'
+        store 'dest_country_name', :field_name => 'DEST_COUNTRY_NAME'
+        store 'dest_wac', :field_name => 'DEST_WAC'
+        store 'bts_aircraft_type_code', :field_name => 'AIRCRAFT_TYPE' # lol no dictionary please
+        store 'year', :field_name => 'YEAR'
+        store 'quarter', :field_name => 'QUARTER'
+        store 'month', :field_name => 'MONTH'
+      end
+    end
+    
+    process "Derive freight share as a fraction of payload" do
+      update_all 'freight_share = (freight + mail) / payload', 'payload > 0'
+    end
+
+    process "Derive load factor, which is passengers divided by the total seats available" do
+      update_all 'load_factor = passengers / total_seats', 'passengers <= total_seats'
+    end
+    
+    process "Derive average seats per departure" do
+      update_all 'seats = total_seats / departures_performed', 'departures_performed > 0'
+    end
+  end
+end
+

data/lib/earth/air/flight_service.rb

@@ -0,0 +1,18 @@
+class FlightService < ActiveRecord::Base
+  set_primary_key :name
+  
+  data_miner do
+    schema do
+      string 'name'
+      string 'bts_service_class_code'
+    end
+    
+    process "derive from flight segments" do
+      FlightSegment.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO flight_services(name, bts_service_class_code)
+        SELECT flight_segments.service_class_id, flight_segments.bts_service_class_code FROM flight_segments WHERE LENGTH(flight_segments.service_class_id) > 0
+      }
+    end
+  end
+end

data/lib/earth/air/flight_service/data_miner.rb

@@ -0,0 +1,4 @@
+FlightService.class_eval do
+#  data_miner do
+#...
+end

data/lib/earth/all.rb

@@ -0,0 +1,10 @@
+require 'earth/air'
+require 'earth/automobile'
+require 'earth/bus'
+require 'earth/diet'
+require 'earth/fuel'
+require 'earth/industry'
+require 'earth/locality'
+require 'earth/pet'
+require 'earth/rail'
+require 'earth/residence'

data/lib/earth/automobile.rb

@@ -0,0 +1,8 @@
+require 'earth/automobile/automobile_fuel_type'
+require 'earth/automobile/automobile_make'
+require 'earth/automobile/automobile_make_fleet_year'
+require 'earth/automobile/automobile_make_year'
+require 'earth/automobile/automobile_model'
+require 'earth/automobile/automobile_model_year'
+require 'earth/automobile/automobile_size_class'
+require 'earth/automobile/automobile_variant'

data/lib/earth/automobile/automobile_fuel_type.rb

@@ -0,0 +1,18 @@
+class AutomobileFuelType < ActiveRecord::Base
+  set_primary_key :code
+  
+  has_many :automobiles, :foreign_key => 'fuel_type_id'
+  
+  scope :ordered, :order => 'name'
+  
+  falls_back_on :emission_factor => 20.781.pounds_per_gallon.to(:kilograms_per_litre) # https://brighterplanet.sifterapp.com/projects/30/issues/428
+  
+  data_miner do
+    tap "Brighter Planet's sanitized automobile fuel type data", Earth.taps_server
+  end
+  
+  CODES = {
+    :electricity => 'El',
+    :diesel => 'D'
+  }
+end

data/lib/earth/automobile/automobile_fuel_type/data_miner.rb

@@ -0,0 +1,45 @@
+AutomobileFuelType.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'code'
+      string   'name'
+      float    'emission_factor'
+      string   'emission_factor_units'
+      float    'annual_distance'
+      string   'annual_distance_units'
+    end
+    
+    import("the fuels used in the 2000 EPA fuel economy guide",
+           :url => 'http://www.fueleconomy.gov/FEG/epadata/00data.zip',
+           :filename => 'Gd6-dsc.txt',
+           :format => :fixed_width,
+           :crop => 21..26, # inclusive
+           :cut => '2-',
+           :select => lambda { |row| /\A[A-Z]/.match row[:code] },
+           :schema => [[ 'code',   2,  { :type => :string }  ],
+                       [ 'spacer', 2 ],
+                       [ 'name',   52, { :type => :string } ]]) do
+      key   'code'
+      store 'name'
+    end
+
+    import "a pre-calculated emission factor and average annual distance for each fuel",
+           :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/models_export/automobile_fuel_type.csv' do
+      key   'code', :field_name => 'code'
+      store 'name'
+      store 'annual_distance', :units => :kilometres
+      store 'emission_factor', :units => :kilograms_per_litre
+    end
+
+    # pull electricity emission factor from residential electricity
+    import "a pre-calculated emission factor for electricity",
+           :url => 'http://spreadsheets.google.com/pub?key=rukxnmuhhsOsrztTrUaFCXQ',
+           :select => lambda { |row| row['name'] == 'electricity' } do
+      key 'name'
+      store 'emission_factor', :units => :kilograms_per_litre
+    end
+    
+    # still need distance estimate for electric cars
+  end
+end
+