earth 0.5.4 → 0.6.0

data/lib/earth/air/aircraft_class/data_miner.rb

@@ -1,56 +1,13 @@
 AircraftClass.class_eval do
   data_miner do
-    process "Ensure Aircraft is populated" do
-      Aircraft.run_data_miner!
-    end
-    
     process "Derive aircraft classes from Aircraft" do
-      connection.select_values("SELECT DISTINCT class_code FROM aircraft WHERE aircraft.class_code IS NOT NULL").each do |class_code|
+      Aircraft.run_data_miner!
+      connection.select_values("SELECT DISTINCT class_code FROM aircraft WHERE class_code IS NOT NULL").each do |class_code|
         AircraftClass.find_or_create_by_code(class_code)
       end
     end
     
-    process "Derive some average characteristics from Aircraft" do
-      AircraftClass.find_each do |aircraft_class|
-        %w{ m3 m2 m1 b }.each do |coefficient|
-          value = AircraftFuelUseEquation.where("aircraft.class_code = '#{aircraft_class.code}'").
-            weighted_average(:"#{coefficient}", :weighted_by => [:aircraft, :passengers])
-          aircraft_class.send("#{coefficient}=", value)
-        end
-#           # do this in sql because we want to preserve nils and weighted_average returns 0 when it gets NULL
-#           connection.execute %{
-# UPDATE aircraft_classes
-# SET aircraft_classes.#{coefficient} = (
-#   SELECT sum(aircraft_fuel_use_equations.#{coefficient} * aircraft.passengers) / sum(aircraft.passengers)
-#   FROM aircraft_fuel_use_equations
-#   INNER JOIN aircraft
-#   ON aircraft.fuel_use_code = aircraft_fuel_use_equations.code
-#   WHERE aircraft.class_code = '#{aircraft_class.code}'
-#   AND aircraft_fuel_use_equations.#{coefficient} IS NOT NULL
-# )
-# WHERE aircraft_classes.code = '#{aircraft_class.code}'
-#           }
-#        end
-        
-        aircraft_class.seats = aircraft_class.aircraft.weighted_average(:seats, :weighted_by => :passengers)
-#         # do this in sql because we want to preserve nils and weighted_average returns 0 when it gets NULL
-#         connection.execute %{
-# UPDATE aircraft_classes
-# SET aircraft_classes.seats = (
-#   SELECT sum(aircraft.seats * aircraft.passengers) / sum(aircraft.passengers)
-#   FROM aircraft
-#   WHERE aircraft.class_code = '#{aircraft_class.code}'
-# )
-# WHERE aircraft_classes.code = '#{aircraft_class.code}'
-#         }
-        
-        aircraft_class.m3_units = 'kilograms_per_cubic_nautical_mile'
-        aircraft_class.m2_units = 'kilograms_per_square_nautical_mile'
-        aircraft_class.m1_units = 'kilograms_per_nautical_mile'
-        aircraft_class.b_units  = 'kilograms'
-        aircraft_class.save
-      end
-    end
+    process :update_averages!
     
     # FIXME TODO verify this
   end

data/lib/earth/air/aircraft_fuel_use_equation.rb

@@ -20,16 +20,14 @@ class AircraftFuelUseEquation < ActiveRecord::Base
     fuel_use_coefficients.all?(&:present?) and fuel_use_coefficients.any?(&:nonzero?)
   end
 
-  force_schema do
-    string 'code'
-    string 'aircraft_description'
-    float  'm3'
-    string 'm3_units'
-    float  'm2'
-    string 'm2_units'
-    float  'm1'
-    string 'm1_units'
-    float  'b'
-    string 'b_units'
-  end
-end
+  col :code
+  col :aircraft_description
+  col :m3, :type => :float
+  col :m3_units
+  col :m2, :type => :float
+  col :m2_units
+  col :m1, :type => :float
+  col :m1_units
+  col :b, :type => :float
+  col :b_units
+end

data/lib/earth/air/aircraft_instance.rb

@@ -0,0 +1,9 @@
+class AircraftInstance < ActiveRecord::Base
+  set_primary_key :id
+  
+  col :id
+  col :registration
+  col :serial_number
+  col :aircraft_description
+  col :airline_name
+end

data/lib/earth/air/aircraft_instance_seat_class.rb

@@ -0,0 +1,12 @@
+class AircraftInstanceSeatClass < ActiveRecord::Base
+  set_primary_key :row_hash
+  
+  col :row_hash
+  col :seat_class_name
+  col :aircraft_instance_id
+  col :seats, :type => :integer
+  col :seat_pitch, :type => :float
+  col :seat_pitch_units
+  col :seat_width, :type => :float
+  col :seat_width_units
+end

data/lib/earth/air/airline.rb

@@ -1,10 +1,8 @@
 class Airline < ActiveRecord::Base
   set_primary_key :name
-
-  force_schema do
-    string 'name'
-    string 'bts_code'
-    string 'iata_code'
-    string 'icao_code'
-  end
-end
+  
+  col :name
+  col :bts_code
+  col :iata_code
+  col :icao_code
+end

data/lib/earth/air/airline/data_miner.rb

@@ -1,10 +1,10 @@
 Airline.class_eval do
   data_miner do
-    import "a Brighter Planet-curated list of airlines",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdDhoVHZmSTlZcHBsRUtPR0dPd0prMkE&output=csv' do
+    import "a Brighter Planet-curated list of airlines and codes not included in our other sources",
+           :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGJoaFpENXRqMEM2NW42am5tNURGU2c&output=csv' do
       key 'name'
       store 'bts_code',  :nullify => true
-      store 'iata_code',  :nullify => true
+      store 'iata_code', :nullify => true
       store 'icao_code', :nullify => true
     end
   end

data/lib/earth/air/airport.rb

@@ -7,13 +7,11 @@ class Airport < ActiveRecord::Base
                    :lat_column_name => :latitude,
                    :lng_column_name => :longitude
 
-  force_schema do
-    string   'iata_code'
-    string   'name'
-    string   'city'
-    string   'country_name'
-    string   'country_iso_3166_code'
-    float    'latitude'
-    float    'longitude'
-  end
-end
+  col :iata_code
+  col :name
+  col :city
+  col :country_name
+  col :country_iso_3166_code
+  col :latitude, :type => :float
+  col :longitude, :type => :float
+end

data/lib/earth/air/airport/data_miner.rb

@@ -46,7 +46,7 @@ Airport.class_eval do
     process "Fill in blank country codes" do
       Country.find_each do |country|
         next unless country.name.present? and country.iso_3166_code.present?
-        update_all %{country_iso_3166_code = "#{country.iso_3166_code}"}, %{country_name LIKE "#{country.name}"}
+        update_all({ :country_iso_3166_code => country.iso_3166_code }, [ "country_name LIKE ?", country.name ])
       end
     end
     

data/lib/earth/air/bts_aircraft.rb

@@ -1,7 +1,5 @@
 class BtsAircraft < ActiveRecord::Base
   set_primary_key :bts_code
-  force_schema do
-    string 'bts_code'
-    string 'description'
-  end
-end
+  col :bts_code
+  col :description
+end

data/lib/earth/air/flight_distance_class.rb

@@ -1,8 +1,6 @@
 class FlightDistanceClass < ActiveRecord::Base
   set_primary_key :name
-  force_schema do
-    string 'name'
-    float  'distance'
-    string 'distance_units'
-  end
-end
+  col :name
+  col :distance, :type => :float
+  col :distance_units
+end

data/lib/earth/air/flight_seat_class.rb

@@ -3,10 +3,8 @@ class FlightSeatClass < ActiveRecord::Base
   
   falls_back_on :multiplier => 1
 
-  force_schema do
-    string   'name'
-    string   'distance_class_name'
-    string   'seat_class_name'
-    float    'multiplier'
-  end
-end
+  col :name
+  col :distance_class_name
+  col :seat_class_name
+  col :multiplier, :type => :float
+end

data/lib/earth/air/flight_segment.rb

@@ -19,46 +19,62 @@ class FlightSegment < ActiveRecord::Base
                 :load_factor      => lambda { weighted_average(:load_factor,      :weighted_by => :passengers) }, # 0.78073233770097  data1 10-12-2010
                 :freight_share    => lambda { weighted_average(:freight_share,    :weighted_by => :passengers) }  # 0.022567224170157 data1 10-12-2010
   
-  force_schema do
-    string  'row_hash'                          # auto-generated primary key
-    string  'origin_airport_iata_code'          # iata code
-    string  'origin_airport_city'               # city
-    string  'origin_country_iso_3166_code'      # iso code
-    string  'destination_airport_iata_code'     # iata code
-    string  'destination_airport_city'          # city
-    string  'destination_country_iso_3166_code' # iso code
-    string  'airline_bts_code'                  # bts code
-    string  'airline_icao_code'                 # icao code
-    string  'airline_name'                      # text description derived from bts or icao code
-    string  'aircraft_bts_code'                 # bts code
-    string  'aircraft_description'              # text description derived from BTS T100 or ICAO TFS
-    integer 'flights'                           # number of flights over month or year
-    integer 'passengers'                        # total passengers on all flights
-    integer 'seats'                             # total seats on all flights
-    float   'seats_per_flight'                  # average seats per flight; make this a float
-    float   'load_factor'                       # passengers / seats
-    float   'freight_share'                     # (freight + mail) / (freight + mail + (passengers * average passenger weight))
-    float   'distance'                          # flight distance
-    string  'distance_units'                    # 'kilometres'
-    float   'payload_capacity'                  # aircraft maximum payload capacity rating; float b/c unit conversion
-    string  'payload_capacity_units'            # 'kilograms'
-    float   'freight'                           # total freight on all flights performed; float b/c unit conversion
-    string  'freight_units'                     # 'kilograms'
-    float   'mail'                              # total mail on all flights performed; float b/c unit conversion
-    string  'mail_units'                        # 'kilograms'
-    integer 'month'                             # month of flight
-    integer 'year'                              # year of flight
-    date    'approximate_date'                  # assumed 14th day of month
-    string  'source'                            # 'BTS T100' or 'ICAO TFS'
-    index   'origin_airport_iata_code'          # index for faster lookup by origin airport
-    index   'origin_airport_city'               # index for faster lookup by origin city
-    index   'destination_airport_iata_code'     # index for faster lookup by destination airport
-    index   'destination_airport_city'          # index for faster lookup by destination city
-    index   'airline_bts_code'                  # index for faster lookup by airline bts code
-    index   'airline_icao_code'                 # index for faster lookup by airline icao code
-    index   'airline_name'                      # index for faster lookup by airline name
-    index   'aircraft_bts_code'                 # index for faster lookup by aircraft bts code
-    index   'aircraft_description'              # index for faster lookup by aircraft
-    index   'year'                              # index for faster lookup by year
+  col :row_hash                          # auto-generated primary key
+  col :origin_airport_iata_code          # iata code
+  col :origin_airport_city               # city
+  col :origin_country_iso_3166_code      # iso code
+  col :destination_airport_iata_code     # iata code
+  col :destination_airport_city          # city
+  col :destination_country_iso_3166_code # iso code
+  col :airline_bts_code                  # bts code
+  col :airline_icao_code                 # icao code
+  col :airline_name                      # text description derived from bts or icao code
+  col :aircraft_bts_code                 # bts code
+  col :aircraft_description              # text description derived from BTS T100 or ICAO TFS
+  col :flights, :type => :integer                           # number of flights over month or year
+  col :passengers, :type => :integer                        # total passengers on all flights
+  col :seats, :type => :integer                             # total seats on all flights
+  col :seats_per_flight, :type => :float                  # average seats per flight; make this a float
+  col :load_factor, :type => :float                       # passengers / seats
+  col :freight_share, :type => :float                     # (freight + mail) / (freight + mail + (passengers * average passenger weight))
+  col :distance, :type => :float                          # flight distance
+  col :distance_units                    # 'kilometres'
+  col :payload_capacity, :type => :float                  # aircraft maximum payload capacity rating; float b/c unit conversion
+  col :payload_capacity_units            # 'kilograms'
+  col :freight, :type => :float                           # total freight on all flights performed; float b/c unit conversion
+  col :freight_units                     # 'kilograms'
+  col :mail, :type => :float                              # total mail on all flights performed; float b/c unit conversion
+  col :mail_units                        # 'kilograms'
+  col :month, :type => :integer                             # month of flight
+  col :year, :type => :integer                              # year of flight
+  col :approximate_date, :type => :date                  # assumed 14th day of month
+  col :source                            # 'BTS T100' or 'ICAO TFS'
+  add_index :origin_airport_iata_code
+  add_index :origin_airport_city
+  add_index :destination_airport_iata_code
+  add_index :destination_airport_city
+  add_index :airline_bts_code
+  add_index :airline_icao_code
+  add_index :airline_name
+  add_index :aircraft_bts_code
+  add_index :aircraft_description
+  add_index :year
+  
+  def self.update_averages!
+    # Derive load factor, which is passengers divided by available seats
+    update_all 'load_factor = passengers / seats', 'seats > 0'
+
+    # Assume a load factor of 1 where passengers > available seats
+    update_all 'load_factor = 1', 'passengers > seats AND seats > 0'
+  
+    # TODO: what is 90.718474
+    # Derive freight share as a fraction of the total weight carried
+    update_all 'freight_share = (freight + mail) / (freight + mail + (passengers * 90.718474))', '(freight + mail + passengers) > 0'
+  
+    # Derive average seats per flight
+    update_all 'seats_per_flight = seats / flights', 'flights > 0'
+    
+    # Add a useful date field
+    update_all 'approximate_date = DATE(year || "-" || month || "-" || "14")', 'month IS NOT NULL'
   end
-end
+end

data/lib/earth/air/flight_segment/data_miner.rb

@@ -10,24 +10,6 @@ FlightSegment.class_eval do
     end
   end
   
-  def self.update_averages!
-    # Derive load factor, which is passengers divided by available seats
-    update_all 'load_factor = passengers / seats', 'seats > 0'
-
-    # Assume a load factor of 1 where passengers > available seats
-    update_all 'load_factor = 1', 'passengers > seats AND seats > 0'
-  
-    # TODO: what is 90.718474
-    # Derive freight share as a fraction of the total weight carried
-    update_all 'freight_share = (freight + mail) / (freight + mail + (passengers * 90.718474))', '(freight + mail + passengers) > 0'
-  
-    # Derive average seats per flight
-    update_all 'seats_per_flight = seats / flights', 'flights > 0'
-    
-    # Add a useful date field
-    update_all 'approximate_date = DATE(CONCAT_WS("-", year, month, "14"))', 'month IS NOT NULL'
-  end
-  
   URL = 'http://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=293&Has_Group=3&Is_Zipped=0'
   FORM_DATA = %{
     UserTableName=T_100_Segment__All_Carriers&
@@ -187,11 +169,11 @@ FlightSegment.class_eval do
     VarType=Char
   }.gsub /[\s]+/,''
   
-  data_miner do
-    months = Hash.new
-    (2009..2011).each do |year|
+  def self.form_data_per_month(year_range)
+    months = {}
+    year_range.each do |year|
       (1..12).each do |month|
-        time = Time.gm year, month
+        time = ::Time.gm year, month
         form_data = FORM_DATA.dup
         form_data.gsub! '__YEAR__', time.year.to_s
         form_data.gsub! '__MONTH_NUMBER__', time.month.to_s
@@ -199,8 +181,11 @@ FlightSegment.class_eval do
         months[time] = form_data
       end
     end
-    
-    months.each do |month, form_data|
+    months
+  end
+  
+  data_miner do
+    form_data_per_month(2009..2011).each do |month, form_data|
       import "T100 flight segment data for #{month.strftime('%B %Y')}",
              :url => URL,
              :form_data => form_data,
@@ -237,7 +222,7 @@ FlightSegment.class_eval do
     process "Look up airline name based on BTS code" do
       connection.select_values("SELECT DISTINCT airline_bts_code FROM flight_segments WHERE airline_bts_code IS NOT NULL").each do |bts_code|
         if airline = Airline.find_by_bts_code(bts_code)
-          update_all %{ airline_name = "#{airline.name}" }, %{ airline_bts_code = "#{bts_code}" }
+          update_all({ :airline_name => airline.name }, :airline_bts_code => bts_code)
         end
       end
     end
@@ -245,7 +230,7 @@ FlightSegment.class_eval do
     process "Look up aircraft description based on BTS code" do
       connection.select_values("SELECT DISTINCT aircraft_bts_code FROM flight_segments WHERE aircraft_bts_code IS NOT NULL").each do |bts_code|
         if aircraft = BtsAircraft.find_by_bts_code(bts_code)
-          update_all %{ aircraft_description = "#{aircraft.description.downcase}" }, %{ aircraft_bts_code = "#{bts_code}" }
+          update_all({ :aircraft_description => aircraft.description.downcase }, :aircraft_bts_code => bts_code)
         end
       end
     end

data/lib/earth/automobile.rb

@@ -13,3 +13,4 @@ require 'earth/automobile/automobile_type_fuel_control'
 require 'earth/automobile/automobile_type_fuel_year'
 require 'earth/automobile/automobile_type_fuel_year_age'
 require 'earth/automobile/automobile_type_fuel_year_control'
+require 'earth/fuel'

data/lib/earth/automobile/automobile_fuel.rb

@@ -1,7 +1,6 @@
 require 'earth/automobile/automobile_type_fuel_year_age'
 require 'earth/automobile/automobile_type_fuel_year'
 require 'earth/automobile/automobile_type_year'
-require 'earth/fuel/greenhouse_gas'
 
 class AutomobileFuel < ActiveRecord::Base
   set_primary_key :name
@@ -10,6 +9,62 @@ class AutomobileFuel < ActiveRecord::Base
   has_many :type_fuel_years,     :class_name => 'AutomobileTypeFuelYear',    :foreign_key => 'fuel_common_name', :primary_key => 'ef_key'
   belongs_to :base_fuel,         :class_name => 'Fuel',                      :foreign_key => 'base_fuel_name'
   belongs_to :blend_fuel,        :class_name => 'Fuel',                      :foreign_key => 'blend_fuel_name'
+    
+  warn_if_blanks_in :distance_key
+  warn_if_blanks_in :ef_key
+  warn do
+    catch :culprit do
+      find_each do |record|
+        throw :culprit, %{Records exist without base_fuel (possibly invalid key "#{record.base_fuel_name}")} unless record.base_fuel
+      end
+      false
+    end
+  end
+  warn do
+    if exists?(['blend_portion IS NOT NULL AND (blend_portion < ? OR blend_portion > ?)', 0, 1])
+      "Blend portions less than 0 or greater than 1"
+    end
+  end
+  warn do
+    %w{co2_emission_factor co2_biogenic_emission_factor}.map do |col|
+      if exists?(["#{col} IS NULL OR #{col} < ?", 0])
+        "Records non-positive #{col}"
+      end
+    end
+  end
+  
+  # FIXME TODO verify that base_fuel_name and blend_fuel_name are found in Fuel if present
+  # FIXME TODO verify that distance_key is found in AutomobileTypeFuelYearAge
+  # FIXME TODO verify that ef_key is found in AutomobileTypeFuelYear
+  
+  # TODO convert these to warn blocks  
+  # ["ch4_emission_factor", "n2o_emission_factor", "hfc_emission_factor"].each do |attribute|
+  #   verify "#{attribute.humanize} should be > 0" do
+  #     find_each do |fuel|
+  #       value = fuel.send(attribute)
+  #       unless value > 0
+  #         raise "Invalid #{attribute.humanize.downcase} for AutomobileFuel #{fuel.name}: #{value} (should be > 0)"
+  #       end
+  #     end
+  #   end
+  # end
+  # 
+  # [["co2_emission_factor_units", "kilograms_per_litre"],
+  #  ["co2_biogenic_emission_factor_units", "kilograms_per_litre"],
+  #  ["ch4_emission_factor_units", "kilograms_co2e_per_litre"],
+  #  ["n2o_emission_factor_units", "kilograms_co2e_per_litre"],
+  #  ["hfc_emission_factor_units", "kilograms_co2e_per_litre"]].each do |pair|
+  #   attribute = pair[0]
+  #   proper_units = pair[1]
+  #   verify "#{attribute.humanize} should be #{proper_units.humanize.downcase}" do
+  #     find_each do |fuel|
+  #       units = fuel.send(attribute)
+  #       unless units == proper_units
+  #         raise "Invalid #{attribute.humanize.downcase} for AutomobileFuel #{fuel.name}: #{units} (should be #{proper_units})"
+  #       end
+  #     end
+  #   end
+  # end
   
   class << self
     def fallback_latest_type_fuel_year_ages
@@ -98,32 +153,30 @@ class AutomobileFuel < ActiveRecord::Base
                 :hfc_emission_factor => lambda { AutomobileFuel.fallback_hfc_emission_factor },
                 :hfc_emission_factor_units => lambda { AutomobileFuel.fallback_hfc_emission_factor_units }
 
-  force_schema do
-    string 'name'
-    string 'code'
-    string 'base_fuel_name'
-    string 'blend_fuel_name'
-    float  'blend_portion' # the portion of the blend that is the blend fuel
-    string 'distance_key' # used to look up annual distance from AutomobileTypeFuelYear
-    string 'ef_key' # used to look up ch4 n2o and hfc emission factors from AutomobileTypeFuelYear
-    float  'annual_distance'
-    string 'annual_distance_units'
-    float  'co2_emission_factor'
-    string 'co2_emission_factor_units'
-    float  'co2_biogenic_emission_factor'
-    string 'co2_biogenic_emission_factor_units'
-    float  'ch4_emission_factor'
-    string 'ch4_emission_factor_units'
-    float  'n2o_emission_factor'
-    string 'n2o_emission_factor_units'
-    float  'hfc_emission_factor'
-    string 'hfc_emission_factor_units'
-    float  'emission_factor' # DEPRECATED but motorcycle needs this
-    string 'emission_factor_units' # DEPRECATED but motorcycle needs this
-  end
+  col :name
+  col :code
+  col :base_fuel_name
+  col :blend_fuel_name
+  col :blend_portion, :type => :float # the portion of the blend that is the blend fuel
+  col :distance_key # used to look up annual distance from AutomobileTypeFuelYear
+  col :ef_key # used to look up ch4 n2o and hfc emission factors from AutomobileTypeFuelYear
+  col :annual_distance, :type => :float
+  col :annual_distance_units
+  col :co2_emission_factor, :type => :float
+  col :co2_emission_factor_units
+  col :co2_biogenic_emission_factor, :type => :float
+  col :co2_biogenic_emission_factor_units
+  col :ch4_emission_factor, :type => :float
+  col :ch4_emission_factor_units
+  col :n2o_emission_factor, :type => :float
+  col :n2o_emission_factor_units
+  col :hfc_emission_factor, :type => :float
+  col :hfc_emission_factor_units
+  col :emission_factor, :type => :float # DEPRECATED but motorcycle needs this
+  col :emission_factor_units # DEPRECATED but motorcycle needs this
 
   CODES = {
     :electricity => 'El',
     :diesel => 'D'
   }
-end
+end