earth 0.6.6 → 0.7.0

data/lib/earth/rail/data_miner.rb

@@ -1 +1,10 @@
 require 'earth/rail/rail_class/data_miner'
+require 'earth/rail/rail_company/data_miner'
+require 'earth/rail/rail_company_traction/data_miner'
+require 'earth/rail/rail_company_traction_class/data_miner'
+require 'earth/rail/rail_class/data_miner'
+require 'earth/rail/rail_fuel/data_miner'
+require 'earth/rail/rail_traction/data_miner'
+require 'earth/rail/national_transit_database_company/data_miner'
+require 'earth/rail/national_transit_database_mode/data_miner'
+require 'earth/rail/national_transit_database_record/data_miner'

data/lib/earth/rail/national_transit_database_company.rb

@@ -0,0 +1,37 @@
+class NationalTransitDatabaseCompany < ActiveRecord::Base
+  set_primary_key :id
+  set_table_name :ntd_companies
+  
+  has_many :ntd_records, :foreign_key => 'company_id', :primary_key => 'id', :class_name => 'NationalTransitDatabaseRecord'
+  
+  def self.rail_companies
+    NationalTransitDatabaseRecord.rail_records.map(&:ntd_company).uniq
+  end
+  
+  def rail_records
+    ntd_records.where(:mode_code => NationalTransitDatabaseMode.rail_modes)
+  end
+  
+  # Methods to calculate summary statistics from rail_records
+  [:rail_passengers, :rail_passenger_distance, :rail_vehicle_distance, :rail_vehicle_time, :rail_electricity, :rail_diesel].each do |method|
+    define_method method do
+      attribute = method.to_s.split('rail_')[1].to_sym
+      rail_records.sum(attribute) > 0 ? rail_records.sum(attribute) : nil 
+    end
+  end
+  
+  # Methods to look up units from from rail_records
+  [:rail_passenger_distance_units, :rail_vehicle_distance_units, :rail_vehicle_time_units, :rail_electricity_units, :rail_diesel_units].each do |method|
+    define_method method do
+      attribute = method.to_s.split('rail_')[1].to_sym
+      units = rail_records.map(&attribute).uniq
+      (units.count == 1 and units[0].present?) ? units[0] : raise("Error: units missing or multiple units in #{name}'s NTD records")
+    end
+  end
+  
+  col :id
+  col :name
+  col :acronym
+  col :zip_code_name
+  col :duns_number
+end

data/lib/earth/rail/national_transit_database_company/data_miner.rb

@@ -0,0 +1,12 @@
+NationalTransitDatabaseCompany.class_eval do
+  data_miner do
+    import "US transit companies from the National Transit Database",
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdHBTZC1GUzg5VEU4QW9NOUo3V1BJWUE&output=csv' do
+      key 'id', :field_name => 'trs_id'
+      store 'name',          :nullify => true
+      store 'acronym',       :nullify => true
+      store 'zip_code_name', :nullify => true
+      store 'duns_number',   :nullify => true
+    end
+  end
+end

data/lib/earth/rail/national_transit_database_mode.rb

@@ -0,0 +1,12 @@
+class NationalTransitDatabaseMode < ActiveRecord::Base
+  set_primary_key :code
+  set_table_name :ntd_modes
+  
+  def self.rail_modes
+    where(:rail_mode => true)
+  end
+  
+  col :code
+  col :name
+  col :rail_mode, :type => :boolean
+end

data/lib/earth/rail/national_transit_database_mode/data_miner.rb

@@ -0,0 +1,10 @@
+NationalTransitDatabaseMode.class_eval do
+  data_miner do
+    import "National Transit Database modes",
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGFTLVpMdFgwSTQzTTFuc2lIVUdfSHc&output=csv' do
+      key 'code'
+      store 'name',      :nullify => true
+      store 'rail_mode', :nullify => true
+    end
+  end
+end

data/lib/earth/rail/national_transit_database_record.rb

@@ -0,0 +1,41 @@
+class NationalTransitDatabaseRecord < ActiveRecord::Base
+  set_primary_key :name
+  set_table_name :ntd_records
+  
+  belongs_to :ntd_company, :foreign_key => 'company_id', :class_name => 'NationalTransitDatabaseCompany'
+  
+  def self.rail_records
+    where(:mode_code => NationalTransitDatabaseMode.rail_modes)
+  end
+  
+  col :name
+  col :company_id
+  col :mode_code
+  col :service_type
+  col :vehicle_distance, :type => :float
+  col :vehicle_distance_units
+  col :vehicle_time, :type => :float
+  col :vehicle_time_units
+  col :passenger_distance, :type => :float
+  col :passenger_distance_units
+  col :passengers, :type => :float
+  col :electricity, :type => :float
+  col :electricity_units
+  col :diesel, :type => :float
+  col :diesel_units
+  col :gasoline, :type => :float
+  col :gasoline_units
+  col :lpg, :type => :float
+  col :lpg_units
+  col :lng, :type => :float
+  col :lng_units
+  col :cng, :type => :float
+  col :cng_units
+  col :kerosene, :type => :float
+  col :kerosene_units
+  col :biodiesel, :type => :float
+  col :biodiesel_units
+  col :other_fuel, :type => :float
+  col :other_fuel_units
+  col :other_fuel_description
+end

data/lib/earth/rail/national_transit_database_record/data_miner.rb

@@ -0,0 +1,57 @@
+NationalTransitDatabaseRecord.class_eval do
+  data_miner do
+    import "US National Transit Database service data",
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdF9Hbktvc3BMNDRWZHhDRU9uMVZfWVE&output=csv' do
+      key 'name', :synthesize => lambda { |row| [row['trs_id'], row['mode_code'], row['service_code']].join(' ') }
+      store 'company_id', :field_name => 'trs_id'
+      store 'mode_code'
+      store 'service_type',       :field_name => 'service_code', :dictionary => { :input => 'code', :output => 'name', :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDBTZENxb3A1SzJaNTVEVGZ5cTdaMVE&output=csv' }
+      store 'vehicle_distance',   :field_name => 'vehicle_or_train_revenue_miles', :units => 'miles'
+      store 'vehicle_time',       :field_name => 'vehicle_or_train_revenue_hours', :units => 'hours'
+      store 'passenger_distance', :field_name => 'passenger_miles',                :units => 'miles'
+      store 'passengers'
+    end
+    
+    import "US National Transit Database fuel consumption data",
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDItVVR5NjA2Y3FCVjgza25Ccy0zS2c&output=csv' do
+      key 'name', :synthesize => lambda { |row| [row['trs_id'], row['mode_code'], row['service_code']].join(' ') }
+      store 'company_id', :field_name => 'trs_id'
+      store 'mode_code'
+      store 'service_type', :field_name => 'service_code', :dictionary => { :input => 'code', :output => 'name', :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDBTZENxb3A1SzJaNTVEVGZ5cTdaMVE&output=csv' }
+      store 'electricity',  :synthesize => lambda { |row| row['electricity_kwh'].to_f + row['battery_kwh'].to_f }, :units => 'kilowatt_hours', :nullify => true
+      store 'diesel',       :field_name => 'diesel_gallons',    :units => 'gallons', :nullify => true
+      store 'gasoline',     :field_name => 'gasoline_gallons',  :units => 'gallons', :nullify => true
+      store 'lpg',          :field_name => 'lpg_gallons',       :units => 'gallons', :nullify => true
+      store 'lng',          :field_name => 'lng_gallons',       :units => 'gallons', :nullify => true
+      store 'cng',          :field_name => 'cng_gallons',       :units => 'gallons', :nullify => true
+      store 'kerosene',     :field_name => 'kerosene_gallons',  :units => 'gallons', :nullify => true
+      store 'biodiesel',    :field_name => 'biodiesel_gallons', :units => 'gallons', :nullify => true
+      store 'other_fuel',                                                            :nullify => true
+      store 'other_fuel_description', :field_name => 'other_fuel_description',       :nullify => true
+    end
+    
+    process "Convert miles to kilometres" do
+      conversion_factor = 1.miles.to(:kilometres)
+      %{ vehicle_distance passenger_distance }.split.each do |field|
+        connection.execute %{
+          UPDATE #{table_name}
+          SET #{field} = #{field} * #{conversion_factor},
+              #{field}_units = 'kilometres'
+          WHERE #{field}_units = 'miles'
+        }
+      end
+    end
+      
+    process "Convert gallons to litres" do
+      conversion_factor = 1.gallons.to(:litres)
+      %{ diesel gasoline lpg lng cng kerosene biodiesel }.split.each do |fuel|
+        connection.execute %{
+          UPDATE #{table_name}
+          SET #{fuel} = #{fuel} * #{conversion_factor},
+              #{fuel}_units = 'litres'
+          WHERE #{fuel}_units = 'gallons'
+        }
+      end
+    end
+  end
+end

data/lib/earth/rail/rail_class.rb

@@ -1,66 +1,5 @@
 class RailClass < ActiveRecord::Base
   set_primary_key :name
-  col :name
-  col :description
-  col :passengers, :type => :float
-  col :distance, :type => :float
-  col :distance_units
-  col :speed, :type => :float
-  col :speed_units
-  col :electricity_intensity, :type => :float
-  col :electricity_intensity_units
-  col :diesel_intensity, :type => :float
-  col :diesel_intensity_units
-  
-  # verify "Passengers, distance, speed, and electricity intensity should be greater than zero" do
-  #   RailClass.all.each do |rail_class|
-  #     %w{ passengers distance speed electricity_intensity }.each do |attribute|
-  #       value = rail_class.send(:"#{attribute}")
-  #       unless value > 0
-  #         raise "Invalid #{attribute} for RailClass #{rail_class.name}: #{value} (should be > 0)"
-  #       end
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Distance units should be kilometres" do
-  #   RailClass.all.each do |rail_class|
-  #     unless rail_class.distance_units == "kilometres"
-  #       raise "Invalid distance units for RailClass #{rail_class.name}: #{rail_class.distance_units} (should be kilometres)"
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Speed units should be kilometres per hour" do
-  #   RailClass.all.each do |rail_class|
-  #     unless rail_class.speed_units == "kilometres_per_hour"
-  #       raise "Invalid speed units for RailClass #{rail_class.name}: #{rail_class.speed_units} (should be kilometres_per_hour)"
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Electricity intensity units should be kilowatt hours per kilometre" do
-  #   RailClass.all.each do |rail_class|
-  #     unless rail_class.electricity_intensity_units == "kilowatt_hours_per_kilometre"
-  #       raise "Invalid electricity intensity units for RailClass #{rail_class.name}: #{rail_class.electricity_intensity_units} (should be kilowatt_hours_per_kilometre)"
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Diesel intensity should be zero or more" do
-  #   RailClass.all.each do |rail_class|
-  #     unless rail_class.diesel_intensity >= 0
-  #       raise "Invalid diesel intensity for RailClass #{rail_class.name}: #{rail_class.diesel_intensity} (should be > 0)"
-  #     end
-  #   end
-  # end
-  # 
-  # verify "Diesel intensity units should be litres per kilometre" do
-  #   RailClass.all.each do |rail_class|
-  #     unless rail_class.diesel_intensity_units == "litres_per_kilometre"
-  #       raise "Invalid distance units for RailClass #{rail_class.name}: #{rail_class.diesel_intensity_units} (should be litres_per_kilometre)"
-  #     end
-  #   end
-  # end
   
-end
+  col :name
+end

data/lib/earth/rail/rail_class/data_miner.rb

@@ -1,18 +1,19 @@
 RailClass.class_eval do
   data_miner do
-    process "Define some unit conversions" do
-      Conversions.register :gallons_per_mile, :litres_per_kilometre, 2.35214583
+    process "Ensure CountryRailClass, CountryRailTractionClass, and RailCompanyTractionClass are populated" do
+      CountryRailTractionClass.run_data_miner!
+      CountryRailClass.run_data_miner!
+      RailCompanyTractionClass.run_data_miner!
     end
-
-    import "a list of rail classes and pre-calculated trip and fuel use characteristics",
-           :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHRYaDNvRXMtZGoxNmRiOHlNWGV6b2c&hl=en&gid=0&output=csv' do
-      key   'name'
-      store 'description'
-      store 'passengers'
-      store 'distance', :units_field_name => 'distance_units'
-      store 'speed', :units_field_name => 'speed_units'
-      store 'electricity_intensity', :units_field_name => 'electricity_intensity_units'
-      store 'diesel_intensity', :units_field_name => 'diesel_intensity_units'
+    
+    process "Derive rail class names from CountryRailClass, CountryRailTractionClass, and RailCompanyTractionClass" do
+      names = []
+      names += CountryRailTractionClass.select("DISTINCT rail_class_name").map(&:rail_class_name)
+      names += CountryRailClass.select("DISTINCT rail_class_name").map(&:rail_class_name)
+      names += RailCompanyTractionClass.select("DISTINCT rail_class_name").map(&:rail_class_name)
+      names.uniq.each do |name|
+        RailClass.find_or_create_by_name(name)
+      end
     end
   end
 end

data/lib/earth/rail/rail_company.rb

@@ -0,0 +1,26 @@
+class RailCompany < ActiveRecord::Base
+  set_primary_key :name
+  
+  belongs_to :country, :foreign_key => 'country_iso_3166_code'
+  
+  col :name
+  col :country_iso_3166_code
+  col :duns_number
+  col :passengers, :type => :float
+  col :passenger_distance, :type => :float
+  col :passenger_distance_units
+  col :trip_distance, :type => :float
+  col :trip_distance_units
+  col :train_distance, :type => :float
+  col :train_distance_units
+  col :train_time, :type => :float
+  col :train_time_units
+  col :speed, :type => :float
+  col :speed_units
+  col :electricity_intensity, :type => :float
+  col :electricity_intensity_units
+  col :diesel_intensity, :type => :float
+  col :diesel_intensity_units
+  col :co2_emission_factor, :type => :float
+  col :co2_emission_factor_units
+end

data/lib/earth/rail/rail_company/data_miner.rb

@@ -0,0 +1,98 @@
+RailCompany.class_eval do
+  data_miner do
+    import "european rail company data from the UIC",
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDRpVkd3T0p1RDY4RzUzc0VUUVEwYmc&output=csv' do
+      key   'name'
+      store 'country_iso_3166_code'
+      store 'passengers',         :nullify => true
+      store 'passenger_distance', :nullify => true, :units_field_name => 'passenger_distance_units'
+      store 'train_distance',     :nullify => true, :units_field_name => 'train_distance_units'
+    end
+    
+    import "Amtrak data",
+           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGZ3UDdJT1NETUtELWVTLUR4R3RZbEE&output=csv' do
+      key   'name'
+      store 'country_iso_3166_code'
+      store 'passengers',            :nullify => true
+      store 'passenger_distance',    :nullify => true, :units_field_name => 'passenger_distance_units'
+      store 'train_distance',        :nullify => true, :units_field_name => 'train_distance_units'
+      store 'train_time',            :nullify => true, :units_field_name => 'train_time_units'
+      store 'electricity_intensity', :nullify => true, :units_field_name => 'electricity_intensity_units'
+      store 'diesel_intensity',      :nullify => true, :units_field_name => 'diesel_intensity_units'
+    end
+    
+    process "Ensure the National Transit Database data is imported" do
+      NationalTransitDatabaseRecord.run_data_miner!
+      NationalTransitDatabaseCompany.run_data_miner!
+      NationalTransitDatabaseMode.run_data_miner!
+    end
+    
+    process "Derive US transit rail company data from the National Transit Database" do
+      NationalTransitDatabaseCompany.rail_companies.each do |ntd_company|
+        company = RailCompany.find_or_create_by_name(ntd_company.name)
+        company.country_iso_3166_code    = 'US'
+        company.duns_number              = ntd_company.duns_number
+        company.passengers               = ntd_company.rail_passengers
+        company.passenger_distance       = ntd_company.rail_passenger_distance
+        company.passenger_distance_units = ntd_company.rail_passenger_distance_units
+        company.train_distance           = ntd_company.rail_vehicle_distance
+        company.train_distance_units     = ntd_company.rail_vehicle_distance_units
+        company.train_time               = ntd_company.rail_vehicle_time
+        company.train_time_units         = ntd_company.rail_vehicle_time_units
+        
+        if ntd_company.rail_electricity.present? and ntd_company.rail_electricity > 0 and ntd_company.rail_passenger_distance > 0
+          company.electricity_intensity = ntd_company.rail_electricity / ntd_company.rail_passenger_distance
+          company.electricity_intensity_units = "#{ntd_company.rail_electricity_units}_per_passenger_#{ntd_company.rail_passenger_distance_units.singularize}"
+        end
+        
+        if ntd_company.rail_diesel.present? and ntd_company.rail_diesel > 0 and ntd_company.rail_passenger_distance > 0
+          company.diesel_intensity = ntd_company.rail_diesel / ntd_company.rail_passenger_distance
+          company.diesel_intensity_units = "#{ntd_company.rail_diesel_units}_per_passenger_#{ntd_company.rail_passenger_distance_units.singularize}"
+        end
+        
+        company.save!
+      end
+    end
+    
+    process "Calculate average trip distance" do
+      RailCompany.find_each do |company|
+        if company.passenger_distance.present? and company.passengers.present? and company.passengers > 0
+          company.trip_distance = company.passenger_distance / company.passengers
+          company.trip_distance_units = company.passenger_distance_units
+          company.save!
+        end
+      end
+    end
+    
+    process "Calculate average trip speed" do
+      RailCompany.find_each do |company|
+        if company.train_distance.present? and company.train_time.present? and company.train_time > 0
+          company.speed = company.train_distance / company.train_time
+          company.speed_units = "#{company.train_distance_units}_per_#{company.train_time_units.singularize}"
+          company.save!
+        end
+      end
+    end
+    
+    process "Ensure RailFuel is populated" do
+      RailFuel.run_data_miner!
+    end
+    
+    process "Calculate co2 emission factor for US rail companies" do
+      RailCompany.where(:country_iso_3166_code => 'US').find_each do |company|
+        company.co2_emission_factor = 0
+        
+        if company.diesel_intensity.present?
+          company.co2_emission_factor += company.diesel_intensity * RailFuel.find_by_name("diesel").co2_emission_factor
+        end
+        
+        if company.electricity_intensity.present?
+          company.co2_emission_factor += company.electricity_intensity * (EgridSubregion.fallback.electricity_co2_emission_factor / (1 - EgridRegion.fallback.loss_factor))
+        end
+        
+        company.co2_emission_factor_units = 'kilograms_per_passenger_kilometre'
+        company.save!
+      end
+    end
+  end
+end

data/lib/earth/rail/rail_company_traction.rb

@@ -0,0 +1,12 @@
+class RailCompanyTraction < ActiveRecord::Base
+  set_primary_key :name
+  col :name
+  col :rail_company_name
+  col :rail_traction_name
+  col :electricity_intensity, :type => :float
+  col :electricity_intensity_units
+  col :diesel_intensity, :type => :float
+  col :diesel_intensity_units
+  col :co2_emission_factor, :type => :float
+  col :co2_emission_factor_units
+end

data/lib/earth/rail/rail_company_traction/data_miner.rb

@@ -0,0 +1,4 @@
+RailCompanyTraction.class_eval do
+  data_miner do
+  end
+end

data/lib/earth/rail/rail_company_traction_class.rb

@@ -0,0 +1,14 @@
+class RailCompanyTractionClass < ActiveRecord::Base
+  set_primary_key :name
+  
+  col :name
+  col :rail_company_name
+  col :rail_traction_name
+  col :rail_class_name
+  col :electricity_intensity, :type => :float
+  col :electricity_intensity_units
+  col :diesel_intensity, :type => :float
+  col :diesel_intensity_units
+  col :co2_emission_factor, :type => :float
+  col :co2_emission_factor_units
+end