earth-ruby19 0.2.3

data/lib/earth/pet/species/data_miner.rb

@@ -0,0 +1,42 @@
+Species.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'name'
+      integer  'population'
+      float    'diet_emission_intensity'
+      string   'diet_emission_intensity_units'
+      float    'weight'
+      string   'weight_units'
+      float    'marginal_dietary_requirement'
+      string   'marginal_dietary_requirement_units'
+      float    'fixed_dietary_requirement'
+      string   'fixed_dietary_requirement_units'
+      float    'minimum_weight'
+      string   'minimum_weight_units'
+      float    'maximum_weight'
+      string   'maximum_weight_units'
+    end
+    
+    process "Define some unit conversions" do
+      # 1 (kilocalories per pound) = 9 224.14105 joules per kilogram
+      Conversions.register :kilocalories_per_pound, :joules_per_kilogram, 9_224.14105
+      # 1 (grams per kilocalories) = 2.39005736 × 10-7 kilograms per joules
+      Conversions.register :grams_per_kilocalorie, :kilograms_per_joule, 2.39005736e-7
+      # 1 joule = 0.000239005736 kilocalories
+      Conversions.register :joules, :kilocalories, 0.000239005736
+    end
+    
+    import "a list of pet species weights, dietary requirements, and diet emissions intensities",
+           :url => 'http://static.brighterplanet.com/science/data/consumables/pets/species.csv' do
+      key 'name', :field_name => 'species'
+      store 'population'
+      store 'weight', :from_units => :pounds, :to_units => :kilograms
+      store 'marginal_dietary_requirement', :from_units => :kilocalories_per_pound, :to_units => :joules_per_kilogram
+      store 'fixed_dietary_requirement', :from_units => :kilocalories, :to_units => :joules
+      store 'diet_emission_intensity', :from_units => :grams_per_kilocalorie, :to_units => :kilograms_per_joule
+      store 'minimum_weight', :from_units => :pounds, :to_units => :kilograms
+      store 'maximum_weight', :from_units => :pounds, :to_units => :kilograms
+    end
+  end
+end
+

data/lib/earth/rail.rb

@@ -0,0 +1 @@
+require 'earth/rail/rail_class'

data/lib/earth/rail/data_miner.rb

@@ -0,0 +1 @@
+require 'earth/rail/rail_class/data_miner'

data/lib/earth/rail/rail_class.rb

@@ -0,0 +1,16 @@
+class RailClass < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :rail_trips
+
+  data_miner do
+    tap "Brighter Planet's rail class data", Earth.taps_server
+  end
+  
+  # speed is missing
+  # https://brighterplanet.sifterapp.com/projects/30/issues/455
+  falls_back_on :passengers => 25.06,
+                :distance => 8.57.miles.to(:kilometres),
+                :diesel_intensity => 0.05.gallons_per_mile.to(:litres_per_kilometre),
+                :electricity_intensity => 8.89.miles.to(:kilometres)
+end

data/lib/earth/rail/rail_class/data_miner.rb

@@ -0,0 +1,36 @@
+RailClass.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string 'name'
+      float  'distance'
+      string 'distance_units'
+      float  'passengers'
+      float  'speed'
+      string 'speed_units'
+      float  'duration'
+      string 'duration_units'
+      float  'electricity_intensity'
+      string 'electricity_intensity_units'
+      float  'diesel_intensity'
+      string 'diesel_intensity_units'
+      string 'description'
+    end
+    
+    process "Define some unit conversions" do
+      Conversions.register :gallons_per_mile, :litres_per_kilometre, 2.35214583
+    end
+
+    import "a list of rail classes and pre-calculated trip and fuel use characteristics",
+           :url => 'http://static.brighterplanet.com/science/data/transport/rail/rail_classes.csv' do
+      key   'name'
+      store 'description'
+      store 'distance', :from_units => :miles, :to_units => :kilometres
+      store 'speed', :from_units => :miles, :to_units => :kilometres
+      store 'duration'
+      store 'passengers'
+      store 'electricity_intensity', :field_name => 'electricity_per_vehicle_mile', :from_units => :miles, :to_units => :kilometres
+      store 'diesel_intensity', :field_name => 'diesel_per_vehicle_mile', :from_units => :gallons_per_mile, :to_units => :litres_per_kilometre
+    end
+  end
+end
+

data/lib/earth/residence.rb

@@ -0,0 +1,8 @@
+require 'earth/residence/air_conditioner_use'
+require 'earth/residence/clothes_machine_use'
+require 'earth/residence/dishwasher_use'
+require 'earth/residence/residence_appliance'
+require 'earth/residence/residence_class'
+require 'earth/residence/residence_fuel_price'
+require 'earth/residence/residence_fuel_type'
+require 'earth/residence/residential_energy_consumption_survey_response'

data/lib/earth/residence/air_conditioner_use.rb

@@ -0,0 +1,13 @@
+class AirConditionerUse < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :residences
+  has_many :residential_energy_consumption_survey_responses
+
+  falls_back_on :fugitive_emission => 0.102295.pounds_per_square_foot.to(:kilograms_per_square_metre), # https://brighterplanet.sifterapp.com/projects/30/issues/430
+                :fugitive_emission_units => 'kilograms_per_square_metre'
+  
+  data_miner do
+    tap "Brighter Planet's sanitized flight service data", Earth.taps_server
+  end
+end

data/lib/earth/residence/air_conditioner_use/data_miner.rb

@@ -0,0 +1,22 @@
+AirConditionerUse.class_eval do
+  data_miner do
+    schema do
+      string 'name'
+      float 'fugitive_emission'
+      string 'fugitive_emission_units'
+    end
+
+    process "derive from ResidentialEnergyConsumptionSurveyResponse" do
+      ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO air_conditioner_uses(name)
+        SELECT DISTINCT residential_energy_consumption_survey_responses.central_ac_use FROM residential_energy_consumption_survey_responses WHERE LENGTH(residential_energy_consumption_survey_responses.central_ac_use) > 0
+      }
+    end
+    
+    import "Ian's precalculated fugitive emissions values", :url => 'http://spreadsheets.google.com/pub?key=ri_380yQZAqBKeqie_TECgg' do
+      key 'name', :field_name => 'air_conditioner_use_name'
+      store 'fugitive_emission', :units_field_name => 'unit', :to_units => :kilograms_per_square_metre
+    end
+  end
+end

data/lib/earth/residence/clothes_machine_use.rb

@@ -0,0 +1,10 @@
+class ClothesMachineUse < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :residences
+  has_many :residential_energy_consumption_survey_responses
+
+  data_miner do
+    tap "Brighter Planet's sanitized clothes machine use data", Earth.taps_server
+  end
+end

data/lib/earth/residence/clothes_machine_use/data_miner.rb

@@ -0,0 +1,28 @@
+ClothesMachineUse.class_eval do
+  data_miner do
+    schema do
+      string 'name'
+      float  'annual_energy_from_electricity_for_clothes_driers'
+      string 'annual_energy_from_electricity_for_clothes_driers_units'
+    end
+
+    process "derive from ResidentialEnergyConsumptionSurveyResponse" do
+      ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO clothes_machine_uses(name)
+        SELECT DISTINCT residential_energy_consumption_survey_responses.clothes_washer_use FROM residential_energy_consumption_survey_responses WHERE LENGTH(residential_energy_consumption_survey_responses.clothes_washer_use) > 0
+      }
+    end
+
+    # sabshere 5/20/10 weird that this uses cohort
+    process "precalculate annual energy use" do
+      find_in_batches do |batch|
+        batch.each do |record|
+          record.annual_energy_from_electricity_for_clothes_driers = ResidentialEnergyConsumptionSurveyResponse.big_cohort(:clothes_machine_use => record).weighted_average :annual_energy_from_electricity_for_clothes_driers
+          record.annual_energy_from_electricity_for_clothes_driers_units = 'joules'
+          record.save!
+        end
+      end
+    end
+  end
+end

data/lib/earth/residence/data_miner.rb

@@ -0,0 +1,8 @@
+require 'earth/residence/air_conditioner_use/data_miner'
+require 'earth/residence/clothes_machine_use/data_miner'
+require 'earth/residence/dishwasher_use/data_miner'
+require 'earth/residence/residence_appliance/data_miner'
+require 'earth/residence/residence_class/data_miner'
+require 'earth/residence/residence_fuel_price/data_miner'
+require 'earth/residence/residence_fuel_type/data_miner'
+require 'earth/residence/residential_energy_consumption_survey_response/data_miner'

data/lib/earth/residence/dishwasher_use.rb

@@ -0,0 +1,10 @@
+class DishwasherUse < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :residences
+  has_many :residential_energy_consumption_survey_responses
+  
+  data_miner do
+    tap "Brighter Planet's sanitized dishwasher use data", Earth.taps_server
+  end
+end

data/lib/earth/residence/dishwasher_use/data_miner.rb

@@ -0,0 +1,28 @@
+DishwasherUse.class_eval do
+  data_miner do
+    schema do
+      string 'name'
+      float 'annual_energy_from_electricity_for_dishwashers'
+      string 'annual_energy_from_electricity_for_dishwashers_units'
+    end
+    
+    process "derive from ResidentialEnergyConsumptionSurveyResponse" do
+      ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO dishwasher_uses(name)
+        SELECT DISTINCT residential_energy_consumption_survey_responses.dishwasher_use_id FROM residential_energy_consumption_survey_responses WHERE LENGTH(residential_energy_consumption_survey_responses.dishwasher_use_id) > 0
+      }
+    end
+    
+    # sabshere 5/25/10 weird that this uses cohort
+    process "precalculate annual energy" do
+      find_in_batches do |batch|
+        batch.each do |record|
+          record.annual_energy_from_electricity_for_dishwashers = ResidentialEnergyConsumptionSurveyResponse.big_cohort(:dishwasher_use => record).weighted_average :annual_energy_from_electricity_for_dishwashers
+          record.annual_energy_from_electricity_for_dishwashers_units = 'joules'
+          record.save!
+        end
+      end
+    end
+  end
+end

data/lib/earth/residence/residence_appliance.rb

@@ -0,0 +1,16 @@
+class ResidenceAppliance < ActiveRecord::Base
+  set_primary_key :name
+
+  class << self
+    def annual_energy_from_electricity_for(appliance_plural)
+      appliance_name = appliance_plural.to_s.singularize
+      if appliance = find_by_name(appliance_name)
+        appliance.annual_energy_from_electricity
+      end
+    end
+  end
+
+  data_miner do
+    tap "Brighter Planet's residence appliance energy information", Earth.taps_server
+  end
+end

data/lib/earth/residence/residence_appliance/data_miner.rb

@@ -0,0 +1,20 @@
+ResidenceAppliance.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'name'
+      float    'annual_energy_from_electricity'
+      string   'annual_energy_from_electricity_units'
+    end
+
+    process "Derive from residential energy consumption survey responses" do
+      ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
+      ResidentialEnergyConsumptionSurveyResponse.column_names.grep(/_count$/).each do |column_name|
+        appliance_name = column_name.sub '_count', ''
+        appliance = find_or_create_by_name appliance_name
+        appliance.annual_energy_from_electricity = ResidentialEnergyConsumptionSurveyResponse.weighted_average "annual_energy_from_electricity_for_#{appliance_name.pluralize}"
+        appliance.save!
+      end
+    end
+  end
+end
+

data/lib/earth/residence/residence_class.rb

@@ -0,0 +1,16 @@
+class ResidenceClass < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :residences
+  has_many :residential_energy_consumption_survey_responses
+
+  CLASSIFICATIONS = ['mobile home', 'house', 'apartment']
+
+  def classification
+    CLASSIFICATIONS.detect { |c| name.downcase.include? c }
+  end
+
+  data_miner do
+    tap "Brighter Planet's sanitized residence class data", Earth.taps_server
+  end
+end

data/lib/earth/residence/residence_class/data_miner.rb

@@ -0,0 +1,15 @@
+ResidenceClass.class_eval do
+  data_miner do
+    schema do
+      string :name
+    end
+
+    process "derive from ResidentialEnergyConsumptionSurveyResponse" do
+      ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO residence_classes(name)
+        SELECT DISTINCT residential_energy_consumption_survey_responses.residence_class_id FROM residential_energy_consumption_survey_responses WHERE LENGTH(residential_energy_consumption_survey_responses.residence_class_id) > 0
+      }
+    end
+  end
+end

data/lib/earth/residence/residence_fuel_price.rb

@@ -0,0 +1,18 @@
+class ResidenceFuelPrice < ActiveRecord::Base
+  set_primary_key :row_hash
+  
+  extend CohortScope
+  self.minimum_cohort_size = 5 # ? FIXME
+  
+  belongs_to :fuel, :class_name => 'ResidenceFuelType', :foreign_key => 'residence_fuel_type_name'
+  belongs_to :locatable, :polymorphic => true
+  
+  data_miner do
+    tap "Brighter Planet's residence fuel price data", Earth.taps_server
+
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+
+end

data/lib/earth/residence/residence_fuel_price/data_miner.rb

@@ -0,0 +1,200 @@
+ResidenceFuelPrice.class_eval do
+  class FuelOilParser
+    def initialize(options = {})
+      # nothing
+    end
+    def add_hints!(bus)
+      bus[:sheet] = 'Data 1'
+      bus[:skip] = 2
+      bus[:select] = lambda { |row| row['year'].to_i > 1989 }
+    end
+    def apply(row)
+      virtual_rows = []
+      row.keys.grep(/(.*) Residual Fuel Oil/) do |location_column_name|
+        match_1 = $1
+        next if (price = row[location_column_name]).blank? or (date = row['Date']).blank?
+        if match_1.starts_with?('U.S.')
+          locatable_id = 'US'
+          locatable_type = 'Country'
+        elsif match_1.include?('PADD')
+          /\(PADD (.*)\)/.match match_1
+          match_2 = $1
+          next if match_2 == '1' # skip PADD 1 because we always prefer subdistricts
+          locatable_id = match_2
+          locatable_type = 'PetroleumAdministrationForDefenseDistrict'
+        else
+          locatable_id = match_1
+          locatable_type = 'State'
+        end
+        date = Time.parse(date)
+        new_row = ActiveSupport::OrderedHash.new
+        new_row['locatable_id'] = locatable_id
+        new_row['locatable_type'] = locatable_type
+        new_row['price'] = price.to_f.cents.to(:dollars)
+        new_row['year'] = date.year
+        new_row['month'] = date.month
+        row_hash = RemoteTable::Transform.row_hash new_row
+        new_row['row_hash'] = row_hash
+        virtual_rows << new_row
+      end
+      virtual_rows
+    end
+  end
+
+  class PropaneParser
+    def initialize(options = {})
+      # nothing
+    end
+    def add_hints!(bus)
+      bus[:sheet] = 'Data 1'
+      bus[:skip] = 2
+      bus[:select] = lambda { |row| row['year'].to_i > 1989 }
+    end
+    def apply(row)
+      virtual_rows = []
+      row.keys.grep(/(.*) Propane Residential Price/) do |location_column_name|
+        match_1 = $1
+        next if (price = row[location_column_name]).blank? or (date = row['Date']).blank?
+        if match_1.starts_with?('U.S.')
+          locatable_id = 'US'
+          locatable_type = 'Country'
+        else
+          /\(PADD (.*)\)/.match match_1
+          match_2 = $1
+          next if match_2 == '1' # skip PADD 1 because we always prefer subdistricts
+          locatable_id = match_2
+          locatable_type = 'PetroleumAdministrationForDefenseDistrict'
+        end
+        date = Time.parse(date)
+        new_row = ActiveSupport::OrderedHash.new
+        new_row['locatable_id'] = locatable_id
+        new_row['locatable_type'] = locatable_type
+        new_row['price'] = price.to_f.cents.to(:dollars)
+        new_row['year'] = date.year
+        new_row['month'] = date.month
+        row_hash = RemoteTable::Transform.row_hash new_row
+        new_row['row_hash'] = row_hash
+        virtual_rows << new_row
+      end
+      virtual_rows
+    end
+  end
+
+  class NaturalGasParser
+    def initialize(options = {})
+      # nothing
+    end
+    def add_hints!(bus)
+      bus[:sheet] = 'Data 1'
+      bus[:skip] = 2
+      bus[:select] = lambda { |row| row['year'].to_i > 1989 }
+    end
+    def apply(row)
+      virtual_rows = []
+      row.keys.grep(/\A(.*) Natural Gas/) do |location_column_name|
+        match_1 = $1
+        next if (price = row[location_column_name]).blank? or (date = row['Date']).blank?
+        if match_1 == 'U.S.'
+          locatable_id = 'US'
+          locatable_type = 'Country'
+        else
+          locatable_id = match_1 # name
+          locatable_type = 'State'
+        end
+        date = Time.parse(date)
+        new_row = ActiveSupport::OrderedHash.new
+        new_row['locatable_id'] = locatable_id
+        new_row['locatable_type'] = locatable_type
+        new_row['price'] = price
+        new_row['year'] = date.year
+        new_row['month'] = date.month
+        row_hash = RemoteTable::Transform.row_hash new_row
+        new_row['row_hash'] = row_hash
+        virtual_rows << new_row
+      end
+      virtual_rows
+    end
+  end
+
+  data_miner do
+    schema Earth.database_options do
+      string  'row_hash'
+      string 'residence_fuel_type_name'
+      integer 'year'
+      integer 'month'
+      float   'price'
+      string  'price_units'
+      string  'price_description'
+      string  'locatable_id'
+      string  'locatable_type'
+      index   ['price', 'residence_fuel_type_name', 'month', 'year', 'locatable_type', 'locatable_id']
+      index   ['price', 'residence_fuel_type_name']
+    end
+    
+    process "Define some unit conversions" do
+      Conversions.register :dollars, :cents, 100
+      Conversions.register :cubic_feet, :cubic_metres, 0.0283168466 # TODO conversions gem has 'cubic_metres'
+    end
+
+    # electricity in dollars per kWh
+    import 'residential electricity prices from the EIA',
+           :url => 'http://www.eia.doe.gov/cneaf/electricity/page/sales_revenue.xls',
+           :select => lambda { |row| row['Year'].to_s.first(4).to_i > 1989 } do
+      key 'row_hash'
+      store 'residence_fuel_type_name', :static => 'electricity'
+      store 'locatable_id', :field_name => 'State' # postal abbrev
+      store 'locatable_type', :static => 'State'
+      store 'price', :field_name => 'Average Retail Price Residential (c/kWh)', :from_units => :cents, :to_units => :dollars
+      store 'price_description', :static => 'dollars_per_kilowatt_hour'
+      store 'year', :field_name => 'Year'
+      store 'month', :field_name => 'Month'
+    end
+    
+    # natural gas in dollars per cubic metre
+    # breaks if date-performance is enabled because DateTime.parse(...1899...) dies
+    import 'residential natural gas prices from the EIA',
+           :url => 'http://tonto.eia.doe.gov/dnav/ng/xls/ng_pri_sum_a_EPG0_FWA_DMcf_a.xls',
+           :transform => { :class => NaturalGasParser } do
+      key 'row_hash'
+      store 'residence_fuel_type_name', :static => 'natural gas'
+      store 'locatable_id'
+      store 'locatable_type'
+      store 'price', :from_units => :cubic_metres, :to_units => :cubic_feet # denominator
+      store 'price_description', :static => 'dollars_per_cubic_metre'
+      store 'year'
+      store 'month'
+    end
+    
+    # dollars per litre
+    import "residential fuel oil prices from the EIA",
+           :url => 'http://tonto.eia.doe.gov/dnav/pet/xls/PET_PRI_RESID_A_EPPR_PTA_CPGAL_M.xls',
+           :transform => { :class => FuelOilParser } do
+      key 'row_hash'
+      store 'residence_fuel_type_name', :static => 'fuel oil'
+      store 'locatable_id'
+      store 'locatable_type'
+      store 'price', :from_units => :litres, :to_units => :gallons # denominator
+      store 'price_description', :static => 'dollars_per_litre'
+      store 'year'
+      store 'month'
+    end
+    
+    # dollars per litre
+    import "residential propane prices from the EIA",
+           :url => 'http://tonto.eia.doe.gov/dnav/pet/xls/PET_PRI_PROP_A_EPLLPA_PRT_CPGAL_M.xls',
+           :transform => { :class => PropaneParser } do
+      key 'row_hash'
+      store 'residence_fuel_type_name', :static => 'propane'
+      store 'locatable_id'
+      store 'locatable_type'
+      store 'price', :from_units => :litres, :to_units => :gallons # denominator
+      store 'price_description', :static => 'dollars_per_litre'
+      store 'year'
+      store 'month'
+    end
+    
+    # per Matt in https://brighterplanet.sifterapp.com/projects/30/issues/410/comments
+    # "For coal and kerosene, there isn't good residential price data available, because hardly anybody actually uses them residentially."
+  end
+end
+