earth 0.5.4 → 0.6.0

data/lib/earth/residence/residence_fuel_price/data_miner.rb

@@ -101,6 +101,13 @@ ResidenceFuelPrice.class_eval do
     end
   end
 
+  # TODO sabshere fix data_miner strangeness
+  # Ran #<DataMiner::Process(ResidenceFuelPrice) auto_upgrade!> in 0
+  # Ran #<DataMiner::Process(ResidenceFuelPrice) Define some unit conversions> in 0
+  # Ran #<DataMiner::Import(ResidenceFuelPrice) residential electricity prices from the EIA> in 93
+  # Skipping {"locatable_id"=>"US", "locatable_type"=>"Country", "price"=>"1.71", "year"=>1990, "month"=>6, "row_hash"=>"142ed08d9adac8e322dd123317b06de8"} because there's no primary key
+  # Skipping {"locatable_id"=>"Alabama", "locatable_type"=>"State", "price"=>"2.75", "year"=>1990, "month"=>6, "row_hash"=>"d56480d06e32e7e99ba5f85444fe4827"} because there's no primary key
+
   data_miner do
     process "Define some unit conversions" do
       Conversions.register :dollars, :cents, 100

data/lib/earth/residence/residence_fuel_type.rb

@@ -3,13 +3,11 @@ class ResidenceFuelType < ActiveRecord::Base
   
   has_many :prices, :class_name => 'ResidenceFuelPrice', :foreign_key => 'residence_fuel_type_name'
   
-  force_schema do
-    string   'name'
-    float    'emission_factor'
-    string   'emission_factor_units'
-    # float    'energy_content'
-    # string   'energy_content_units'
-  end
+  col :name
+  col :emission_factor, :type => :float
+  col :emission_factor_units
+  # col :energy_content, :type => :float
+  # col :energy_content_units
   
   def price_per_unit(relaxations = [])
     conditions = { :residence_fuel_type_name => self }
@@ -33,4 +31,4 @@ class ResidenceFuelType < ActiveRecord::Base
       find_by_name fuel.to_s.humanize.downcase
     end
   end
-end
+end

data/lib/earth/residence/residential_energy_consumption_survey_response.rb

@@ -31,96 +31,94 @@ class ResidentialEnergyConsumptionSurveyResponse < ActiveRecord::Base
   ]
   
   # sabshere 9/20/10 sorted with sort -d -t "'" -k 2 ~/Desktop/parts.txt
-  force_schema do
-    integer  'id'
-    string  'air_conditioner_use_id'
-    float    'annual_energy_from_electricity_for_air_conditioners'
-    string   'annual_energy_from_electricity_for_air_conditioners_units'
-    float    'annual_energy_from_electricity_for_clothes_driers'
-    string   'annual_energy_from_electricity_for_clothes_driers_units'
-    float    'annual_energy_from_electricity_for_dishwashers'
-    string   'annual_energy_from_electricity_for_dishwashers_units'
-    float    'annual_energy_from_electricity_for_freezers'
-    string   'annual_energy_from_electricity_for_freezers_units'
-    float    'annual_energy_from_electricity_for_heating_space'
-    string   'annual_energy_from_electricity_for_heating_space_units'
-    float    'annual_energy_from_electricity_for_heating_water'
-    string   'annual_energy_from_electricity_for_heating_water_units'
-    float    'annual_energy_from_electricity_for_other_appliances'
-    string   'annual_energy_from_electricity_for_other_appliances_units'
-    float    'annual_energy_from_electricity_for_refrigerators'
-    string   'annual_energy_from_electricity_for_refrigerators_units'
-    float    'annual_energy_from_fuel_oil_for_appliances'
-    string   'annual_energy_from_fuel_oil_for_appliances_units'
-    float    'annual_energy_from_fuel_oil_for_heating_space'
-    string   'annual_energy_from_fuel_oil_for_heating_space_units'
-    float    'annual_energy_from_fuel_oil_for_heating_water'
-    string   'annual_energy_from_fuel_oil_for_heating_water_units'
-    float    'annual_energy_from_kerosene'
-    string   'annual_energy_from_kerosene_units'
-    float    'annual_energy_from_natural_gas_for_appliances'
-    string   'annual_energy_from_natural_gas_for_appliances_units'
-    float    'annual_energy_from_natural_gas_for_heating_space'
-    string   'annual_energy_from_natural_gas_for_heating_space_units'
-    float    'annual_energy_from_natural_gas_for_heating_water'
-    string   'annual_energy_from_natural_gas_for_heating_water_units'
-    float    'annual_energy_from_propane_for_appliances'
-    string   'annual_energy_from_propane_for_appliances_units'
-    float    'annual_energy_from_propane_for_heating_space'
-    string   'annual_energy_from_propane_for_heating_space_units'
-    float    'annual_energy_from_propane_for_heating_water'
-    string   'annual_energy_from_propane_for_heating_water_units'
-    float    'annual_energy_from_wood'
-    string   'annual_energy_from_wood_units'
-    boolean  'attached_1car_garage'
-    boolean  'attached_2car_garage'
-    boolean  'attached_3car_garage'
-    float    'bathrooms'
-    integer  'bedrooms'
-    string   'census_division_name'
-    integer  'census_division_number'
-    string   'census_region_name'
-    integer  'census_region_number'
-    string   'central_ac_use'
-    string   'clothes_dryer_use'
-    string  'clothes_machine_use_id'
-    string   'clothes_washer_use'
-    string   'construction_period'
-    date     'construction_year'
-    integer  'cooling_degree_days'
-    string   'cooling_degree_days_units'
-    boolean  'detached_1car_garage'
-    boolean  'detached_2car_garage'
-    boolean  'detached_3car_garage'
-    string   'dishwasher_use_id'
-    integer  'efficient_lights_on_1_to_4_hours'
-    integer  'efficient_lights_on_4_to_12_hours'
-    integer  'efficient_lights_on_over_12_hours'
-    float    'floorspace'
-    string   'floorspace_units'
-    integer  'freezer_count'    
-    integer  'full_bathrooms'
-    integer  'half_bathrooms'
-    boolean  'heated_garage'
-    integer  'heating_degree_days'
-    string   'heating_degree_days_units'
-    float    'lighting_efficiency'
-    float    'lighting_use'
-    string   'lighting_use_units'
-    integer  'lights_on_1_to_4_hours'
-    integer  'lights_on_4_to_12_hours'
-    integer  'lights_on_over_12_hours'
-    integer  'outdoor_all_night_gas_lights'
-    integer  'outdoor_all_night_lights'
-    boolean  'ownership'
-    integer  'refrigerator_count'
-    string   'residence_class_id'
-    integer  'residents'
-    float    'rooms'
-    boolean  'thermostat_programmability'
-    integer  'total_rooms'
-    string   'urbanity_id'
-    float    'weighting'
-    string   'window_ac_use'
-  end
-end
+  col :id, :type => :integer
+  col :air_conditioner_use_id
+  col :annual_energy_from_electricity_for_air_conditioners, :type => :float
+  col :annual_energy_from_electricity_for_air_conditioners_units
+  col :annual_energy_from_electricity_for_clothes_driers, :type => :float
+  col :annual_energy_from_electricity_for_clothes_driers_units
+  col :annual_energy_from_electricity_for_dishwashers, :type => :float
+  col :annual_energy_from_electricity_for_dishwashers_units
+  col :annual_energy_from_electricity_for_freezers, :type => :float
+  col :annual_energy_from_electricity_for_freezers_units
+  col :annual_energy_from_electricity_for_heating_space, :type => :float
+  col :annual_energy_from_electricity_for_heating_space_units
+  col :annual_energy_from_electricity_for_heating_water, :type => :float
+  col :annual_energy_from_electricity_for_heating_water_units
+  col :annual_energy_from_electricity_for_other_appliances, :type => :float
+  col :annual_energy_from_electricity_for_other_appliances_units
+  col :annual_energy_from_electricity_for_refrigerators, :type => :float
+  col :annual_energy_from_electricity_for_refrigerators_units
+  col :annual_energy_from_fuel_oil_for_appliances, :type => :float
+  col :annual_energy_from_fuel_oil_for_appliances_units
+  col :annual_energy_from_fuel_oil_for_heating_space, :type => :float
+  col :annual_energy_from_fuel_oil_for_heating_space_units
+  col :annual_energy_from_fuel_oil_for_heating_water, :type => :float
+  col :annual_energy_from_fuel_oil_for_heating_water_units
+  col :annual_energy_from_kerosene, :type => :float
+  col :annual_energy_from_kerosene_units
+  col :annual_energy_from_natural_gas_for_appliances, :type => :float
+  col :annual_energy_from_natural_gas_for_appliances_units
+  col :annual_energy_from_natural_gas_for_heating_space, :type => :float
+  col :annual_energy_from_natural_gas_for_heating_space_units
+  col :annual_energy_from_natural_gas_for_heating_water, :type => :float
+  col :annual_energy_from_natural_gas_for_heating_water_units
+  col :annual_energy_from_propane_for_appliances, :type => :float
+  col :annual_energy_from_propane_for_appliances_units
+  col :annual_energy_from_propane_for_heating_space, :type => :float
+  col :annual_energy_from_propane_for_heating_space_units
+  col :annual_energy_from_propane_for_heating_water, :type => :float
+  col :annual_energy_from_propane_for_heating_water_units
+  col :annual_energy_from_wood, :type => :float
+  col :annual_energy_from_wood_units
+  col :attached_1car_garage, :type => :integer
+  col :attached_2car_garage, :type => :integer
+  col :attached_3car_garage, :type => :integer
+  col :bathrooms, :type => :float
+  col :bedrooms, :type => :integer
+  col :census_division_name
+  col :census_division_number, :type => :integer
+  col :census_region_name
+  col :census_region_number, :type => :integer
+  col :central_ac_use
+  col :clothes_dryer_use
+  col :clothes_machine_use_id
+  col :clothes_washer_use
+  col :construction_period
+  col :construction_year, :type => :date
+  col :cooling_degree_days, :type => :integer
+  col :cooling_degree_days_units
+  col :detached_1car_garage, :type => :integer
+  col :detached_2car_garage, :type => :integer
+  col :detached_3car_garage, :type => :integer
+  col :dishwasher_use_id
+  col :efficient_lights_on_1_to_4_hours, :type => :integer
+  col :efficient_lights_on_4_to_12_hours, :type => :integer
+  col :efficient_lights_on_over_12_hours, :type => :integer
+  col :floorspace, :type => :float
+  col :floorspace_units
+  col :freezer_count, :type => :integer
+  col :full_bathrooms, :type => :integer
+  col :half_bathrooms, :type => :integer
+  col :heated_garage, :type => :integer
+  col :heating_degree_days, :type => :integer
+  col :heating_degree_days_units
+  col :lighting_efficiency, :type => :float
+  col :lighting_use, :type => :float
+  col :lighting_use_units
+  col :lights_on_1_to_4_hours, :type => :integer
+  col :lights_on_4_to_12_hours, :type => :integer
+  col :lights_on_over_12_hours, :type => :integer
+  col :outdoor_all_night_gas_lights, :type => :integer
+  col :outdoor_all_night_lights, :type => :integer
+  col :ownership, :type => :boolean
+  col :refrigerator_count, :type => :integer
+  col :residence_class_id
+  col :residents, :type => :integer
+  col :rooms, :type => :float
+  col :thermostat_programmability, :type => :boolean
+  col :total_rooms, :type => :integer
+  col :urbanity_id
+  col :weighting, :type => :float
+  col :window_ac_use
+end

data/lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb

@@ -66,16 +66,17 @@ ResidentialEnergyConsumptionSurveyResponse.class_eval do
       store 'efficient_lights_on_over_12_hours', :field_name => 'LGT12EE'
       store 'outdoor_all_night_lights', :field_name => 'NOUTLGTNT'
       store 'outdoor_all_night_gas_lights', :field_name => 'NGASLIGHT'
+      # integers where we treat anything other than true (for example legitimate skip or "occupied without paying rent") as 0
+      store 'heated_garage', :synthesize => lambda { |row| row['GARGHEAT'] == '1' ? 1 : 0 }
+      store 'attached_1car_garage', :synthesize => lambda { |row| row['GARAGE1C'] == '1' ? 1 : 0 }
+      store 'detached_1car_garage', :synthesize => lambda { |row| row['DGARG1C'] == '1' ? 1 : 0 }
+      store 'attached_2car_garage', :synthesize => lambda { |row| row['GARAGE2C'] == '1' ? 1 : 0 }
+      store 'detached_2car_garage', :synthesize => lambda { |row| row['DGARG2C'] == '1' ? 1 : 0 }
+      store 'attached_3car_garage', :synthesize => lambda { |row| row['GARAGE3C'] == '1' ? 1 : 0 }
+      store 'detached_3car_garage', :synthesize => lambda { |row| row['DGARG3C'] == '1' ? 1 : 0 }
       # booleans where we treat anything other than true (for example legitimate skip or "occupied without paying rent") as false
       store 'ownership', :synthesize => lambda { |row| row['KOWNRENT'] == '1' }
       store 'thermostat_programmability', :synthesize => lambda { |row| row['PROTHERM'] == '1' }
-      store 'heated_garage', :synthesize => lambda { |row| row['GARGHEAT'] == '1' }
-      store 'attached_1car_garage', :synthesize => lambda { |row| row['GARAGE1C'] == '1' }
-      store 'detached_1car_garage', :synthesize => lambda { |row| row['DGARG1C'] == '1' }
-      store 'attached_2car_garage', :synthesize => lambda { |row| row['GARAGE2C'] == '1' }
-      store 'detached_2car_garage', :synthesize => lambda { |row| row['DGARG2C'] == '1' }
-      store 'attached_3car_garage', :synthesize => lambda { |row| row['GARAGE3C'] == '1' }
-      store 'detached_3car_garage', :synthesize => lambda { |row| row['DGARG3C'] == '1' }
     end
 
     # Rather than nullify the continuous variables that EIA identifies as LEGITIMATE SKIPS, we convert them to zero
@@ -158,7 +159,7 @@ ResidentialEnergyConsumptionSurveyResponse.class_eval do
     end
     
     process 'Add a new field "lighting_efficiency" that estimates what percentage of light bulbs in a house are energy-efficient' do
-      update_all 'lighting_efficiency = (2*efficient_lights_on_1_to_4_hours + 8*efficient_lights_on_4_to_12_hours + 16*efficient_lights_on_over_12_hours) / lighting_use'
+      update_all 'lighting_efficiency = (2*efficient_lights_on_1_to_4_hours + 8*efficient_lights_on_4_to_12_hours + 16*efficient_lights_on_over_12_hours) / lighting_use', 'lighting_use > 0'
     end
     
     process "synthesize air conditioner use from central AC and window AC use" do

data/lib/earth/shipping/carrier.rb

@@ -8,13 +8,60 @@ class Carrier < ActiveRecord::Base
                 :transport_emission_factor => 0.0005266,
                 :corporate_emission_factor => 0.221
   
-  force_schema do
-    string 'name'
-    float  'package_volume'
-    float  'route_inefficiency_factor'
-    float  'transport_emission_factor'
-    string 'transport_emission_factor_units'
-    float  'corporate_emission_factor'
-    string 'corporate_emission_factor_units'
-  end
-end
+  col :name
+  col :package_volume, :type => :float
+  col :route_inefficiency_factor, :type => :float
+  col :transport_emission_factor, :type => :float
+  col :transport_emission_factor_units
+  col :corporate_emission_factor, :type => :float
+  col :corporate_emission_factor_units
+  
+  # verify "Package volume should be greater than zero" do
+  #   Carrier.all.each do |carrier|
+  #     unless carrier.package_volume > 0
+  #       raise "Invalid package volume for Carrier #{carrier.name}: #{carrier.package_volume} (should be > 0)"
+  #     end
+  #   end
+  # end
+  # 
+  # verify "Route inefficiency factor should be one or more" do
+  #   Carrier.all.each do |carrier|
+  #     unless carrier.route_inefficiency_factor >= 1.0
+  #       raise "Invalid route inefficiency factor for Carrier #{carrier.name}: #{carrier.route_inefficiency_factor} (should be >= 1.0)"
+  #     end
+  #   end
+  # end
+  # 
+  # verify "Transport emission factor should be greater than zero" do
+  #   Carrier.all.each do |carrier|
+  #     unless carrier.transport_emission_factor > 0
+  #       raise "Invalid transport emission factor for Carrier #{carrier.name}: #{carrier.transport_emission_factor} (should be > 0)"
+  #     end
+  #   end
+  # end
+  # 
+  # verify "Transport emission factor units should never be missing" do
+  #   Carrier.all.each do |carrier|
+  #     unless carrier.transport_emission_factor_units.present?
+  #       raise "Missing transport emission factor units for Carrier #{carrier.name}"
+  #     end
+  #   end
+  # end
+  # 
+  # verify "Corporate emission factor should be greater than zero" do
+  #   Carrier.all.each do |carrier|
+  #     unless carrier.corporate_emission_factor > 0
+  #       raise "Invalid corporate emission factor for Carrier #{carrier.name}: #{carrier.corporate_emission_factor} (should be > 0)"
+  #     end
+  #   end
+  # end
+  # 
+  # verify "Corporate emission factor units should never be missing" do
+  #   Carrier.all.each do |carrier|
+  #     unless carrier.corporate_emission_factor_units.present?
+  #       raise "Missing corporate emission factor units for Carrier #{carrier.name}"
+  #     end
+  #   end
+  # end
+  
+end

data/lib/earth/shipping/carrier/data_miner.rb

@@ -22,53 +22,5 @@ Carrier.class_eval do
     #   update_all "transport_emission_factor = (#{CarrierMode.weighted_average_relation(:transport_emission_factor, :weighted_by => :package_volume).where(conditional_relation).to_sql})"
     #   what about transport_emission_factor_units?
     # end
-    
-    verify "Package volume should be greater than zero" do
-      Carrier.all.each do |carrier|
-        unless carrier.package_volume > 0
-          raise "Invalid package volume for Carrier #{carrier.name}: #{carrier.package_volume} (should be > 0)"
-        end
-      end
-    end
-    
-    verify "Route inefficiency factor should be one or more" do
-      Carrier.all.each do |carrier|
-        unless carrier.route_inefficiency_factor >= 1.0
-          raise "Invalid route inefficiency factor for Carrier #{carrier.name}: #{carrier.route_inefficiency_factor} (should be >= 1.0)"
-        end
-      end
-    end
-    
-    verify "Transport emission factor should be greater than zero" do
-      Carrier.all.each do |carrier|
-        unless carrier.transport_emission_factor > 0
-          raise "Invalid transport emission factor for Carrier #{carrier.name}: #{carrier.transport_emission_factor} (should be > 0)"
-        end
-      end
-    end
-    
-    verify "Transport emission factor units should never be missing" do
-      Carrier.all.each do |carrier|
-        unless carrier.transport_emission_factor_units.present?
-          raise "Missing transport emission factor units for Carrier #{carrier.name}"
-        end
-      end
-    end
-    
-    verify "Corporate emission factor should be greater than zero" do
-      Carrier.all.each do |carrier|
-        unless carrier.corporate_emission_factor > 0
-          raise "Invalid corporate emission factor for Carrier #{carrier.name}: #{carrier.corporate_emission_factor} (should be > 0)"
-        end
-      end
-    end
-    
-    verify "Corporate emission factor units should never be missing" do
-      Carrier.all.each do |carrier|
-        unless carrier.corporate_emission_factor_units.present?
-          raise "Missing corporate emission factor units for Carrier #{carrier.name}"
-        end
-      end
-    end
   end
 end

data/lib/earth/shipping/carrier_mode.rb

@@ -4,13 +4,44 @@ class CarrierMode < ActiveRecord::Base
   belongs_to :carrier, :foreign_key => 'carrier_name', :primary_key => 'name'
   belongs_to :mode,    :foreign_key => 'mode_name',    :primary_key => 'name', :class_name => 'ShipmentMode'
 
-  force_schema do
-    string 'name'
-    string 'carrier_name'
-    string 'mode_name'
-    float  'package_volume'
-    float  'route_inefficiency_factor'
-    float  'transport_emission_factor'
-    string 'transport_emission_factor_units'
-  end
-end
+  col :name
+  col :carrier_name
+  col :mode_name
+  col :package_volume, :type => :float
+  col :route_inefficiency_factor, :type => :float
+  col :transport_emission_factor, :type => :float
+  col :transport_emission_factor_units
+  
+  # Don't need to check that carrier_name appears in carriers b/c carriers is derived from carrier_modes.carrier_name
+  # Don't need to check that mode_name appears in shipment_modes b/c shipment_modes is derived from carrier_modes.mode_name
+  # FIXME TODO test for valid transport_emission_factor_units
+  # %w{carrier_name mode_name transport_emission_factor_units}.each do |attribute|
+  #   verify "#{attribute.humanize} should never be missing" do
+  #     CarrierMode.all.each do |carrier_mode|
+  #       value = carrier_mode.send(:"#{attribute}")
+  #       unless value.present?
+  #         raise "Missing #{attribute.humanize.downcase} for CarrierMode #{carrier_mode.name}"
+  #       end
+  #     end
+  #   end
+  # end
+  # 
+  # %w{package_volume transport_emission_factor}.each do |attribute|
+  #   verify "#{attribute.humanize} should be greater than zero" do
+  #     CarrierMode.all.each do |carrier_mode|
+  #       value = carrier_mode.send(:"#{attribute}")
+  #       unless value > 0
+  #         raise "Invalid #{attribute.humanize.downcase} for CarrierMode #{carrier_mode.name}: #{value} (should be > 0)"
+  #       end
+  #     end
+  #   end
+  # end
+  # 
+  # verify "Route inefficiency factor should be one or more" do
+  #   CarrierMode.all.each do |carrier_mode|
+  #     unless carrier_mode.route_inefficiency_factor >= 1.0
+  #       raise "Invalid route inefficiency factor for CarrierMode #{carrier_mode.name}: #{carrier_mode.route_inefficiency_factor} (should be >= 1.0)"
+  #     end
+  #   end
+  # end
+end

data/lib/earth/shipping/carrier_mode/data_miner.rb

@@ -9,38 +9,5 @@ CarrierMode.class_eval do
       store 'route_inefficiency_factor'
       store 'transport_emission_factor', :units_field_name => 'transport_emission_factor_units'
     end
-    
-    # Don't need to check that carrier_name appears in carriers b/c carriers is derived from carrier_modes.carrier_name
-    # Don't need to check that mode_name appears in shipment_modes b/c shipment_modes is derived from carrier_modes.mode_name
-    # FIXME TODO test for valid transport_emission_factor_units
-    %w{carrier_name mode_name transport_emission_factor_units}.each do |attribute|
-      verify "#{attribute.humanize} should never be missing" do
-        CarrierMode.all.each do |carrier_mode|
-          value = carrier_mode.send(:"#{attribute}")
-          unless value.present?
-            raise "Missing #{attribute.humanize.downcase} for CarrierMode #{carrier_mode.name}"
-          end
-        end
-      end
-    end
-    
-    %w{package_volume transport_emission_factor}.each do |attribute|
-      verify "#{attribute.humanize} should be greater than zero" do
-        CarrierMode.all.each do |carrier_mode|
-          value = carrier_mode.send(:"#{attribute}")
-          unless value > 0
-            raise "Invalid #{attribute.humanize.downcase} for CarrierMode #{carrier_mode.name}: #{value} (should be > 0)"
-          end
-        end
-      end
-    end
-    
-    verify "Route inefficiency factor should be one or more" do
-      CarrierMode.all.each do |carrier_mode|
-        unless carrier_mode.route_inefficiency_factor >= 1.0
-          raise "Invalid route inefficiency factor for CarrierMode #{carrier_mode.name}: #{carrier_mode.route_inefficiency_factor} (should be >= 1.0)"
-        end
-      end
-    end
   end
 end