earth 0.0.18

data/lib/earth/residence/dishwasher_use/data_miner.rb

@@ -0,0 +1,4 @@
+DishwasherUse.class_eval do
+#  data_miner do
+#...
+end

data/lib/earth/residence/residence_appliance.rb

@@ -0,0 +1,7 @@
+class ResidenceAppliance < ActiveRecord::Base
+  set_primary_key :name
+
+  data_miner do
+    tap "Brighter Planet's residence appliance energy information", Earth.taps_server
+  end
+end

data/lib/earth/residence/residence_appliance/data_miner.rb

@@ -0,0 +1,19 @@
+ResidenceAppliance.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'name'
+      float    'annual_energy_from_electricity'
+      string   'annual_energy_from_electricity_units'
+    end
+
+    process "Derive from residential energy consumption survey responses" do
+      ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
+      ResidentialEnergyConsumptionSurveyResponse.column_names.select { |column| column.match(/annual.*_count/) }.map { |column| column.gsub '_count', '' }.each do |appliance_name|
+        appliance = find_or_create_by_name appliance_name
+        appliance.annual_energy_from_electricity = ResidentialEnergyConsumptionSurveyResponse.weighted_average "annual_energy_from_electricity_for_#{appliance_name.pluralize}"
+        appliance.save!
+      end
+    end
+  end
+end
+

data/lib/earth/residence/residence_class.rb

@@ -0,0 +1,26 @@
+class ResidenceClass < ActiveRecord::Base
+  set_primary_key :name
+  
+  has_many :residences
+  has_many :residential_energy_consumption_survey_responses
+
+  CLASSIFICATIONS = ['mobile home', 'house', 'apartment']
+
+  def classification
+    CLASSIFICATIONS.detect { |c| name.downcase.include? c }
+  end
+
+  data_miner do
+    schema do
+      string :name
+    end
+
+    process "derive from ResidentialEnergyConsumptionSurveyResponse" do
+      ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
+      connection.execute %{
+        INSERT IGNORE INTO residence_classes(name)
+        SELECT DISTINCT residential_energy_consumption_survey_responses.residence_class_id FROM residential_energy_consumption_survey_responses WHERE LENGTH(residential_energy_consumption_survey_responses.residence_class_id) > 0
+      }
+    end
+  end
+end

data/lib/earth/residence/residence_class/data_miner.rb

@@ -0,0 +1,4 @@
+ResidenceClass.class_eval do
+#  data_miner do
+#...
+end

data/lib/earth/residence/residence_fuel_price.rb

@@ -0,0 +1,18 @@
+class ResidenceFuelPrice < ActiveRecord::Base
+  set_primary_key :row_hash
+  
+  extend CohortScope
+  self.minimum_cohort_size = 5 # ? FIXME
+  
+  belongs_to :fuel, :class_name => 'ResidenceFuelType', :foreign_key => 'residence_fuel_type_name'
+  belongs_to :locatable, :polymorphic => true
+  
+  data_miner do
+    tap "Brighter Planet's residence fuel price data", Earth.taps_server
+
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+
+end

data/lib/earth/residence/residence_fuel_price/data_miner.rb

@@ -0,0 +1,200 @@
+ResidenceFuelPrice.class_eval do
+  class FuelOilParser
+    def initialize(options = {})
+      # nothing
+    end
+    def add_hints!(bus)
+      bus[:sheet] = 'Data 1'
+      bus[:skip] = 2
+      bus[:select] = lambda { |row| row['year'].to_i > 1989 }
+    end
+    def apply(row)
+      virtual_rows = []
+      row.keys.grep(/(.*) Residual Fuel Oil/) do |location_column_name|
+        match_1 = $1
+        next if (price = row[location_column_name]).blank? or (date = row['Date']).blank?
+        if match_1.starts_with?('U.S.')
+          locatable_id = 'US'
+          locatable_type = 'Country'
+        elsif match_1.include?('PADD')
+          /\(PADD (.*)\)/.match match_1
+          match_2 = $1
+          next if match_2 == '1' # skip PADD 1 because we always prefer subdistricts
+          locatable_id = match_2
+          locatable_type = 'PetroleumAdministrationForDefenseDistrict'
+        else
+          locatable_id = match_1
+          locatable_type = 'State'
+        end
+        date = Time.parse(date)
+        new_row = ActiveSupport::OrderedHash.new
+        new_row['locatable_id'] = locatable_id
+        new_row['locatable_type'] = locatable_type
+        new_row['price'] = price.to_f.cents.to(:dollars)
+        new_row['year'] = date.year
+        new_row['month'] = date.month
+        row_hash = RemoteTable::Transform.row_hash new_row
+        new_row['row_hash'] = row_hash
+        virtual_rows << new_row
+      end
+      virtual_rows
+    end
+  end
+
+  class PropaneParser
+    def initialize(options = {})
+      # nothing
+    end
+    def add_hints!(bus)
+      bus[:sheet] = 'Data 1'
+      bus[:skip] = 2
+      bus[:select] = lambda { |row| row['year'].to_i > 1989 }
+    end
+    def apply(row)
+      virtual_rows = []
+      row.keys.grep(/(.*) Propane Residential Price/) do |location_column_name|
+        match_1 = $1
+        next if (price = row[location_column_name]).blank? or (date = row['Date']).blank?
+        if match_1.starts_with?('U.S.')
+          locatable_id = 'US'
+          locatable_type = 'Country'
+        else
+          /\(PADD (.*)\)/.match match_1
+          match_2 = $1
+          next if match_2 == '1' # skip PADD 1 because we always prefer subdistricts
+          locatable_id = match_2
+          locatable_type = 'PetroleumAdministrationForDefenseDistrict'
+        end
+        date = Time.parse(date)
+        new_row = ActiveSupport::OrderedHash.new
+        new_row['locatable_id'] = locatable_id
+        new_row['locatable_type'] = locatable_type
+        new_row['price'] = price.to_f.cents.to(:dollars)
+        new_row['year'] = date.year
+        new_row['month'] = date.month
+        row_hash = RemoteTable::Transform.row_hash new_row
+        new_row['row_hash'] = row_hash
+        virtual_rows << new_row
+      end
+      virtual_rows
+    end
+  end
+
+  class NaturalGasParser
+    def initialize(options = {})
+      # nothing
+    end
+    def add_hints!(bus)
+      bus[:sheet] = 'Data 1'
+      bus[:skip] = 2
+      bus[:select] = lambda { |row| row['year'].to_i > 1989 }
+    end
+    def apply(row)
+      virtual_rows = []
+      row.keys.grep(/\A(.*) Natural Gas/) do |location_column_name|
+        match_1 = $1
+        next if (price = row[location_column_name]).blank? or (date = row['Date']).blank?
+        if match_1 == 'U.S.'
+          locatable_id = 'US'
+          locatable_type = 'Country'
+        else
+          locatable_id = match_1 # name
+          locatable_type = 'State'
+        end
+        date = Time.parse(date)
+        new_row = ActiveSupport::OrderedHash.new
+        new_row['locatable_id'] = locatable_id
+        new_row['locatable_type'] = locatable_type
+        new_row['price'] = price
+        new_row['year'] = date.year
+        new_row['month'] = date.month
+        row_hash = RemoteTable::Transform.row_hash new_row
+        new_row['row_hash'] = row_hash
+        virtual_rows << new_row
+      end
+      virtual_rows
+    end
+  end
+
+  data_miner do
+    schema Earth.database_options do
+      string  'row_hash'
+      string 'residence_fuel_type_name'
+      integer 'year'
+      integer 'month'
+      float   'price'
+      string  'price_units'
+      string  'price_description'
+      string  'locatable_id'
+      string  'locatable_type'
+      index   ['price', 'residence_fuel_type_name', 'month', 'year', 'locatable_type', 'locatable_id']
+      index   ['price', 'residence_fuel_type_name']
+    end
+    
+    process "Define some unit conversions" do
+      Conversions.register :dollars, :cents, 100
+      Conversions.register :cubic_feet, :cubic_metres, 0.0283168466 # TODO conversions gem has 'cubic_metres'
+    end
+
+    # electricity in dollars per kWh
+    import 'residential electricity prices from the EIA',
+           :url => 'http://www.eia.doe.gov/cneaf/electricity/page/sales_revenue.xls',
+           :select => lambda { |row| row['Year'].to_s.first(4).to_i > 1989 } do
+      key 'row_hash'
+      store 'residence_fuel_type_name', :static => 'electricity'
+      store 'locatable_id', :field_name => 'State' # postal abbrev
+      store 'locatable_type', :static => 'State'
+      store 'price', :field_name => 'Average Retail Price Residential (c/kWh)', :from_units => :cents, :to_units => :dollars
+      store 'price_description', :static => 'dollars_per_kilowatt_hour'
+      store 'year', :field_name => 'Year'
+      store 'month', :field_name => 'Month'
+    end
+    
+    # natural gas in dollars per cubic metre
+    # breaks if date-performance is enabled because DateTime.parse(...1899...) dies
+    import 'residential natural gas prices from the EIA',
+           :url => 'http://tonto.eia.doe.gov/dnav/ng/xls/ng_pri_sum_a_EPG0_FWA_DMcf_a.xls',
+           :transform => { :class => NaturalGasParser } do
+      key 'row_hash'
+      store 'residence_fuel_type_name', :static => 'natural gas'
+      store 'locatable_id'
+      store 'locatable_type'
+      store 'price', :from_units => :cubic_metres, :to_units => :cubic_feet # denominator
+      store 'price_description', :static => 'dollars_per_cubic_metre'
+      store 'year'
+      store 'month'
+    end
+    
+    # dollars per litre
+    import "residential fuel oil prices from the EIA",
+           :url => 'http://tonto.eia.doe.gov/dnav/pet/xls/PET_PRI_RESID_A_EPPR_PTA_CPGAL_M.xls',
+           :transform => { :class => FuelOilParser } do
+      key 'row_hash'
+      store 'residence_fuel_type_name', :static => 'fuel oil'
+      store 'locatable_id'
+      store 'locatable_type'
+      store 'price', :from_units => :litres, :to_units => :gallons # denominator
+      store 'price_description', :static => 'dollars_per_litre'
+      store 'year'
+      store 'month'
+    end
+    
+    # dollars per litre
+    import "residential propane prices from the EIA",
+           :url => 'http://tonto.eia.doe.gov/dnav/pet/xls/PET_PRI_PROP_A_EPLLPA_PRT_CPGAL_M.xls',
+           :transform => { :class => PropaneParser } do
+      key 'row_hash'
+      store 'residence_fuel_type_name', :static => 'propane'
+      store 'locatable_id'
+      store 'locatable_type'
+      store 'price', :from_units => :litres, :to_units => :gallons # denominator
+      store 'price_description', :static => 'dollars_per_litre'
+      store 'year'
+      store 'month'
+    end
+    
+    # per Matt in https://brighterplanet.sifterapp.com/projects/30/issues/410/comments
+    # "For coal and kerosene, there isn't good residential price data available, because hardly anybody actually uses them residentially."
+  end
+end
+

data/lib/earth/residence/residence_fuel_type.rb

@@ -0,0 +1,33 @@
+class ResidenceFuelType < ActiveRecord::Base
+  set_primary_key :name
+  
+  
+  has_many :prices, :class_name => 'ResidenceFuelPrice', :foreign_key => 'residence_fuel_type_name'
+  
+  data_miner do
+    tap "Brighter Planet's residence fuel types data", Earth.taps_server
+  end
+  
+  def price_per_unit(relaxations = [])
+    conditions = { :residence_fuel_type_name => self }
+    relaxations.push Hash.new
+    relaxations.grab do |relaxation|
+      relaxation_conditions = Hash.new
+      if timeframe = relaxation[:timeframe]
+        relaxation_conditions[:year] = timeframe.from.year
+        relaxation_conditions[:month] = timeframe.from.month..timeframe.to.yesterday.month
+      end
+      if location = relaxation[:location]
+        relaxation_conditions[:locatable_type] = location.class.to_s
+        relaxation_conditions[:locatable_id] = location.id
+      end
+      ResidenceFuelPrice.average :price, :conditions => conditions.merge(relaxation_conditions)
+    end
+  end
+
+  class << self
+    def [](fuel)
+      find_by_name fuel.to_s.humanize.downcase
+    end
+  end
+end

data/lib/earth/residence/residence_fuel_type/data_miner.rb

@@ -0,0 +1,18 @@
+ResidenceFuelType.class_eval do
+  data_miner do
+    schema Earth.database_options do
+      string   'name'
+      float    'emission_factor'
+      string   'emission_factor_units'
+      # float    'energy_content'
+      # string   'energy_content_units'
+    end
+    
+    import "a list of residential fuels and their emissions factors",
+           :url => 'http://spreadsheets.google.com/pub?key=rukxnmuhhsOsrztTrUaFCXQ' do
+      key 'name'
+      store 'emission_factor', :field_name => 'emission_factor', :units_field_name => 'units'
+    end
+  end
+end
+

data/lib/earth/residence/residential_energy_consumption_survey_response.rb

@@ -0,0 +1,67 @@
+class ResidentialEnergyConsumptionSurveyResponse < ActiveRecord::Base
+  set_primary_key :department_of_energy_identifier
+    
+  belongs_to :census_division,     :foreign_key => 'census_division_number'
+  belongs_to :census_region,       :foreign_key => 'census_region_number'
+  # what follows are entirely derived here
+  belongs_to :residence_class
+  belongs_to :urbanity
+  belongs_to :dishwasher_use
+  belongs_to :air_conditioner_use
+  belongs_to :clothes_machine_use
+  
+  extend CohortScope
+  self.minimum_cohort_size = 5
+  SUBCOHORT_THRESHOLD = 5 # per Matt
+  
+  INPUT_CHARACTERISTICS = [
+    :census_region,
+    :heating_degree_days,
+    :cooling_degree_days,
+    :residence_class,
+    :rooms,
+    :bedrooms,
+    :bathrooms,
+    :floorspace,
+    :residents,
+    :urbanity,
+    :construction_year,
+    :ownership,
+  ]
+  
+  data_miner do
+    tap "Brighter Planet's sanitized RECS 2005", Earth.taps_server
+        
+    process "rename certain columns so that we can use them as association names" do
+      connection.rename_column :residential_energy_consumption_survey_responses, :dishwasher_use, :dishwasher_use_id
+    end
+    
+    process "synthesize air conditioner use from central AC and window AC use" do
+      connection.add_column :residential_energy_consumption_survey_responses, :air_conditioner_use_id, :string
+      update_all "air_conditioner_use_id = 'Turned on just about all summer'",                        " central_ac_use = 'Turned on just about all summer'                        OR window_ac_use = 'Turned on just about all summer'"
+      update_all "air_conditioner_use_id = 'Turned on quite a bit'",                                  "(central_ac_use = 'Turned on quite a bit'                                  OR window_ac_use = 'Turned on quite a bit')                                  AND air_conditioner_use_id IS NULL"
+      update_all "air_conditioner_use_id = 'Turned on only a few days or nights when really needed'", "(central_ac_use = 'Turned on only a few days or nights when really needed' OR window_ac_use = 'Turned on only a few days or nights when really needed') AND air_conditioner_use_id IS NULL"
+      update_all "air_conditioner_use_id = 'Not used at all'",                                        "(central_ac_use = 'Not used at all'                                        OR window_ac_use = 'Not used at all')                                        AND air_conditioner_use_id IS NULL"
+    end
+    
+    process "synthesize clothes machine use from washer and dryer use" do
+      connection.add_column :residential_energy_consumption_survey_responses, :clothes_machine_use_id, :string
+      update_all "clothes_machine_use_id = clothes_washer_use",         "                                                    clothes_dryer_use = 'Use it every time you wash clothes'"
+      update_all "clothes_machine_use_id = NULL",                       "clothes_washer_use IS NULL                      AND clothes_dryer_use = 'Use it for some, but not all, loads of wash'"
+      update_all "clothes_machine_use_id = '1 load or less each week'", "clothes_washer_use = '1 load or less each week' AND clothes_dryer_use = 'Use it for some, but not all, loads of wash'"
+      update_all "clothes_machine_use_id = '1 load or less each week'", "clothes_washer_use = '2 to 4 loads'             AND clothes_dryer_use = 'Use it for some, but not all, loads of wash'"
+      update_all "clothes_machine_use_id = '2 to 4 loads'",             "clothes_washer_use = '5 to 9 loads'             AND clothes_dryer_use = 'Use it for some, but not all, loads of wash'"
+      update_all "clothes_machine_use_id = '5 to 9 loads'",             "clothes_washer_use = '10 to 15 loads'           AND clothes_dryer_use = 'Use it for some, but not all, loads of wash'"
+      update_all "clothes_machine_use_id = '10 to 15 loads'",           "clothes_washer_use = 'More than 15 loads'       AND clothes_dryer_use = 'Use it for some, but not all, loads of wash'"
+      update_all "clothes_machine_use_id = NULL",                       "clothes_washer_use IS NULL                      AND clothes_dryer_use = 'Use it infrequently'"
+      update_all "clothes_machine_use_id = '1 load or less each week'", "clothes_washer_use = '1 load or less each week' AND clothes_dryer_use = 'Use it infrequently'"
+      update_all "clothes_machine_use_id = '1 load or less each week'", "clothes_washer_use = '5 to 9 loads'             AND clothes_dryer_use = 'Use it infrequently'"
+      update_all "clothes_machine_use_id = '2 to 4 loads'",             "clothes_washer_use = '10 to 15 loads'           AND clothes_dryer_use = 'Use it infrequently'"
+      update_all "clothes_machine_use_id = '5 to 9 loads'",             "clothes_washer_use = 'More than 15 loads'       AND clothes_dryer_use = 'Use it infrequently'"
+    end
+
+    process "pull dependencies" do
+      run_data_miner_on_belongs_to_associations
+    end
+  end
+end

data/lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb

@@ -0,0 +1,264 @@
+ResidentialEnergyConsumptionSurveyResponse.class_eval do
+  data_miner do
+    process "Don't re-import too often" do
+      raise DataMiner::Skip unless DataMiner::Run.allowed? ResidentialEnergyConsumptionSurveyResponse
+    end
+    
+    schema Earth.database_options do
+      integer  'department_of_energy_identifier'
+      string   'residence_class_id'
+      date     'construction_year'
+      string   'construction_period'
+      string   'urbanity_id'
+      string   'dishwasher_use_id'
+      string   'central_ac_use'
+      string   'window_ac_use'
+      string   'clothes_washer_use'
+      string   'clothes_dryer_use'
+      integer  'census_division_number'
+      string   'census_division_name'
+      integer  'census_region_number'
+      string   'census_region_name'
+      float    'rooms'
+      float    'floorspace'
+      string   'floorspace_units'
+      integer  'residents'
+      boolean  'ownership'
+      boolean  'thermostat_programmability'
+      integer  'refrigerator_count'
+      integer  'freezer_count'    
+      float    'annual_energy_from_fuel_oil_for_heating_space'
+      string   'annual_energy_from_fuel_oil_for_heating_space_units'
+      float    'annual_energy_from_fuel_oil_for_heating_water'
+      string   'annual_energy_from_fuel_oil_for_heating_water_units'
+      float    'annual_energy_from_fuel_oil_for_appliances'
+      string   'annual_energy_from_fuel_oil_for_appliances_units'
+      float    'annual_energy_from_natural_gas_for_heating_space'
+      string   'annual_energy_from_natural_gas_for_heating_space_units'
+      float    'annual_energy_from_natural_gas_for_heating_water'
+      string   'annual_energy_from_natural_gas_for_heating_water_units'
+      float    'annual_energy_from_natural_gas_for_appliances'
+      string   'annual_energy_from_natural_gas_for_appliances_units'
+      float    'annual_energy_from_propane_for_heating_space'
+      string   'annual_energy_from_propane_for_heating_space_units'
+      float    'annual_energy_from_propane_for_heating_water'
+      string   'annual_energy_from_propane_for_heating_water_units'
+      float    'annual_energy_from_propane_for_appliances'
+      string   'annual_energy_from_propane_for_appliances_units'
+      float    'annual_energy_from_wood'
+      string   'annual_energy_from_wood_units'
+      float    'annual_energy_from_kerosene'
+      string   'annual_energy_from_kerosene_units'
+      float    'annual_energy_from_electricity_for_clothes_driers'
+      string   'annual_energy_from_electricity_for_clothes_driers_units'
+      float    'annual_energy_from_electricity_for_dishwashers'
+      string   'annual_energy_from_electricity_for_dishwashers_units'
+      float    'annual_energy_from_electricity_for_freezers'
+      string   'annual_energy_from_electricity_for_freezers_units'
+      float    'annual_energy_from_electricity_for_refrigerators'
+      string   'annual_energy_from_electricity_for_refrigerators_units'
+      float    'annual_energy_from_electricity_for_air_conditioners'
+      string   'annual_energy_from_electricity_for_air_conditioners_units'
+      float    'annual_energy_from_electricity_for_heating_space'
+      string   'annual_energy_from_electricity_for_heating_space_units'
+      float    'annual_energy_from_electricity_for_heating_water'
+      string   'annual_energy_from_electricity_for_heating_water_units'
+      float    'annual_energy_from_electricity_for_other_appliances'
+      string   'annual_energy_from_electricity_for_other_appliances_units'
+      float    'weighting'
+      float    'lighting_use'
+      string   'lighting_use_units'
+      float    'lighting_efficiency'
+      integer  'heating_degree_days'
+      string   'heating_degree_days_units'
+      integer  'cooling_degree_days'
+      string   'cooling_degree_days_units'
+      integer  'total_rooms'
+      integer  'full_bathrooms'
+      integer  'bedrooms'
+      integer  'half_bathrooms'
+      float    'bathrooms'
+      boolean  'heated_garage'
+      boolean  'attached_1car_garage'
+      boolean  'detached_1car_garage'
+      boolean  'attached_2car_garage'
+      boolean  'detached_2car_garage'
+      boolean  'attached_3car_garage'
+      boolean  'detached_3car_garage'
+      integer  'lights_on_1_to_4_hours'
+      integer  'efficient_lights_on_1_to_4_hours'
+      integer  'lights_on_4_to_12_hours'
+      integer  'efficient_lights_on_4_to_12_hours'
+      integer  'lights_on_over_12_hours'
+      integer  'efficient_lights_on_over_12_hours'
+      integer  'outdoor_all_night_lights'
+      integer  'outdoor_all_night_gas_lights'
+      integer  'air_conditioner_use_id'
+      integer  'clothes_machine_use_id'
+    end
+    
+    process "Define some unit conversions" do
+      Conversions.register :kbtus, :joules, 1_000.0 * 1_055.05585
+      Conversions.register :square_feet, :square_metres, 0.09290304
+    end
+
+    # conversions are NOT performed here, since we first have to zero out legitimate skips
+    # otherwise you will get values like "999 pounds = 453.138778 kilograms" (where 999 is really a legit skip)
+    import 'the 2005 EIA Residential Energy Consumption Survey microdata',
+           :url => 'http://www.eia.doe.gov/emeu/recs/recspubuse05/datafiles/RECS05alldata.csv',
+           :headers => :upcase do
+      key   'department_of_energy_identifier', :field_name => 'DOEID'
+      
+      store 'residence_class_id', :field_name => 'TYPEHUQ', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/typehuq/typehuq.csv' }
+      store 'construction_year', :field_name => 'YEARMADE', :dictionary => { :input => 'Code', :sprintf => '%02d', :output => 'Date in the middle (synthetic)', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/yearmade/yearmade.csv' }
+      store 'construction_period', :field_name => 'YEARMADE', :dictionary => { :input => 'Code', :sprintf => '%02d', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/yearmade/yearmade.csv' }
+      store 'urbanity_id', :field_name => 'URBRUR', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/urbrur/urbrur.csv' }
+      store 'dishwasher_use_id', :field_name => 'DWASHUSE', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/dwashuse/dwashuse.csv' }
+      store 'central_ac_use', :field_name => 'USECENAC', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/usecenac/usecenac.csv' }
+      store 'window_ac_use', :field_name => 'USEWWAC', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/usewwac/usewwac.csv' }
+      store 'clothes_washer_use', :field_name => 'WASHLOAD', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/washload/washload.csv' }
+      store 'clothes_dryer_use', :field_name => 'DRYRUSE', :dictionary => { :input => 'Code', :output => 'Description', :url => 'http://github.com/brighterplanet/manually_curated_data/raw/master/dryruse/dryruse.csv' }
+      
+      store 'census_division_number', :field_name => 'DIVISION'
+      store 'census_division_name', :field_name => 'DIVISION', :dictionary => { :input => 'number', :output => 'name', :url => 'http://data.brighterplanet.com/census_divisions.csv' }
+      store 'census_region_number', :field_name => 'DIVISION', :dictionary => { :input => 'number', :output => 'census_region_number', :url => 'http://data.brighterplanet.com/census_divisions.csv' }
+      store 'census_region_name', :field_name => 'DIVISION', :dictionary => { :input => 'number', :output => 'census_region_name', :url => 'http://data.brighterplanet.com/census_divisions.csv' }
+      
+      store 'floorspace', :field_name => 'TOTSQFT', :units => :square_metres
+      store 'residents', :field_name => 'NHSLDMEM'
+      store 'refrigerator_count', :field_name => 'NUMFRIG'
+      store 'freezer_count', :field_name => 'NUMFREEZ'
+      store 'heating_degree_days', :field_name => 'HD65', :units => :degrees_fahrenheit_days # FIXME imperial
+      store 'cooling_degree_days', :field_name => 'CD65', :units => :degrees_fahrenheit_days # FIXME imperial
+      store 'annual_energy_from_fuel_oil_for_heating_space', :field_name => 'BTUFOSPH', :units => :joules
+      store 'annual_energy_from_fuel_oil_for_heating_water', :field_name => 'BTUFOWTH', :units => :joules
+      store 'annual_energy_from_fuel_oil_for_appliances', :field_name => 'BTUFOAPL', :units => :joules
+      store 'annual_energy_from_natural_gas_for_heating_space', :field_name => 'BTUNGSPH', :units => :joules
+      store 'annual_energy_from_natural_gas_for_heating_water', :field_name => 'BTUNGWTH', :units => :joules
+      store 'annual_energy_from_natural_gas_for_appliances', :field_name => 'BTUNGAPL', :units => :joules
+      store 'annual_energy_from_propane_for_heating_space', :field_name => 'BTULPSPH', :units => :joules
+      store 'annual_energy_from_propane_for_heating_water', :field_name => 'BTULPWTH', :units => :joules
+      store 'annual_energy_from_propane_for_appliances', :field_name => 'BTULPAPL', :units => :joules
+      store 'annual_energy_from_wood', :field_name => 'BTUWOOD', :units => :joules
+      store 'annual_energy_from_kerosene', :field_name => 'BTUKER', :units => :joules
+      store 'annual_energy_from_electricity_for_clothes_driers', :field_name => 'BTUELCDR', :units => :joules
+      store 'annual_energy_from_electricity_for_dishwashers', :field_name => 'BTUELDWH', :units => :joules
+      store 'annual_energy_from_electricity_for_freezers', :field_name => 'BTUELFZZ', :units => :joules
+      store 'annual_energy_from_electricity_for_refrigerators', :field_name => 'BTUELRFG', :units => :joules
+      store 'annual_energy_from_electricity_for_air_conditioners', :field_name => 'BTUELCOL', :units => :joules
+      store 'annual_energy_from_electricity_for_heating_space', :field_name => 'BTUELSPH', :units => :joules
+      store 'annual_energy_from_electricity_for_heating_water', :field_name => 'BTUELWTH', :units => :joules
+      store 'annual_energy_from_electricity_for_other_appliances', :field_name => 'BTUELAPL', :units => :joules
+      store 'weighting', :field_name => 'NWEIGHT'
+      store 'lighting_use', :static => nil, :units => :hours
+      store 'total_rooms', :field_name => 'TOTROOMS'
+      store 'full_bathrooms', :field_name => 'NCOMBATH'
+      store 'half_bathrooms', :field_name => 'NHAFBATH'
+      store 'bedrooms', :field_name => 'BEDROOMS'
+      store 'lights_on_1_to_4_hours', :field_name => 'LGT1'
+      store 'efficient_lights_on_1_to_4_hours', :field_name => 'LGT1EE'
+      store 'lights_on_4_to_12_hours', :field_name => 'LGT4'
+      store 'efficient_lights_on_4_to_12_hours', :field_name => 'LGT4EE'
+      store 'lights_on_over_12_hours', :field_name => 'LGT12'
+      store 'efficient_lights_on_over_12_hours', :field_name => 'LGT12EE'
+      store 'outdoor_all_night_lights', :field_name => 'NOUTLGTNT'
+      store 'outdoor_all_night_gas_lights', :field_name => 'NGASLIGHT'
+      # booleans where we treat anything other than true (for example legitimate skip or "occupied without paying rent") as false
+      store 'ownership', :synthesize => lambda { |row| row['KOWNRENT'] == '1' }
+      store 'thermostat_programmability', :synthesize => lambda { |row| row['PROTHERM'] == '1' }
+      store 'heated_garage', :synthesize => lambda { |row| row['GARGHEAT'] == '1' }
+      store 'attached_1car_garage', :synthesize => lambda { |row| row['GARAGE1C'] == '1' }
+      store 'detached_1car_garage', :synthesize => lambda { |row| row['DGARG1C'] == '1' }
+      store 'attached_2car_garage', :synthesize => lambda { |row| row['GARAGE2C'] == '1' }
+      store 'detached_2car_garage', :synthesize => lambda { |row| row['DGARG2C'] == '1' }
+      store 'attached_3car_garage', :synthesize => lambda { |row| row['GARAGE3C'] == '1' }
+      store 'detached_3car_garage', :synthesize => lambda { |row| row['DGARG3C'] == '1' }
+    end
+
+    # Rather than nullify the continuous variables that EIA identifies as LEGITIMATE SKIPS, we convert them to zero
+    # This makes it easier to derive useful information like "how many rooms does the house have?"
+    process "Zero out what the EIA calls LEGITIMATE SKIPS" do
+      %w{
+        annual_energy_from_electricity_for_air_conditioners
+        annual_energy_from_electricity_for_clothes_driers
+        annual_energy_from_electricity_for_dishwashers
+        annual_energy_from_electricity_for_freezers
+        annual_energy_from_electricity_for_heating_space
+        annual_energy_from_electricity_for_heating_water
+        annual_energy_from_electricity_for_other_appliances
+        annual_energy_from_electricity_for_refrigerators
+        annual_energy_from_fuel_oil_for_appliances
+        annual_energy_from_fuel_oil_for_heating_space
+        annual_energy_from_fuel_oil_for_heating_water
+        annual_energy_from_kerosene
+        annual_energy_from_propane_for_appliances
+        annual_energy_from_propane_for_heating_space
+        annual_energy_from_propane_for_heating_water
+        annual_energy_from_natural_gas_for_appliances
+        annual_energy_from_natural_gas_for_heating_space
+        annual_energy_from_natural_gas_for_heating_water
+        annual_energy_from_wood
+        lights_on_1_to_4_hours
+        lights_on_over_12_hours
+        efficient_lights_on_over_12_hours
+        efficient_lights_on_1_to_4_hours
+        lights_on_4_to_12_hours
+        efficient_lights_on_4_to_12_hours
+        outdoor_all_night_gas_lights
+        outdoor_all_night_lights
+      }.each do |attr_name|
+        max = maximum attr_name, :select => "CONVERT(#{attr_name}, UNSIGNED INTEGER)"
+        # if the maximum value of a row is all 999's, then it's a LEGITIMATE SKIP and we should set it to zero
+        if /^9+$/.match(max.to_i.to_s)
+          update_all "#{attr_name} = 0", "#{attr_name} = #{max}"
+        end
+      end
+    end
+
+    process "Convert units to metric" do
+      [
+        [ 'floorspace', :square_feet, :square_metres ],
+        [ 'annual_energy_from_fuel_oil_for_heating_space', :kbtus, :joules ],
+        [ 'annual_energy_from_fuel_oil_for_heating_water', :kbtus, :joules ],
+        [ 'annual_energy_from_fuel_oil_for_appliances', :kbtus, :joules ],
+        [ 'annual_energy_from_natural_gas_for_heating_space', :kbtus, :joules ],
+        [ 'annual_energy_from_natural_gas_for_heating_water', :kbtus, :joules ],
+        [ 'annual_energy_from_natural_gas_for_appliances', :kbtus, :joules ],
+        [ 'annual_energy_from_propane_for_heating_space', :kbtus, :joules ],
+        [ 'annual_energy_from_propane_for_heating_water', :kbtus, :joules ],
+        [ 'annual_energy_from_propane_for_appliances', :kbtus, :joules ],
+        [ 'annual_energy_from_wood', :kbtus, :joules ],
+        [ 'annual_energy_from_kerosene', :kbtus, :joules ],
+        [ 'annual_energy_from_electricity_for_clothes_driers', :kbtus, :joules ],
+        [ 'annual_energy_from_electricity_for_dishwashers', :kbtus, :joules ],
+        [ 'annual_energy_from_electricity_for_freezers', :kbtus, :joules ],
+        [ 'annual_energy_from_electricity_for_refrigerators', :kbtus, :joules ],
+        [ 'annual_energy_from_electricity_for_air_conditioners', :kbtus, :joules ],
+        [ 'annual_energy_from_electricity_for_heating_space', :kbtus, :joules ],
+        [ 'annual_energy_from_electricity_for_heating_water', :kbtus, :joules ],
+        [ 'annual_energy_from_electricity_for_other_appliances', :kbtus, :joules ],
+      ].each do |attr_name, from_units, to_units|
+        update_all "#{attr_name} = #{attr_name} * #{Conversions::Unit.exchange_rate from_units, to_units}"
+      end
+    end
+      
+    process 'Add a new field "rooms" that estimates how many rooms are in the house' do
+      update_all 'rooms = total_rooms + full_bathrooms/2 + half_bathrooms/4 + heated_garage*(attached_1car_garage + detached_1car_garage + 2*(attached_2car_garage + detached_2car_garage) + 3*(attached_3car_garage + detached_3car_garage))'
+    end
+    
+    process 'Add a new field "bathrooms" that synthesizes half and full bathrooms into one number' do
+      update_all 'bathrooms = full_bathrooms + 0.5 * half_bathrooms'
+    end
+    
+    process 'Add a new field "lighting_use" that estimates how many hours light bulbs are turned on in the house' do
+      update_all 'lighting_use = 2*(lights_on_1_to_4_hours + efficient_lights_on_1_to_4_hours) + 8*(lights_on_4_to_12_hours + efficient_lights_on_4_to_12_hours) + 16*(lights_on_over_12_hours + efficient_lights_on_over_12_hours) + 12*(outdoor_all_night_lights + outdoor_all_night_gas_lights)'
+    end
+    
+    process 'Add a new field "lighting_efficiency" that estimates what percentage of light bulbs in a house are energy-efficient' do
+      update_all 'lighting_efficiency = (2*efficient_lights_on_1_to_4_hours + 8*efficient_lights_on_4_to_12_hours + 16*efficient_lights_on_over_12_hours) / lighting_use'
+    end
+    
+    # FIXME add precalc bathrooms per https://github.com/brighterplanet/cm1/commit/77df97c50311f3c59aad891f018bf3d487afeb98
+  end
+end
+