earth 0.11.7 → 0.11.8

data/lib/earth/fuel.rb

@@ -1,5 +1,3 @@
-require 'earth/fuel/fuel'
-require 'earth/fuel/fuel_price'
-require 'earth/fuel/fuel_type'
-require 'earth/fuel/fuel_year'
-require 'earth/fuel/greenhouse_gas'
+# lib/earth/fuel.rb
+require 'earth'
+Earth.require_related __FILE__

data/lib/earth/fuel/data_miner.rb

@@ -1,5 +1,3 @@
-require 'earth/fuel/fuel/data_miner'
-require 'earth/fuel/fuel_price/data_miner'
-require 'earth/fuel/fuel_type/data_miner'
-require 'earth/fuel/fuel_year/data_miner'
-require 'earth/fuel/greenhouse_gas/data_miner'
+# lib/earth/fuel/data_miner.rb
+require 'earth'
+Earth.require_related __FILE__

data/lib/earth/fuel/fuel_price.rb

@@ -6,4 +6,4 @@ class FuelPrice < ActiveRecord::Base
   col :name
   col :price, :type => :float
   col :price_units
-end
+end

data/lib/earth/hospitality.rb

@@ -1,5 +1,3 @@
-require 'earth/hospitality/census_region_lodging_class'
-require 'earth/hospitality/commercial_building_energy_consumption_survey_response'
-require 'earth/hospitality/country_lodging_class'
-require 'earth/hospitality/lodging_class'
-require 'earth/hospitality/lodging_property'
+# lib/earth/hospitality.rb
+require 'earth'
+Earth.require_related __FILE__

data/lib/earth/hospitality/commercial_building_energy_consumption_survey_response.rb

@@ -1,3 +1,4 @@
+require 'earth/locality'
 class CommercialBuildingEnergyConsumptionSurveyResponse < ActiveRecord::Base
   set_primary_key :id
   set_table_name :cbecs_responses
@@ -8,6 +9,7 @@ class CommercialBuildingEnergyConsumptionSurveyResponse < ActiveRecord::Base
   col :id,                     :type => :integer
   col :census_region_number,   :type => :integer
   col :census_division_number, :type => :integer
+  col :climate_zone_number,    :type => :integer
   col :heating_degree_days,    :type => :integer
   col :cooling_degree_days,    :type => :integer
   col :construction_year,      :type => :integer

data/lib/earth/hospitality/commercial_building_energy_consumption_survey_response/data_miner.rb

@@ -1,5 +1,10 @@
+require 'earth/locality/data_miner'
 CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
   data_miner do
+    process 'Define some unit conversions' do
+      Conversions.register :hundred_cubic_feet, :cubic_metres, 2.83168466
+    end
+    
     import 'building characteristics from the 2003 EIA Commercial Building Energy Consumption Survey',
            :url => 'http://www.eia.gov/emeu/cbecs/cbecs2003/public_use_2003/data/FILE01.csv',
            :skip => 1,
@@ -7,6 +12,7 @@ CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
       key 'id', :field_name => 'PUBID8'
       store 'census_region_number',   :field_name => 'REGION8'
       store 'census_division_number', :field_name => 'CENDIV8'
+      store 'climate_zone_number',    :synthesize => Proc.new { |row| row['CLIMATE8'].to_i == 7 ? nil : row['CLIMATE8'].to_i }
       store 'area',                   :field_name => 'SQFT8',   :from_units => :square_feet, :to_units => :square_metres
       store 'principal_activity',     :field_name => 'PBA8',    :dictionary => { :input => 'principal_activity_code', :output => 'principal_activity', :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdHRGVHczYXRoU2dFLV90aDdET0dQLUE&single=true&gid=2&output=csv' }
       store 'floors',                 :field_name => 'NFLOOR8', :dictionary => { :input => 'floors_code', :output => 'floors', :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdHRGVHczYXRoU2dFLV90aDdET0dQLUE&single=true&gid=0&output=csv' }
@@ -27,9 +33,9 @@ CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
       store 'first_activity',        :field_name => 'ACT18',    :dictionary => { :input => 'activity_code', :output => 'activity', :url => 'https://docs.google.com/spreadsheet/pub?&key=0AoQJbWqPrREqdHRGVHczYXRoU2dFLV90aDdET0dQLUE&single=true&gid=4&output=csv' }
       store 'second_activity',       :field_name => 'ACT28',    :dictionary => { :input => 'activity_code', :output => 'activity', :url => 'https://docs.google.com/spreadsheet/pub?&key=0AoQJbWqPrREqdHRGVHczYXRoU2dFLV90aDdET0dQLUE&single=true&gid=4&output=csv' }
       store 'third_activity',        :field_name => 'ACT38',    :dictionary => { :input => 'activity_code', :output => 'activity', :url => 'https://docs.google.com/spreadsheet/pub?&key=0AoQJbWqPrREqdHRGVHczYXRoU2dFLV90aDdET0dQLUE&single=true&gid=4&output=csv' }
-      store 'first_activity_share',  :synthesize => lambda { |row| row['ACT1PCT8'].blank? ? nil : row['ACT1PCT8'].to_f / 100.0 }
-      store 'second_activity_share', :synthesize => lambda { |row| row['ACT2PCT8'].blank? ? nil : row['ACT2PCT8'].to_f / 100.0 }
-      store 'third_activity_share',  :synthesize => lambda { |row| row['ACT3PCT8'].blank? ? nil : row['ACT3PCT8'].to_f / 100.0 }
+      store 'first_activity_share',  :synthesize => Proc.new { |row| row['ACT1PCT8'].blank? ? nil : row['ACT1PCT8'].to_f / 100.0 }
+      store 'second_activity_share', :synthesize => Proc.new { |row| row['ACT2PCT8'].blank? ? nil : row['ACT2PCT8'].to_f / 100.0 }
+      store 'third_activity_share',  :synthesize => Proc.new { |row| row['ACT3PCT8'].blank? ? nil : row['ACT3PCT8'].to_f / 100.0 }
       store 'lodging_rooms',         :field_name => 'LODGRM8'
     end
     
@@ -40,7 +46,7 @@ CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
       key 'id', :field_name => 'PUBID8'
       store 'heating_degree_days', :field_name => 'HDD658'
       store 'cooling_degree_days', :field_name => 'CDD658'
-      store 'electricity_use',     :field_name => 'ELCNS8', :units => :kilowatt_hours
+      store 'electricity_use',     :synthesize => Proc.new { |row| row['ELCNS8'].to_i }, :units => :kilowatt_hours
     end
     
     import 'fuel use characteristics from the 2003 EIA Commercial Building Energy Consumption Survey',
@@ -48,9 +54,9 @@ CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
            :skip => 1,
            :headers => ["PUBID8", "REGION8", "CENDIV8", "SQFT8", "SQFTC8", "YRCONC8", "PBA8", "ELUSED8", "NGUSED8", "FKUSED8", "PRUSED8", "STUSED8", "HWUSED8", "ADJWT8", "STRATUM8", "PAIR8", "NGCNS8", "NGBTU8", "NGEXP8", "ZNGCNS8", "ZNGEXP8", "FKCNS8", "FKBTU8", "FKEXP8", "ZFKCNS8", "ZFKEXP8", "DHUSED8", "DHHT18", "DHHT28", "DHCOOL8", "DHWATR8", "DHCOOK8", "DHMANU8", "DHOTH8", "DHCNS8", "DHBTU8", "DHEXP8", "ZDHCNS8", "ZDHEXP8"] do
       key 'id', :field_name => 'PUBID8'
-      store 'natural_gas_use',   :synthesize => lambda { |row| row['NGCNS8'].to_i == 0 ? nil : (row['NGCNS8'].to_f * 100.0).cubic_feet.to(:cubic_metres) }, :units => :cubic_metres
-      store 'fuel_oil_use',      :synthesize => lambda { |row| row['FKCNS8'].to_i == 0 ? nil : row['FKCNS8'].to_f.gallons.to(:litres) }, :units => :litres
-      store 'district_heat_use', :synthesize => lambda { |row| row['DHBTU8'].to_i == 0 ? nil : row['DHBTU8'].to_f.kbtus.to(:megajoules) }, :units => :megajoules
+      store 'natural_gas_use',   :field_name => 'NGCNS8', :from_units => :hundred_cubic_feet, :to_units => :cubic_metres #!!!!!!!!!!!!
+      store 'fuel_oil_use',      :field_name => 'FKCNS8', :from_units => :gallons,            :to_units => :litres
+      store 'district_heat_use', :field_name => 'DHBTU8', :from_units => :kbtus,              :to_units => :megajoules
     end
   end
 end

data/lib/earth/hospitality/country_lodging_class.rb

@@ -1,3 +1,4 @@
+require 'earth/locality'
 class CountryLodgingClass < ActiveRecord::Base
   set_primary_key :name
   

data/lib/earth/hospitality/country_lodging_class/data_miner.rb

@@ -1,3 +1,4 @@
+require 'earth/locality/data_miner'
 CountryLodgingClass.class_eval do
   data_miner do
     import "a curated list of country lodging classes",
@@ -18,11 +19,12 @@ CountryLodgingClass.class_eval do
         intensities = {}
         
         [:natural_gas, :fuel_oil, :electricity, :district_heat].each do |fuel|
-          intensities[fuel] = cbecs_responses.inject(0) do |sum, response|
+          intensity = cbecs_responses.inject(0) do |sum, response|
             next sum unless response.send("#{fuel}_use").present?
             occupied_room_nights = 365.0 / 7.0 / 12.0 * response.months_used * response.weekly_hours / 24.0 * response.lodging_rooms * 0.59
             sum + (response.weighting * response.send("#{fuel}_use") / occupied_room_nights)
-          end / cbecs_responses.sum(:weighting)
+          end
+          intensities[fuel] = intensity / cbecs_responses.sum(:weighting)
         end
         
         lodging_class.natural_gas_intensity         = intensities[:natural_gas]

data/lib/earth/hospitality/data_miner.rb

@@ -1,4 +1,3 @@
-require 'earth/hospitality/census_region_lodging_class/data_miner'
-require 'earth/hospitality/commercial_building_energy_consumption_survey_response/data_miner'
-require 'earth/hospitality/country_lodging_class/data_miner'
-require 'earth/hospitality/lodging_class/data_miner'
+# lib/earth/hospitality/data_miner.rb
+require 'earth'
+Earth.require_related __FILE__

data/lib/earth/hospitality/lodging_class.rb

@@ -1,15 +1,6 @@
+require 'earth/locality'
 class LodgingClass < ActiveRecord::Base
   set_primary_key :name
   
   col :name
-  
-  # DEPRECATED - cut once new Lodging has been phased in
-  col :electricity_intensity, :type => :float
-  col :electricity_intensity_units
-  col :natural_gas_intensity, :type => :float
-  col :natural_gas_intensity_units
-  col :fuel_oil_intensity, :type => :float
-  col :fuel_oil_intensity_units
-  col :district_heat_intensity, :type => :float
-  col :district_heat_intensity_units
 end

data/lib/earth/hospitality/lodging_class/data_miner.rb

@@ -1,37 +1,14 @@
+require 'earth/locality/data_miner'
 LodgingClass.class_eval do
   data_miner do
-    # DEPRECATED - once new Lodging is phased in replace this with an import of lodging class names from the CountryLodgingClass google doc
-    import "a list of lodging classes and pre-calculated energy intensities",
-           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGZZWmZtWEJlYzhRNXlPdWpBTldlcUE&output=csv' do
-      key   'name'
-      store 'electricity_intensity',   :units_field_name => 'electricity_intensity_units'
-      store 'natural_gas_intensity',   :units_field_name => 'natural_gas_intensity_units'
-      store 'fuel_oil_intensity',      :units_field_name => 'fuel_oil_intensity_units'
-      store 'district_heat_intensity', :units_field_name => 'district_heat_intensity_units'
+    process "Ensure CountryLodgingClass is populated" do
+      CountryLodgingClass.run_data_miner!
     end
     
-    process "Convert natural gas intensities to metric units" do
-      conversion_factor = 2.83168466 # Google: 2.83168466 cubic m / 100 cubic ft
-      where(:natural_gas_intensity_units => 'hundred_cubic_feet_per_room_night').update_all(%{
-        natural_gas_intensity = 1.0 * natural_gas_intensity * #{conversion_factor},
-        natural_gas_intensity_units = 'cubic_metres_per_room_night'
-      })
-    end
-    
-    process "Convert fuel oil intensities to metric units" do
-      conversion_factor = 3.78541178 # Google: 3.78541178 l / gal
-      where(:fuel_oil_intensity_units => 'gallons_per_room_night').update_all(%{
-        fuel_oil_intensity = 1.0 * fuel_oil_intensity * #{conversion_factor},
-        fuel_oil_intensity_units = 'litres_per_room_night'
-      })
-    end
-    
-    process "Convert district heat intensities to metric units" do
-      conversion_factor = 1.05505585 # Google: 1.05505585 MJ / 1000 Btu
-      where(:district_heat_intensity_units => 'thousand_btu_per_room_night').update_all(%{
-        district_heat_intensity = 1.0 * district_heat_intensity * #{conversion_factor},
-        district_heat_intensity_units = 'megajoules_per_room_night'
-      })
+    process "Derive lodging classes from CountryLodgingClass" do
+      connection.select_values("SELECT DISTINCT lodging_class_name FROM #{CountryLodgingClass.quoted_table_name}").each do |lodging_class_name|
+        find_or_create_by_name lodging_class_name
+      end
     end
   end
 end

data/lib/earth/hospitality/lodging_property.rb

@@ -1,3 +1,4 @@
+require 'earth/locality'
 # Copyright 2011 Brighter Planet, Inc.
 class LodgingProperty < ActiveRecord::Base
   set_primary_key :northstar_id

data/lib/earth/industry.rb

@@ -1,13 +1,3 @@
-require 'earth/industry/cbecs_energy_intensity'
-require 'earth/industry/industry_product_line'
-require 'earth/industry/industry_sector'
-require 'earth/industry/industry'
-require 'earth/industry/industry_product'
-require 'earth/industry/mecs_energy'
-require 'earth/industry/mecs_ratio'
-require 'earth/industry/merchant'
-require 'earth/industry/merchant_category_industry'
-require 'earth/industry/merchant_category'
-require 'earth/industry/product_line'
-require 'earth/industry/product_line_industry_product'
-require 'earth/industry/sector'
+# lib/earth/industry.rb
+require 'earth'
+Earth.require_related __FILE__

data/lib/earth/industry/cbecs_energy_intensity.rb

@@ -1,55 +1,11 @@
+require 'earth/locality'
 class CbecsEnergyIntensity < ActiveRecord::Base
-  col :naics_code, :type => :string
+  set_primary_key :name
+  
+  col :name
+  col :naics_code
   col :census_division_number, :type => :integer
   col :total_electricity_consumption, :type => :float
   col :total_floorspace, :type => :float
   col :electricity_intensity, :type => :float
-
-  CENSUS_DIVISIONS = {
-    'New England Division' => {
-      :column => 0,
-      :code => 1,
-      :table => 'c17'
-    },
-    'Middle Atlantic Division' => {
-      :column => 1,
-      :code => 2,
-      :table => 'c17'
-    },
-    'East North Central Division' => {
-      :column => 2,
-      :code => 3,
-      :table => 'c17'
-    },
-    'West North Central Division' => {
-      :column => 0,
-      :code => 4,
-      :table => 'c18'
-    },
-    'South Atlantic Division' => {
-      :column => 1,
-      :code => 5,
-      :table => 'c18'
-    },
-    'East South Central Division' => {
-      :column => 2,
-      :code => 6,
-      :table => 'c18'
-    },
-    'West South Central Division' => {
-      :column => 0,
-      :code => 7,
-      :table => 'c19'
-    },
-    'Mountain Division' => {
-      :column => 1,
-      :code => 8,
-      :table => 'c19'
-    },
-    'Pacific Division' => {
-      :column => 2,
-      :code => 9,
-      :table => 'c19'
-    }
-  }
 end

data/lib/earth/industry/cbecs_energy_intensity/data_miner.rb

@@ -1,48 +1,100 @@
+require 'earth/eia'
+require 'earth/locality/data_miner'
 CbecsEnergyIntensity.class_eval do
-  data_miner do
-    NAICS_CODE_SYNTHESIZER = Proc.new do |row|
-      case row[0].gsub(/\./, '')
-      when 'Education' then
-        611110
-      when 'Food Sales' then
-        445
-      when 'Food Service' then
-        722
-      when 'Health Care' then
-        622110
-      when 'Inpatient' then
-        622110
-      when 'Outpatient' then
-        622111
-      when 'Lodging' then
-        721
-      when 'Retail (Other Than Mall)' then
-        44
-      when 'Office' then
-        #TODO
-      when 'Public Assembly' then
-        #TODO
-      when 'Public Order and Safety' then
-        922120
-      when 'Religious Worship' then
-        813110
-      when 'Service' then
-        #TODO
-      when 'Warehouse and Storage' then
-        493110
-      when 'Other' then
-        #TODO
-      when 'Vacant' then
-        #TODO
-      end
+  const_set(:CENSUS_DIVISIONS, {
+    'New England Division' => {
+      :column => 0,
+      :code => 1,
+      :table => 'c17'
+    },
+    'Middle Atlantic Division' => {
+      :column => 1,
+      :code => 2,
+      :table => 'c17'
+    },
+    'East North Central Division' => {
+      :column => 2,
+      :code => 3,
+      :table => 'c17'
+    },
+    'West North Central Division' => {
+      :column => 0,
+      :code => 4,
+      :table => 'c18'
+    },
+    'South Atlantic Division' => {
+      :column => 1,
+      :code => 5,
+      :table => 'c18'
+    },
+    'East South Central Division' => {
+      :column => 2,
+      :code => 6,
+      :table => 'c18'
+    },
+    'West South Central Division' => {
+      :column => 0,
+      :code => 7,
+      :table => 'c19'
+    },
+    'Mountain Division' => {
+      :column => 1,
+      :code => 8,
+      :table => 'c19'
+    },
+    'Pacific Division' => {
+      :column => 2,
+      :code => 9,
+      :table => 'c19'
+    }
+  })
+  
+  const_set(:NAICS_CODE_SYNTHESIZER, lambda { |row|
+    case row[0].to_s
+    when /Education/
+      611110
+    when /Food Sales/
+      445
+    when /Food Service/
+      722
+    when /Health Care/
+      622110
+    when /Inpatient/
+      622110
+    when /Outpatient/
+      622111
+    when /Lodging/
+      721
+    when /Retail (Other Than Mall)/
+      44
+    when /Office/
+      #TODO
+    when /Public Assembly/
+      #TODO
+    when /Public Order and Safety/
+      922120
+    when /Religious Worship/
+      813110
+    when /Service/
+      #TODO
+    when /Warehouse and Storage/
+      493110
+    when /Other/
+      #TODO
+    when /Vacant/
+      #TODO
     end
-
+  })
+  
+  data_miner do
     CbecsEnergyIntensity::CENSUS_DIVISIONS.each do |division, data|
-      import "2003 CBECS #{data[:table].capitalize} - Electricity Consumption and Intensity - #{division}",
+      import "2003 CBECS #{data[:table].upcase} - Electricity Consumption and Intensity - #{division}",
         :url => "http://www.eia.gov/emeu/cbecs/cbecs2003/detailed_tables_2003/2003set10/2003excel/#{data[:table]}.xls",
-        :crop => (21..36) do
-        key :name, :synthesize => Proc.new { |row| "#{Industry.format_naics_code(row[0])}-#{data[:code]}" }
-        store :naics_code, :synthesize =>  NAICS_CODE_SYNTHESIZER
+        :headers => false,
+        :select => ::Proc.new { |row| CbecsEnergyIntensity::NAICS_CODE_SYNTHESIZER.call(row) },
+        :crop => (21..37) do
+        key :name, :synthesize => ::Proc.new { |row| "#{Industry.format_naics_code(CbecsEnergyIntensity::NAICS_CODE_SYNTHESIZER.call(row))}-#{data[:code]}" }
+        store :naics_code, :synthesize => CbecsEnergyIntensity::NAICS_CODE_SYNTHESIZER
         store :census_division_number, :static => data[:code]
         store :total_electricity_consumption, :field_number => data[:column] + 1
         store :total_floorspace, :field_number => data[:column] + 4
@@ -51,7 +103,7 @@ CbecsEnergyIntensity.class_eval do
     end
 
     process :normalize_fuels do
-      EIA.normalize(CbecsEnergyIntensity, [:total_electricity_consumption, :total_floorspace, :electricity_intensity])
+      Earth::EIA.normalize(CbecsEnergyIntensity, [:total_electricity_consumption, :total_floorspace, :electricity_intensity])
     end
   end
 end

data/lib/earth/industry/data_miner.rb

@@ -1,4 +1,3 @@
-require 'earth/industry/cbecs_energy_intensity/data_miner'
-require 'earth/industry/industry/data_miner'
-require 'earth/industry/mecs_energy/data_miner'
-require 'earth/industry/mecs_ratio/data_miner'
+# lib/earth/industry/data_miner.rb
+require 'earth'
+Earth.require_related __FILE__