earth 0.11.13 → 0.11.14

data/README.markdown

@@ -83,7 +83,7 @@ ft = AutomobileFuel.first
 
 ### Data storage
 
-You can store Earth data in any relational database. On your very first run, you will need to create the tables for data each model. This is done using minirecord with the `apply_schemas` option:
+You can store Earth data in any relational database. On your very first run, you will need to create the tables for data each model. This is done using [the `active_record_inline_schema` library](https://github.com/seamusabshere/active_record_inline_schema) if you pass the `apply_schemas` option:
 
 ``` ruby
 require 'activerecord'

data/bin/earth_tester.rb

@@ -16,7 +16,7 @@ require 'active_record'
 
 case ENV['EARTH_DB_ADAPTER']
 when 'mysql'
-  adapter = 'mysql'
+  adapter = 'mysql2'
   database = 'test_earth'
   username = 'root'
   password = 'password'

data/earth.gemspec

@@ -5,7 +5,6 @@ require "earth/version"
 Gem::Specification.new do |s|
   s.name        = "earth"
   s.version     = Earth::VERSION
-  s.date = "2012-03-05"
   s.platform    = Gem::Platform::RUBY
   s.authors = ["Seamus Abshere", "Derek Kastner", "Andy Rossmeissl"]
   s.email = %q{andy@rossmeissl.net}
@@ -36,7 +35,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'fixed_width-multibyte'
   s.add_runtime_dependency 'fuzzy_match', '>=1.1.0'
   s.add_runtime_dependency 'geokit'
-  s.add_runtime_dependency 'mini_record-compat' # need https://github.com/DAddYE/mini_record/pull/7
+  s.add_runtime_dependency 'active_record_inline_schema'
   s.add_runtime_dependency 'remote_table', '>=1.3.0'
   s.add_runtime_dependency 'table_warnings', '>=0.0.6'
   s.add_runtime_dependency 'to_regexp'
@@ -45,7 +44,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'charisma'
   s.add_development_dependency 'cucumber'
-  s.add_development_dependency 'mysql' # for bin/earth_tester.rb
+  s.add_development_dependency 'mysql2' # for bin/earth_tester.rb; use mysql2 for utf-8 compatibility
   s.add_development_dependency 'pg'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'rdoc'

data/lib/earth.rb

@@ -7,7 +7,7 @@ require 'falls_back_on'
 require 'weighted_average'
 require 'fixed_width'
 require 'errata'
-require 'mini_record'
+require 'active_record_inline_schema'
 require 'table_warnings'
 require 'fuzzy_match'
 require 'fuzzy_match/cached_result'

data/lib/earth/air/aircraft/data_miner.rb

@@ -22,6 +22,10 @@ Aircraft.class_eval do
   end
   
   data_miner do
+    process "Start from scratch" do
+      delete_all
+    end
+    
     ('A'..'Z').each do |letter|
       import("aircraft made by whitelisted manufacturers whose ICAO code starts with '#{letter}' from the FAA",
              :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",

data/lib/earth/air/airline/data_miner.rb

@@ -1,6 +1,10 @@
 Airline.class_eval do
   data_miner do
-    import "a Brighter Planet-curated list of airlines and codes not included in our other sources",
+    process "Start from scratch" do
+      delete_all
+    end
+    
+    import "a Brighter Planet-curated list of airlines and codes not included in our proprietary sources",
            :url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdGJoaFpENXRqMEM2NW42am5tNURGU2c&output=csv' do
       key 'name'
       store 'bts_code',  :nullify => true

data/lib/earth/air/airport/data_miner.rb

@@ -17,6 +17,10 @@ Airport.class_eval do
   end
   
   data_miner do
+    process "Start from scratch" do
+      delete_all
+    end
+    
     import "the OpenFlights.org airports database",
            :url => 'https://openflights.svn.sourceforge.net/svnroot/openflights/openflights/data/airports.dat',
            :headers => %w{ id name city country_name iata_code icao_code latitude longitude altitude timezone daylight_savings },

data/lib/earth/air/flight_segment.rb

@@ -21,8 +21,8 @@ class FlightSegment < ActiveRecord::Base
       # Assume a load factor of 1 where passengers > available seats
       where('passengers > seats AND seats > 0').update_all 'load_factor = 1'
       
-      # FIXME TODO: what is 90.718474
-      # Derive freight share as a fraction of the total weight carried
+      # Derive freight share as a fraction of the total weight carried -- assume 90.718474 kg (200 lbs) per passenger (includes checked baggage)
+      # FIXME TODO assume 100 kg per passenger?
       where('(freight + mail + passengers) > 0').update_all 'freight_share = 1.0 * (freight + mail) / (freight + mail + (passengers * 90.718474))'
       
       # Derive average seats per flight

data/lib/earth/air/flight_segment/data_miner.rb

@@ -183,6 +183,10 @@ FlightSegment.class_eval do
   end
   
   data_miner do
+    process "Start from scratch" do
+      delete_all
+    end
+    
     form_data_per_month(2009..::Time.now.year).each do |month, form_data|
       import "T100 flight segment data for #{month.strftime('%B %Y')}",
              :url => URL,

data/lib/earth/computation/computation_carrier_instance_class.rb

@@ -2,7 +2,7 @@ require 'earth/locality'
 class ComputationCarrierInstanceClass < ActiveRecord::Base
   self.primary_key = "name"
   
-  belongs_to :computation_carrier, :foreign_key => 'computation_carrier_name'
+  belongs_to :carrier, :class_name => 'ComputationCarrier', :foreign_key => 'computation_carrier_name'
   
   falls_back_on :name => 'fallback',
                 :electricity_intensity => lambda { ComputationCarrierInstanceClass.find_by_name('Amazon m1.small').electricity_intensity },

data/lib/earth/hospitality/commercial_building_energy_consumption_survey_response.rb

@@ -45,9 +45,12 @@ class CommercialBuildingEnergyConsumptionSurveyResponse < ActiveRecord::Base
   col :fuel_oil_energy_units
   col :district_heat_use,      :type => :float
   col :district_heat_use_units
+  col :district_heat_energy,   :type => :float
+  col :district_heat_energy_units
   col :stratum,                :type => :integer
   col :pair,                   :type => :integer
   col :weighting,              :type => :float
+  col :room_nights,            :type => :float
   col :electricity_per_room_night, :type => :float   # for lodging fuzzy weighting
   col :electricity_per_room_night_units              # for lodging fuzzy weighting
   col :natural_gas_per_room_night, :type => :float   # for lodging fuzzy weighting
@@ -56,12 +59,4 @@ class CommercialBuildingEnergyConsumptionSurveyResponse < ActiveRecord::Base
   col :fuel_oil_per_room_night_units                 # for lodging fuzzy weighting
   col :district_heat_per_room_night, :type => :float # for lodging fuzzy weighting
   col :district_heat_per_room_night_units            # for lodging fuzzy weighting
-  col :electricity_energy_intensity, :type => :float
-  col :electricity_energy_intensity_units
-  col :natural_gas_energy_intensity, :type => :float
-  col :natural_gas_energy_intensity_units
-  col :fuel_oil_energy_intensity, :type => :float
-  col :fuel_oil_energy_intensity_units
-  col :district_heat_energy_intensity, :type => :float
-  col :district_heat_energy_intensity_units
 end

data/lib/earth/hospitality/commercial_building_energy_consumption_survey_response/data_miner.rb

@@ -64,32 +64,23 @@ CommercialBuildingEnergyConsumptionSurveyResponse.class_eval do
       store 'fuel_oil_use',         :field_name => 'FKCNS8', :from_units => :gallons,            :to_units => :litres
       store 'fuel_oil_energy',      :field_name => 'FKBTU8', :from_units => :kbtus,              :to_units => :megajoules
       store 'district_heat_use',    :field_name => 'DHBTU8', :from_units => :kbtus,              :to_units => :megajoules
+      store 'district_heat_energy', :field_name => 'DHBTU8', :from_units => :kbtus,              :to_units => :megajoules
     end
     
-    process "Derive fuel intensities per room night for lodging buildings" do
-      # days/year * weeks/day * years/month = weeks/month
-      # weeks/month * months used in a year = weeks used in a year
-      # weeks used in a year * hours/week = hours used in a year
-      # hours used in a year * days/hour = days used in a year
+    process "Derive room nights for lodging records" do
+      lodging_records.
+        #                    nights/year * weeks/night * years/month  * months      * hours/week   * nights/hour * rooms = room-nights
+        #                               => weeks/year => weeks/month => weeks      => hours       => nights     => room-nights
+        update_all "room_nights = (365.0 * (1 / 7.0)   * (1 / 12.0)   * months_used * weekly_hours * (1 / 24.0)  * lodging_rooms)"
+    end
+    
+    process "Derive fuel intensities per room night for lodging records" do
       [:natural_gas, :fuel_oil, :electricity, :district_heat].each do |fuel|
         lodging_records.update_all %{
-          #{fuel}_per_room_night = #{fuel}_use / (365.0 / 7.0 / 12.0 * months_used * weekly_hours / 24.0 * lodging_rooms),
+          #{fuel}_per_room_night = #{fuel}_use / room_nights,
           #{fuel}_per_room_night_units = #{fuel}_use_units || '_per_room_night'
         }
       end
     end
-    
-    process "Derive fuel intensities per unit area" do
-      [:natural_gas, :fuel_oil, :electricity].each do |fuel|
-        update_all %{
-          #{fuel}_energy_intensity = #{fuel}_energy / area,
-          #{fuel}_energy_intensity_units = 'megajoules_per_square_metre'
-        }
-        update_all %{
-          district_heat_energy_intensity = district_heat_use / area,
-          district_heat_energy_intensity_units = 'megajoules_per_square_metre'
-        }
-      end
-    end
   end
 end

data/lib/earth/hospitality/country_lodging_class/data_miner.rb

@@ -16,30 +16,24 @@ CountryLodgingClass.class_eval do
     
     process "Calculate US lodging class fuel intensities from CommercialBuildingEnergyConsumptionSurveyResponse" do
       occupancy_rate = Country.united_states.lodging_occupancy_rate
-      connection.select_values("SELECT DISTINCT cbecs_detailed_activity FROM #{CountryLodgingClass.quoted_table_name}").each do |cbecs_activity|
-        [:natural_gas, :fuel_oil, :electricity, :district_heat].each do |fuel|
-          where(:cbecs_detailed_activity => cbecs_activity).update_all(%{
-            #{fuel}_intensity = (
-              SELECT SUM(
-                weighting * #{fuel}_use / (365.0 / 7.0 / 12.0 * months_used * weekly_hours / 24.0 * lodging_rooms * #{occupancy_rate})
-              ) / SUM(weighting)
-              FROM #{CommercialBuildingEnergyConsumptionSurveyResponse.quoted_table_name}
-              WHERE detailed_activity = '#{cbecs_activity}'
-            )
-          })
+      lodging_records = CommercialBuildingEnergyConsumptionSurveyResponse.lodging_records
+      
+      Country.united_states.lodging_classes.map(&:cbecs_detailed_activity).uniq.each do |cbecs_activity|
+        %w{natural_gas fuel_oil electricity district_heat}.each do |fuel|
+          cbecs_column = (fuel + '_use').to_sym
+          
+          where(:cbecs_detailed_activity => cbecs_activity).update_all %{
+            #{fuel}_intensity = (#{
+              lodging_records.where(:detailed_activity => cbecs_activity).
+                weighted_average_relation(cbecs_column, :disaggregate_by => :room_nights, :weighted_by => :weighting).to_sql
+            }) / #{occupancy_rate},
+            #{fuel}_intensity_units = '#{lodging_records.first.send("#{fuel}_use_units")}_per_occupied_room_night',
+            weighting = (#{
+              lodging_records.where(:detailed_activity => cbecs_activity).sum(:weighting) /
+              where(:cbecs_detailed_activity => cbecs_activity).count
+            })
+          }
         end
-        
-        where(:cbecs_detailed_activity => cbecs_activity).update_all(%{
-          natural_gas_intensity_units = 'cubic_metres_per_room_night',
-          fuel_oil_intensity_units    = 'litres_per_room_night',
-          electricity_intensity_units = 'kilowatt_hours_per_room_night',
-          district_heat_intensity_units = 'megajoules_per_room_night',
-          weighting = (
-            SELECT SUM(weighting)
-            FROM #{CommercialBuildingEnergyConsumptionSurveyResponse.quoted_table_name}
-            WHERE detailed_activity = '#{cbecs_activity}'
-          ) / #{where(:cbecs_detailed_activity => cbecs_activity).count}
-        })
       end
     end
   end

data/lib/earth/hospitality/lodging_property.rb

@@ -26,4 +26,8 @@ class LodgingProperty < ActiveRecord::Base
   col :pools_indoor,      :type => :float # float b/c fallback needs to be a float
   col :pools_outdoor,     :type => :float # float b/c fallback needs to be a float
   col :update_date
+  add_index :city
+  add_index :locality
+  add_index :postcode
+  add_index :country_iso_3166_alpha_3_code
 end

data/lib/earth/industry/mecs_energy.rb

@@ -28,21 +28,30 @@ class MecsEnergy < ActiveRecord::Base
   FUELS = [:electricity, :residual_fuel_oil, :distillate_fuel_oil,
            :natural_gas, :lpg_and_ngl, :coal, :coke_and_breeze, :other_fuel]
   
-   # Find the first record whose naics_code matches code.
+   # Find the first record whose naics_code matches code and that has valid fuel ratios.
    # If no record found chop off the last character of code and try again, and so on.
    def self.find_by_naics_code(code)
      find_by_naics_code_and_census_region_number(code, nil)
    end
    
-  # Find the first record whose census_region_number matches number and whose naics_code matches code.
-  # If no record found chop off the last character of code and try again, and so on.
-  def self.find_by_naics_code_and_census_region_number(code, number)
+   # Find the first record whose census_region_number matches number, whose naics_code matches code, and that has valid fuel ratios.
+   # If none found and we know census region number, try looking nationwide
+   # If none found and looking nationwide, chop off the last character of code and try again looking in census region
+   # And so on
+  def self.find_by_naics_code_and_census_region_number(code, number, original_number = number)
     if code.blank?
       record = nil
     else
       code = Industry.format_naics_code code
-      record = where(:census_region_number => number, :naics_code => code).first
-      record ||= find_by_naics_code_and_census_region_number(code[0..-2], number)
+      candidate = where(:census_region_number => number, :naics_code => code).first
+      
+      if candidate.try(:fuel_ratios).present?
+        record = candidate
+      elsif number.present?
+        record = find_by_naics_code_and_census_region_number(code, nil, original_number)
+      else
+        record = find_by_naics_code_and_census_region_number(code[0..-2], original_number)
+      end
     end
     record
   end

data/lib/earth/industry/mecs_ratio.rb

@@ -12,21 +12,30 @@ class MecsRatio < ActiveRecord::Base
   col :energy_per_dollar_of_shipments, :type => :float
   col :energy_per_dollar_of_shipments_units
   
-  # Find the first record whose naics_code matches code.
+  # Find the first record whose naics_code matches code and whose energy per dollar shipment is present.
   # If no record found chop off the last character of code and try again, and so on.
   def self.find_by_naics_code(code)
-    find_by_naics_code_and_census_region_number(code, nil)
+    candidate = find_by_naics_code_and_census_region_number(code, nil)
   end
   
-  # Find the first record whose census_region_number matches number and whose naics_code matches code.
-  # If none found, chop off the last character of code and try again, and so on.
-  def self.find_by_naics_code_and_census_region_number(code, number)
+  # Find the first record whose census_region_number matches number, whose naics_code matches code, and whose energy per dollar of shipments is present.
+  # If none found and we know census region number, try looking nationwide
+  # If none found and looking nationwide, chop off the last character of code and try again looking in census region
+  # And so on
+  def self.find_by_naics_code_and_census_region_number(code, number, original_number = number)
     if code.blank?
       record = nil
     else
       code = Industry.format_naics_code code
-      record = where(:census_region_number => number, :naics_code => code).first
-      record ||= find_by_naics_code_and_census_region_number(code[0..-2], number)
+      candidate = where(:census_region_number => number, :naics_code => code).first
+      
+      if candidate.try(:energy_per_dollar_of_shipments).present?
+        record = candidate
+      elsif number.present?
+        record = find_by_naics_code_and_census_region_number(code, nil, original_number)
+      else
+        record = find_by_naics_code_and_census_region_number(code[0..-2], original_number)
+      end
     end
     record
   end

data/lib/earth/industry/sic_1987.rb

@@ -4,7 +4,6 @@ class Sic1987 < ActiveRecord::Base
   
   has_many :naics_2002_sic_1987_concordances, :foreign_key => :sic_1987_code
   has_many :naics_2002, :through => :naics_2002_sic_1987_concordances
-  has_many :industries, :through => :naics_2002
   
   # for data import
   def self.format_description(description)

data/lib/earth/locality/country/data_miner.rb

@@ -9,28 +9,21 @@ Country.class_eval do
     import "OpenGeoCode.org's Country Codes to Country Names list",
            :url => 'http://opengeocode.org/download/countrynames.txt',
            :format => :delimited,
-           :delimiter => ';',
+           :delimiter => '; ',
            :headers => false,
            :skip => 22 do
       key 'iso_3166_code', :field_number => 0
       store 'iso_3166_alpha_3_code', :field_number => 1
       store 'iso_3166_numeric_code', :field_number => 2
-      store 'name', :field_number => 5 # romanized version
+      store 'name', :field_number => 5 # romanized version with utf-8 characters
     end
     
-    import "heating degree day data from WRI CAIT",
+    import "heating and cooling degree day data from WRI CAIT",
            :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDN4MkRTSWtWRjdfazhRdWllTkVSMkE&output=csv',
            :select => Proc.new { |record| record['country'] != 'European Union (27)' },
            :errata => { :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDNSMUtCV0h4cUF4UnBKZlNkczlNbFE&output=csv' } do
       key 'name', :field_name => 'country'
       store 'heating_degree_days', :units => :degrees_celsius
-    end
-    
-    import "cooling degree day data from WRI CAIT",
-           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDN4MkRTSWtWRjdfazhRdWllTkVSMkE&output=csv',
-           :select => Proc.new { |record| record['country'] != 'European Union (27)' },
-           :errata => { :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDNSMUtCV0h4cUF4UnBKZlNkczlNbFE&output=csv' } do
-      key 'name', :field_name => 'country'
       store 'cooling_degree_days', :units => :degrees_celsius
     end
     
@@ -142,13 +135,13 @@ Country.class_eval do
       find_each do |country|
         if country.lodging_classes.any?
           country.lodging_natural_gas_intensity = country.lodging_classes.weighted_average(:natural_gas_intensity)
-          country.lodging_natural_gas_intensity_units = 'cubic_metres_per_room_night' # FIXME TODO derive this
+          country.lodging_natural_gas_intensity_units = 'cubic_metres_per_occupied_room_night' # FIXME TODO derive this
           country.lodging_fuel_oil_intensity = country.lodging_classes.weighted_average(:fuel_oil_intensity)
-          country.lodging_fuel_oil_intensity_units = 'gallons_per_room_night' # FIXME TODO derive this
+          country.lodging_fuel_oil_intensity_units = 'gallons_per_occupied_room_night' # FIXME TODO derive this
           country.lodging_electricity_intensity = country.lodging_classes.weighted_average(:electricity_intensity)
-          country.lodging_electricity_intensity_units = 'kilowatt_hours_per_room_night' # FIXME TODO derive this
+          country.lodging_electricity_intensity_units = 'kilowatt_hours_per_occupied_room_night' # FIXME TODO derive this
           country.lodging_district_heat_intensity = country.lodging_classes.weighted_average(:district_heat_intensity)
-          country.lodging_district_heat_intensity_units = 'megajoules_per_room_night' # FIXME TODO derive this
+          country.lodging_district_heat_intensity_units = 'megajoules_per_occupied_room_night' # FIXME TODO derive this
           country.save!
         end
       end

data/lib/earth/locality/zip_code/data_miner.rb

@@ -9,7 +9,7 @@ ZipCode.class_eval do
   
   data_miner do
     import 'the Mapping Hacks zipcode database',
-           :url => 'http://mappinghacks.com/data/zipcode.zip',
+           :url => 'http://archive.data.brighterplanet.com/zipcode.zip', # http://mappinghacks.com/data/zipcode.zip'
            :filename => 'zipcode.csv' do
       key   'name', :field_name => 'zip', :sprintf => '%05d'
       store 'state_postal_abbreviation', :field_name => 'state'
@@ -32,13 +32,11 @@ ZipCode.class_eval do
       store 'climate_division_name'
     end
     
-    # TODO: download file directly from US census
     # NOTE: ZCTAs are not zip codes but are based on the most common zip code in the area they cover - see http://www.census.gov/geo/ZCTA/zcta.html
     import 'US Census 2010 zip code tabulation area populations',
-           :url => 'https://docs.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdHJsVjlKNXNLNEQyWENjZ1owd2hFS3c&output=csv',
-           :skip => 1 do
-      key 'name', :field_name => 'Id2', :sprintf => '%05d'
-      store 'population', :field_name => 'Total'
+           :url => 'http://www.census.gov/geo/www/2010census/zcta_rel/zcta_county_rel_10.txt' do
+      key 'name', :field_name => 'ZCTA5', :sprintf => '%05d'
+      store 'population', :field_name => 'ZPOP'
     end
     
     import 'misc zip code data not included in other sources',

data/lib/earth/version.rb

@@ -1,3 +1,3 @@
 module Earth
-  VERSION = "0.11.13"
+  VERSION = "0.11.14"
 end

data/spec/earth/air/aircraft_spec.rb

@@ -3,7 +3,7 @@ require 'earth/air/aircraft'
 require 'earth/air/aircraft/data_miner'
 
 describe Aircraft do
-  describe 'import', :slow => true do
+  describe 'import', :data_miner => true do
     it 'should treat empty cells as null' do
       Aircraft.auto_upgrade!
       Aircraft.run_data_miner!

data/spec/earth/automobile/automobile_fuel_spec.rb

@@ -3,7 +3,7 @@ require 'earth/automobile'
 require 'earth/automobile/data_miner'
 
 describe AutomobileFuel do
-  describe 'import', :slow => true do
+  describe 'import', :data_miner => true do
     it 'should import data without problems' do
       AutomobileFuel.run_data_miner!
       AutomobileFuel.all.count.should > 0