earth 0.11.18 → 0.11.19

data/DEVELOPERS.markdown

@@ -0,0 +1,19 @@
+# Developer notes
+
+## Explicitly requiring cross-domain dependencies
+
+Currently there's very little automatic resolution of dependencies. So, let's say you have
+
+    class ElectricMarket
+      belongs_to :state
+    end
+
+You will need to require *locality* in 2 places:
+
+    # lib/earth/electricity/electric_market.rb
+    require 'earth/locality'
+
+    # lib/earth/electricity/electric_market/data_miner.rb
+    require 'earth/locality/data_miner'
+
+If you don't, `State`'s data miner script will be empty when it is called by `ElectricMarket.run_data_miner_on_belongs_to_associations!`

data/Rakefile

@@ -17,29 +17,6 @@ task :console do
   IRB.start
 end
 
-require 'cucumber'
-require 'cucumber/rake/task'
-desc 'Run all cucumber tests'
-Cucumber::Rake::Task.new(:features) do |t|
-  if ENV['CUCUMBER_FORMAT']
-    t.cucumber_opts = "features --format #{ENV['CUCUMBER_FORMAT']}"
-  else
-    t.cucumber_opts = 'features --format pretty'
-  end
-end
-
-directory 'log/'
-task :features => 'log/'
-
-if RUBY_VERSION =~ /^1\.8/
-  desc "Run cucumber tests with RCov"
-  Cucumber::Rake::Task.new(:features_with_coverage) do |t|
-    t.cucumber_opts = "features --format pretty"
-    t.rcov = true
-    t.rcov_opts = ['--exclude', 'features']
-  end
-end
-
 require 'rspec/core/rake_task'
 desc "Run all examples"
 RSpec::Core::RakeTask.new(:examples) do |c|
@@ -59,11 +36,11 @@ if RUBY_VERSION =~ /^1\.8/
   end
 end
 
-task :test => [:features, :examples]
+task :test => :examples
 task :default => :test
 
-require 'rake/rdoctask'
 require 'earth/version'
+require 'rdoc/task'
 Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_dir = 'rdoc'
   rdoc.title = "earth #{Earth::VERSION}"

data/errata/electricity/electric_market/nrel_errata.csv

@@ -0,0 +1 @@
+date,name,email,type,section,action,x,y,condition,notes

data/errata/electricity/electric_utility/eia_errata.csv

@@ -0,0 +1 @@
+date,name,email,type,section,action,x,y,condition,notes

data/lib/earth.rb

@@ -72,7 +72,9 @@ module Earth
       ::Object.const_defined?(resource)
     end.each do |resource|
       resource_model = resource.constantize
-      resource_model.data_miner_script.append_once :process, :run_data_miner_on_parent_associations!
+      unless options[:skip_parent_associations]
+        resource_model.data_miner_script.append_once :process, :run_data_miner_on_parent_associations!
+      end
       if options[:load_data_miner]
         resource_model.data_miner_script.prepend_once :process, :auto_upgrade!
       else

data/lib/earth/air/flight_segment.rb

@@ -1,4 +1,5 @@
 require 'earth/locality'
+require 'fuzzy_match/cached_result'
 
 class FlightSegment < ActiveRecord::Base
   self.primary_key = "row_hash"

data/lib/earth/air/flight_segment/data_miner.rb

@@ -186,8 +186,7 @@ FlightSegment.class_eval do
     process "Start from scratch" do
       delete_all
     end
-    FlightSegment.form_data_per_month(2011..2011).each do |month, form_data|
-    # FlightSegment.form_data_per_month(2009..::Time.now.year).each do |month, form_data|
+    FlightSegment.form_data_per_month(2009..::Time.now.year).each do |month, form_data|
       import "T100 flight segment data for #{month.strftime('%B %Y')}",
              :url => URL,
              :form_data => form_data,

data/lib/earth/electricity.rb

@@ -0,0 +1,3 @@
+# lib/earth/electricity.rb
+require 'earth'
+Earth.require_related __FILE__

data/lib/earth/electricity/electric_market.rb

@@ -0,0 +1,12 @@
+require 'earth/locality'
+
+class ElectricMarket < ActiveRecord::Base
+  self.primary_key = "id" # string
+  
+  belongs_to :electric_utility, :foreign_key => 'electric_utility_eia_id'
+  belongs_to :zip_code, :foreign_key => 'zip_code_name'
+  
+  col :id
+  col :electric_utility_eia_id, :type => :integer
+  col :zip_code_name
+end

data/lib/earth/electricity/electric_market/data_miner.rb

@@ -0,0 +1,21 @@
+require 'earth/locality/data_miner'
+
+ElectricMarket.class_eval do
+  data_miner do
+    import 'IOU markets',
+           :url => 'http://en.openei.org/datasets/files/899/pub/iou_zipcodes.csv',
+           :errata => { :url => "file://#{Earth::ERRATA_DIR}/electricity/electric_market/nrel_errata.csv" } do
+      key 'id', :synthesize => proc { |row| row['compid'].to_i.to_s + '-' + row['zip'] }
+      store 'zip_code_name', :field_name => 'zip'
+      store 'electric_utility_eia_id', :field_name => 'compid'
+    end
+    
+    import 'Non-IOU markets',
+           :url => 'http://en.openei.org/datasets/files/899/pub/non-iou_zipcodes.csv',
+           :errata => { :url => "file://#{Earth::ERRATA_DIR}/electricity/electric_market/nrel_errata.csv" } do
+      key 'id', :synthesize => proc { |row| row['compid'].to_i.to_s + '-' + row['zip'] }
+      store 'zip_code_name', :field_name => 'zip'
+      store 'electric_utility_eia_id', :field_name => 'compid'
+    end
+  end
+end

data/lib/earth/electricity/electric_utility.rb

@@ -0,0 +1,24 @@
+require 'earth/locality'
+
+class ElectricUtility < ActiveRecord::Base
+  self.primary_key = "eia_id"
+  
+  belongs_to :state, :foreign_key => 'state_postal_abbreviation'
+  has_many :electric_markets
+  has_many :zip_codes, :through => :electric_markets
+  
+  col :eia_id, :type => :integer
+  col :name
+  col :nickname
+  col :state_postal_abbreviation
+  col :nerc_region_abbreviation
+  col :second_nerc_region_abbreviation
+
+  def green_button_implementer?
+    GreenButtonAdoption.implemented? name, nickname 
+  end
+
+  def green_button_committer?
+    GreenButtonAdoption.committed? name, nickname
+  end
+end

data/lib/earth/electricity/electric_utility/data_miner.rb

@@ -0,0 +1,26 @@
+require 'earth/locality/data_miner'
+
+ElectricUtility.class_eval do
+  data_miner do
+    process "make sure green button adoption is populated" do
+      GreenButtonAdoption.run_data_miner!
+    end
+
+    import 'EIA Form 861 records from 2010 (revised)',
+           :url => 'http://www.eia.gov/Ftproot/pub/electricity/f86110.zip',
+           :filename => 'file1_2010.xls',
+           :skip => 7,
+           :errata => { :url => "file://#{Earth::ERRATA_DIR}/electricity/electric_utility/eia_errata.csv" } do
+      key   'eia_id',                    :field_name => 'UTILITY_ID'
+      store 'name',                      :field_name => 'UTILITY_NAME'
+      store 'state_postal_abbreviation', :field_name => 'MAIL_STATE',    :nullify => true
+      store 'nerc_region_abbreviation',        :synthesize => proc { |row| (first_nerc = row['NERC_LOCATION'].match(/^(\w+)/))? first_nerc.captures[0] : nil }
+      store 'second_nerc_region_abbreviation', :synthesize => proc { |row| (second_nerc = row['NERC_LOCATION'].match(/^\w+ ?[&,\/] ?(\w+)$/)) ? second_nerc.captures[0] : nil }
+    end
+
+    import 'Aliases', :url => 'https://docs.google.com/spreadsheet/pub?key=0AtyCBJLCFHlwdEM5WjVxRjBKWVJRcTJ3c1BhUnlSVXc&single=true&gid=0&output=csv' do
+      key 'eia_id', :field_name => 'utility_id'
+      store 'nickname', :field_name => 'alias'
+    end
+  end
+end

data/lib/earth/electricity/green_button_adoption.rb

@@ -0,0 +1,20 @@
+class GreenButtonAdoption < ActiveRecord::Base
+  self.primary_key = "electric_utility_name"
+  
+  col :electric_utility_name
+  col :implemented, :type => :boolean
+  col :committed, :type => :boolean
+
+  class << self
+    def implemented?(*names)
+      names.any? do |name|
+        find_by_electric_utility_name(name).try :implemented?
+      end
+    end
+    def committed?(*names)
+      names.any? do |name|
+        find_by_electric_utility_name(name).try :committed?
+      end
+    end
+  end
+end

data/lib/earth/electricity/green_button_adoption/data_miner.rb

@@ -0,0 +1,19 @@
+GreenButtonAdoption.class_eval do
+  data_miner do
+    import 'Green Button implementers',
+           :url => 'http://greenbuttondata.org/greenadopt.html',
+           :row_css => '#adopt+p+h2+table li.implemented',
+           :headers => %w{electric_utility_name} do
+      key 'electric_utility_name'
+      store 'implemented', :static => true
+    end
+
+    import 'Green Button committers',
+           :url => 'http://greenbuttondata.org/greenadopt.html',
+           :row_css => '#adopt+p+h2+table li.committed',
+           :headers => %w{electric_utility_name} do
+      key 'electric_utility_name'
+      store 'committed', :static => true
+    end
+  end
+end

data/lib/earth/locality/zip_code.rb

@@ -6,6 +6,8 @@ class ZipCode < ActiveRecord::Base
   belongs_to :egrid_subregion,  :foreign_key => 'egrid_subregion_abbreviation'
   belongs_to :climate_division, :foreign_key => 'climate_division_name'
   belongs_to :state,            :foreign_key => 'state_postal_abbreviation'
+  has_many :electric_markets,   :foreign_key => 'zip_code_name'
+  has_many :electric_utilities, :through => :electric_markets
   
   def country
     Country.united_states

data/lib/earth/rail/national_transit_database_company.rb

@@ -22,11 +22,12 @@ class NationalTransitDatabaseCompany < ActiveRecord::Base
   end
   
   # Methods to look up units from from rail_records
+  # TODO this looks inefficient
   [:rail_passenger_distance_units, :rail_vehicle_distance_units, :rail_vehicle_time_units, :rail_electricity_units, :rail_diesel_units].each do |method|
     define_method method do
       attribute = method.to_s.split('rail_')[1].to_sym
-      units = rail_records.map(&attribute).uniq
-      (units.count == 1 and units[0].present?) ? units[0] : raise("Error: units missing or multiple units in #{name}'s NTD records")
+      units = rail_records.map(&attribute).uniq.compact
+      (units.length == 1 and units[0].present?) ? units[0] : raise("Error: units missing or multiple units in #{name}'s NTD records")
     end
   end
   

data/lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb

@@ -4,7 +4,7 @@ ResidentialEnergyConsumptionSurveyResponse.class_eval do
     # conversions are NOT performed here, since we first have to zero out legitimate skips
     # otherwise you will get values like "999 pounds = 453.138778 kilograms" (where 999 is really a legit skip)
     import 'the 2005 EIA Residential Energy Consumption Survey microdata',
-           :url => 'http://www.eia.doe.gov/emeu/recs/recspubuse05/datafiles/RECS05alldata.csv',
+           :url => 'http://www.eia.gov/emeu/recs/recspubuse05/datafiles/RECS05alldata.csv',
            :headers => :upcase do
       key   'id', :field_name => 'DOEID'
       

data/lib/earth/version.rb

@@ -1,3 +1,3 @@
 module Earth
-  VERSION = "0.11.18"
+  VERSION = "0.11.19"
 end

data/spec/earth/air/aircraft_spec.rb

@@ -2,17 +2,18 @@ require 'spec_helper'
 require 'earth/air/aircraft'
 
 describe Aircraft do
-  before :all do
-    Aircraft.auto_upgrade!
-  end
-  
   describe 'import', :data_miner => true do
     before do
-      require 'earth/air/aircraft/data_miner'
+      Earth.init :air, :load_data_miner => true, :skip_parent_associations => :true
     end
     
     it 'should import data' do
       Aircraft.run_data_miner!
+    end
+  end
+  
+  describe "verify imported data", :sanity => true do
+    it "should have all the data" do
       Aircraft.all.count.should == 437
     end
   end

data/spec/earth/air/airport_spec.rb

@@ -2,22 +2,21 @@ require 'spec_helper'
 require 'earth/air/airport'
 
 describe Airport do
-  before :all do
-    Airport.auto_upgrade!
-  end
-  
   describe "when importing data", :data_miner => true do
     before do
-      require 'earth/air/airport/data_miner'
+      Earth.init :air, :load_data_miner => true, :skip_parent_associations => :true
     end
     
     it "imports all airports" do
       Airport.run_data_miner!
-      Airport.count.should == 5324
     end
   end
   
   describe "verify imported data", :sanity => true do
+    it "should have all the data" do
+      Airport.count.should == 5324
+    end
+    
     it "should have name" do
       Airport.where(:name => nil).count.should == 0
     end

data/spec/earth/air/bts_aircraft_spec.rb

@@ -2,18 +2,19 @@ require 'spec_helper'
 require 'earth/air/bts_aircraft'
 
 describe BtsAircraft do
-  before :all do
-    BtsAircraft.auto_upgrade!
-  end
-  
   describe 'import', :data_miner => true do
     before do
-      require 'earth/air/bts_aircraft/data_miner'
+      Earth.init :air, :load_data_miner => true, :skip_parent_associations => :true
     end
     
     it 'should import data' do
       BtsAircraft.run_data_miner!
-      BtsAircraft.all.count.should == 378
+    end
+  end
+  
+  describe 'verify imported data', :sanity => true do
+    it 'should have all the data' do
+      BtsAircraft.all.count.should == 379
     end
   end
 end

data/spec/earth/air/flight_distance_class_spec.rb

@@ -2,22 +2,21 @@ require 'spec_helper'
 require 'earth/air/flight_distance_class'
 
 describe FlightDistanceClass do
-  before :all do
-    FlightDistanceClass.auto_upgrade!
-  end
-  
   describe "when importing data", :data_miner => true do
     before do
-      require 'earth/air/flight_distance_class/data_miner'
+      Earth.init :air, :load_data_miner => true, :skip_parent_associations => :true
     end
     
     it "imports all naics codes" do
       FlightDistanceClass.run_data_miner!
-      FlightDistanceClass.count.should == 2
     end
   end
   
   describe "verify imported data", :sanity => true do
+    it "should have all the data" do
+      FlightDistanceClass.count.should == 2
+    end
+    
     it "should have distances > 0" do
       FlightDistanceClass.where('distance <= 0 OR distance IS NULL').count.should == 0
     end

data/spec/earth/air/flight_segment_spec.rb

@@ -2,22 +2,21 @@ require 'spec_helper'
 require 'earth/air/flight_segment'
 
 describe FlightSegment do
-  before :all do
-    FlightSegment.auto_upgrade!
-  end
-  
   describe 'import', :data_miner => true do
     before do
-      require 'earth/air/flight_segment/data_miner'
+      Earth.init :air, :load_data_miner => true, :skip_parent_associations => :true
     end
     
     it 'should import data' do
       FlightSegment.run_data_miner!
-      FlightSegment.all.count.should == 1_149_003
     end
   end
   
   describe "verify imported data", :sanity => true do
+    it "should have all the data" do
+      FlightSegment.all.count.should == 1_149_003
+    end
+    
     it "should have year from 2009 to present" do
       FlightSegment.where("year IS NULL OR year < 2009 OR year > #{::Time.now.year}").count.should == 0
     end

data/spec/earth/automobile/automobile_fuel_spec.rb

@@ -1,16 +1,23 @@
 require 'spec_helper'
 require 'earth/automobile'
-require 'earth/automobile/data_miner'
 
 describe AutomobileFuel do
   describe 'import', :data_miner => true do
+    before do
+      Earth.init :automobile, :load_data_miner => true, :skip_parent_associations => :true
+    end
+    
     it 'should import data without problems' do
       AutomobileFuel.run_data_miner!
-      AutomobileFuel.all.count.should > 0
     end
-    
+  end
+  
+  describe 'verify imported data', :sanity => true do
+    it 'should have all the data' do
+      AutomobileFuel.all.count.should == 9
+    end
     it 'correctly assigns hfc_emission_factor' do
-      AutomobileFuel.first.hfc_emission_factor.should == 0.0
+      AutomobileFuel.first.hfc_emission_factor.should == 0.124799
     end
   end
 end