earth 0.5.0 → 0.5.2

data/lib/earth/residence/residential_energy_consumption_survey_response.rb

@@ -30,11 +30,97 @@ class ResidentialEnergyConsumptionSurveyResponse < ActiveRecord::Base
     :ownership,
   ]
   
-  data_miner do
-    tap "Brighter Planet's sanitized RECS 2005", Earth.taps_server
-    
-    process "pull dependencies" do
-      run_data_miner_on_belongs_to_associations
-    end
+  # sabshere 9/20/10 sorted with sort -d -t "'" -k 2 ~/Desktop/parts.txt
+  create_table do
+    integer  'id'
+    string  'air_conditioner_use_id'
+    float    'annual_energy_from_electricity_for_air_conditioners'
+    string   'annual_energy_from_electricity_for_air_conditioners_units'
+    float    'annual_energy_from_electricity_for_clothes_driers'
+    string   'annual_energy_from_electricity_for_clothes_driers_units'
+    float    'annual_energy_from_electricity_for_dishwashers'
+    string   'annual_energy_from_electricity_for_dishwashers_units'
+    float    'annual_energy_from_electricity_for_freezers'
+    string   'annual_energy_from_electricity_for_freezers_units'
+    float    'annual_energy_from_electricity_for_heating_space'
+    string   'annual_energy_from_electricity_for_heating_space_units'
+    float    'annual_energy_from_electricity_for_heating_water'
+    string   'annual_energy_from_electricity_for_heating_water_units'
+    float    'annual_energy_from_electricity_for_other_appliances'
+    string   'annual_energy_from_electricity_for_other_appliances_units'
+    float    'annual_energy_from_electricity_for_refrigerators'
+    string   'annual_energy_from_electricity_for_refrigerators_units'
+    float    'annual_energy_from_fuel_oil_for_appliances'
+    string   'annual_energy_from_fuel_oil_for_appliances_units'
+    float    'annual_energy_from_fuel_oil_for_heating_space'
+    string   'annual_energy_from_fuel_oil_for_heating_space_units'
+    float    'annual_energy_from_fuel_oil_for_heating_water'
+    string   'annual_energy_from_fuel_oil_for_heating_water_units'
+    float    'annual_energy_from_kerosene'
+    string   'annual_energy_from_kerosene_units'
+    float    'annual_energy_from_natural_gas_for_appliances'
+    string   'annual_energy_from_natural_gas_for_appliances_units'
+    float    'annual_energy_from_natural_gas_for_heating_space'
+    string   'annual_energy_from_natural_gas_for_heating_space_units'
+    float    'annual_energy_from_natural_gas_for_heating_water'
+    string   'annual_energy_from_natural_gas_for_heating_water_units'
+    float    'annual_energy_from_propane_for_appliances'
+    string   'annual_energy_from_propane_for_appliances_units'
+    float    'annual_energy_from_propane_for_heating_space'
+    string   'annual_energy_from_propane_for_heating_space_units'
+    float    'annual_energy_from_propane_for_heating_water'
+    string   'annual_energy_from_propane_for_heating_water_units'
+    float    'annual_energy_from_wood'
+    string   'annual_energy_from_wood_units'
+    boolean  'attached_1car_garage'
+    boolean  'attached_2car_garage'
+    boolean  'attached_3car_garage'
+    float    'bathrooms'
+    integer  'bedrooms'
+    string   'census_division_name'
+    integer  'census_division_number'
+    string   'census_region_name'
+    integer  'census_region_number'
+    string   'central_ac_use'
+    string   'clothes_dryer_use'
+    string  'clothes_machine_use_id'
+    string   'clothes_washer_use'
+    string   'construction_period'
+    date     'construction_year'
+    integer  'cooling_degree_days'
+    string   'cooling_degree_days_units'
+    boolean  'detached_1car_garage'
+    boolean  'detached_2car_garage'
+    boolean  'detached_3car_garage'
+    string   'dishwasher_use_id'
+    integer  'efficient_lights_on_1_to_4_hours'
+    integer  'efficient_lights_on_4_to_12_hours'
+    integer  'efficient_lights_on_over_12_hours'
+    float    'floorspace'
+    string   'floorspace_units'
+    integer  'freezer_count'    
+    integer  'full_bathrooms'
+    integer  'half_bathrooms'
+    boolean  'heated_garage'
+    integer  'heating_degree_days'
+    string   'heating_degree_days_units'
+    float    'lighting_efficiency'
+    float    'lighting_use'
+    string   'lighting_use_units'
+    integer  'lights_on_1_to_4_hours'
+    integer  'lights_on_4_to_12_hours'
+    integer  'lights_on_over_12_hours'
+    integer  'outdoor_all_night_gas_lights'
+    integer  'outdoor_all_night_lights'
+    boolean  'ownership'
+    integer  'refrigerator_count'
+    string   'residence_class_id'
+    integer  'residents'
+    float    'rooms'
+    boolean  'thermostat_programmability'
+    integer  'total_rooms'
+    string   'urbanity_id'
+    float    'weighting'
+    string   'window_ac_use'
   end
 end

data/lib/earth/residence.rb

@@ -6,3 +6,5 @@ require 'earth/residence/residence_class'
 require 'earth/residence/residence_fuel_price'
 require 'earth/residence/residence_fuel_type'
 require 'earth/residence/residential_energy_consumption_survey_response'
+
+require 'earth/locality'

data/lib/earth/shipping/carrier/data_miner.rb

@@ -1,15 +1,5 @@
 Carrier.class_eval do
   data_miner do
-    schema Earth.database_options do
-      string 'name'
-      float  'package_volume'
-      float  'route_inefficiency_factor'
-      float  'transport_emission_factor'
-      string 'transport_emission_factor_units'
-      float  'corporate_emission_factor'
-      string 'corporate_emission_factor_units'
-    end
-    
     import "a list of shipping companies and their characteristics",
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdG1ONU1HZDdZTFJNclFYVkRzR0k5Z2c&hl=en&gid=0&output=csv' do
       key   'name'

data/lib/earth/shipping/carrier.rb

@@ -8,7 +8,13 @@ class Carrier < ActiveRecord::Base
                 :transport_emission_factor => 0.0005266,
                 :corporate_emission_factor => 0.221
   
-  data_miner do
-    tap "Brighter Planet's shipping company data", Earth.taps_server
+  create_table do
+    string 'name'
+    float  'package_volume'
+    float  'route_inefficiency_factor'
+    float  'transport_emission_factor'
+    string 'transport_emission_factor_units'
+    float  'corporate_emission_factor'
+    string 'corporate_emission_factor_units'
   end
 end

data/lib/earth/shipping/carrier_mode/data_miner.rb

@@ -1,15 +1,5 @@
 CarrierMode.class_eval do
   data_miner do
-    schema Earth.database_options do
-      string 'name'
-      string 'carrier_name'
-      string 'mode_name'
-      float  'package_volume'
-      float  'route_inefficiency_factor'
-      float  'transport_emission_factor'
-      string 'transport_emission_factor_units'
-    end
-    
     import "a list of carrier modes and their characteristics",
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGRsRkJOd0NPd0FETTI0NmpYUlBsN2c&hl=en&gid=0&output=csv' do
       key   'name'

data/lib/earth/shipping/carrier_mode.rb

@@ -3,8 +3,14 @@ class CarrierMode < ActiveRecord::Base
   
   belongs_to :carrier, :foreign_key => 'carrier_name', :primary_key => 'name'
   belongs_to :mode,    :foreign_key => 'mode_name',    :primary_key => 'name', :class_name => 'ShipmentMode'
-  
-  data_miner do
-    tap "Brighter Planet's carrier mode data", Earth.taps_server
+
+  create_table do
+    string 'name'
+    string 'carrier_name'
+    string 'mode_name'
+    float  'package_volume'
+    float  'route_inefficiency_factor'
+    float  'transport_emission_factor'
+    string 'transport_emission_factor_units'
   end
 end

data/lib/earth/shipping/shipment_mode/data_miner.rb

@@ -1,12 +1,5 @@
 ShipmentMode.class_eval do
   data_miner do
-    schema Earth.database_options do
-      string 'name'
-      float  'route_inefficiency_factor'
-      float  'transport_emission_factor'
-      string 'transport_emission_factor_units'
-    end
-    
     import "a list of shipment modes and their characteristics",
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGR2a2RYcEg2RnRKbkRQcXc0ZkQ5U0E&hl=en&gid=0&output=csv' do
       key   'name'

data/lib/earth/shipping/shipment_mode.rb

@@ -3,7 +3,10 @@ class ShipmentMode < ActiveRecord::Base
   
   has_many :carrier_modes, :foreign_key => 'mode_name', :primary_key => 'name'
   
-  data_miner do
-    tap "Brighter Planet's shipment mode data", Earth.taps_server
+  create_table do
+    string 'name'
+    float  'route_inefficiency_factor'
+    float  'transport_emission_factor'
+    string 'transport_emission_factor_units'
   end
 end

data/lib/earth/version.rb

@@ -1,3 +1,3 @@
 module Earth
-  VERSION = "0.5.0"
+  VERSION = "0.5.2"
 end

data/lib/earth.rb

@@ -1,6 +1,5 @@
 require 'active_record'
 require 'cohort_scope'
-require 'earth/base'
 require 'earth/conversions_ext'
 require 'earth/inflectors'
 require 'data_miner'
@@ -8,31 +7,36 @@ require 'falls_back_on'
 require 'weighted_average'
 require 'fixed_width'
 require 'errata'
+require 'create_table'
+require 'loose_tight_dictionary'
+require 'loose_tight_dictionary/cached_result'
 
-# The earth module is an interface for establishing a taps server (used to fetch 
-# data) and for loading data models from various domains.
+# hackety hack
+def INSERT_IGNORE(cmd)
+  if ActiveRecord::Base.connection.adapter_name.downcase == 'sqlite'
+    prefix = 'INSERT'
+  else
+    prefix = 'INSERT IGNORE'
+  end
+  ActiveRecord::Base.connection.execute "#{prefix} #{cmd}"
+end
+
+# The earth module is an interface for loading data models from various domains.
 module Earth
   extend self
 
-  def taps_server
-    @taps_server || 'http://carbon:neutral@data.brighterplanet.com:5000'
-  end
-
-  # taps_server is a URL. See the data_miner gem docs
-  def taps_server=(val)
-    @taps_server = val
-  end
-  
-  # Takes argument like Earth.resource_names(['air'])
+  # Takes argument like Earth.search(['air'])
   # Default is search all domains
   # For example, <tt>[ 'Aircraft', 'Airline' ]</tt>
-  def resource_names(search_domains = nil)
-    if search_domains.nil?
-      resources.keys
+  def search(*search_domains)
+    search_domains = search_domains.empty? ? [:all] : search_domains.flatten.map(&:to_sym)
+    if search_domains.include? :all
+      resources
     else
-      resources.inject([]) do |list, (name, data)|
-        list << name if search_domains.include? data[:domain]
-        list
+      resource_map.select do |resource, domain|
+        search_domains.include? domain.to_sym
+      end.map do |resource, domain|
+        resource
       end
     end
   end
@@ -42,21 +46,24 @@ module Earth
   end
 
   def domains
-    @domains ||= resources.map { |(name, data)| data[:domain] }.uniq.sort
+    resource_map.values.uniq.sort
   end
-
+  
   def resources
-    return @resources unless @resources.nil?
-    earth_files = Dir[File.join(Earth.gem_root, 'lib', 'earth', '*')]
-    domain_dirs = earth_files.find_all { |f| File.directory?(f) }
-    @resources = domain_dirs.inject({}) do |hsh, domain_dir|
-      Dir[File.join(domain_dir, '*.rb')].each do |resource_file|
-        resource_camel = File.basename(resource_file, '.rb').camelcase
-        unless resource_camel == 'DataMiner'
-          hsh[resource_camel] = { :domain => File.basename(domain_dir) }
+    resource_map.keys.sort
+  end
+
+  def resource_map
+    @resource_map ||= Dir[File.join(gem_root, 'lib', 'earth', '*')].select do |path|
+      File.directory? path
+    end.inject({}) do |memo, domain_path|
+      Dir[File.join(domain_path, '*.rb')].each do |resource_path|
+        resource = File.basename(resource_path, '.rb').camelcase
+        unless resource == 'DataMiner'
+          memo[resource] = File.basename domain_path
         end
       end
-      hsh
+      memo
     end
   end
 
@@ -68,85 +75,86 @@ module Earth
   # Earth.init should be performed after a connection is made to the database and 
   # before any domain models are referenced.
   def init(*args)
-    load_plugins
-
-    domains = []
-    options = {}
-    args.each do |arg|
-      if arg.is_a?(Hash)
-        options = arg
-      else
-        domains << arg
-      end
-    end
-
-    load_domains(domains, options)
-    load_schemas(options) if options[:apply_schemas]
-  end
+    options = args.last.is_a?(Hash) ? args.pop.symbolize_keys : {}
+    domains = args.empty? ? [ :all ] : args.map(&:to_sym)
 
-  def database_options
-    if ActiveRecord::Base.connection.adapter_name.downcase == 'sqlite'
-      {}
-    else
-      { :options => 'ENGINE=InnoDB default charset=utf8 collate=utf8_general_ci' }
-    end
+    _load_plugins
+    _load_domains domains, options
+    _decorate_resources options
+    _load_schemas search(domains), options
   end
 
-private
-  def load_domains(domains, options)
+  private
+  
+  # TODO sabshere don't use directories to specify domains
+  # * you have 20 million data_miner.rb files which are easy to confuse
+  # * you have to go all over the filesystem to figure things out
+  def _load_domains(domains, options)
+    return if domains.include? :none
     if domains.empty? or domains.include?(:all)
       # sabshere 9/16/10 why maintain this separately?
       require 'earth/all'
       require 'earth/data_miner' if options[:apply_schemas] or options[:load_data_miner]
-    elsif !domains.include?(:none)
+    else
       domains.each do |domain| 
         require "earth/#{domain}"
-        if options[:apply_schemas] or options[:load_data_miner]
-          begin
-            require "earth/#{domain}/data_miner"
-          rescue LoadError
-          end
-        end
+        require "earth/#{domain}/data_miner" if options[:apply_schemas] or options[:load_data_miner]
       end
     end
   end
 
-  def load_plugins
-    require 'earth/active_record_ext'
+  def _load_plugins
     Dir[File.join(Earth.gem_root, 'vendor', '**', 'init.rb')].each do |pluginit|
       $:.unshift File.join(File.dirname(pluginit), 'lib')
       load pluginit
     end
   end
+  
+  def _append_pull_dependencies_step_to_data_miner(resource)
+    resource_model = resource.constantize
+    return if resource_model.data_miner_config.steps.any? { |step| step.description == :run_data_miner_on_parent_associations! }
 
-  def load_schemas(options = {})
-    load_data_miner_schemas(options)
+    pull_dependencies_step = DataMiner::Process.new resource_model.data_miner_config, :run_data_miner_on_parent_associations!
+
+    resource_model.data_miner_config.steps.push pull_dependencies_step
   end
+  
+  def _prepend_create_table_step_to_data_miner(resource)
+    resource_model = resource.constantize
+    return if resource_model.data_miner_config.steps.any? { |step| step.description == :create_table! }
+
+    create_table_step = DataMiner::Process.new resource_model.data_miner_config, :create_table!
 
-  def load_data_miner_schemas(options = {})
-    models = Module.constants.select do |k|
-      const = Object.const_get(k) if Object.const_defined?(k)
-      if const.instance_of?(Class)
-        const.superclass == ActiveRecord::Base || 
-          const.superclass == Earth::Base
+    resource_model.data_miner_config.steps.unshift create_table_step
+  end
+  
+  TAPS_STEP = 'Tap the Brighter Planet data server'
+  TAPS_SOURCE = 'http://carbon:neutral@data.brighterplanet.com'
+  def _prepend_taps_step_to_data_miner(resource)
+    resource_model = resource.constantize
+    return if resource_model.data_miner_config.steps.any? { |step| step.description == TAPS_STEP }
+    
+    taps_step = DataMiner::Tap.new resource_model.data_miner_config, TAPS_STEP, TAPS_SOURCE
+    
+    resource_model.data_miner_config.steps.unshift taps_step
+  end
+  
+  def _decorate_resources(options)
+    resources.each do |resource|
+      next unless ::Object.const_defined?(resource)
+      _append_pull_dependencies_step_to_data_miner resource
+      if options[:apply_schemas] or options[:load_data_miner]
+        _prepend_create_table_step_to_data_miner resource
       else
-        false
-      end
-    end
-    models.sort.each do |model|
-      klass = Object.const_get(model)
-      if klass.respond_to?(:execute_schema) and (!klass.table_exists? || options[:force_schema])
-        klass.execute_schema 
+        _prepend_taps_step_to_data_miner resource
       end
     end
   end
-end
 
-def INSERT_IGNORE(cmd)
-  if ActiveRecord::Base.connection.adapter_name.downcase == 'sqlite'
-    prefix = 'INSERT'
-  else
-    prefix = 'INSERT IGNORE'
+  def _load_schemas(selected_resources, options)
+    return unless options[:apply_schemas]
+    selected_resources.each do |resource|
+      resource.constantize.create_table!
+    end
   end
-  ActiveRecord::Base.connection.execute "#{prefix} #{cmd}"
 end

data/spec/earth/air/aircraft_spec.rb

@@ -5,7 +5,7 @@ require 'earth/air/aircraft/data_miner'
 describe Aircraft do
   describe 'import', :slow => true do
     it 'should treat empty cells as null' do
-      Aircraft.execute_schema
+      Aircraft.create_table!
       Aircraft.run_data_miner!
       Aircraft.count.should > 1
       Aircraft.where(:brighter_planet_aircraft_class_code => nil).should_not be_empty

data/spec/earth_spec.rb

@@ -8,38 +8,38 @@ describe Earth do
 
     it 'should require all Earth models' do
       lambda do
-        Earth.resource_names.each { |k| k.constantize }
+        Earth.resources.each { |k| k.constantize }
       end.should_not raise_error(NameError)
     end
 
     it 'should include data_miner definitions' do
       lambda do
-        Earth.resource_names.each { |k| k.constantize.should_receive(:data_miner) }
+        Earth.resources.each { |k| k.constantize.should_receive(:data_miner) }
       end
       require 'earth/data_miner'
     end
   end
 
-  describe '.resources' do
-    it 'should get a list of resources' do
-      resources = Earth.resources
-      resources.keys.count.should == 76
-      resources['FuelType'][:domain].should == 'fuel'
+  describe '.resource_map' do
+    it 'should get a list of resource_map' do
+      resource_map = Earth.resource_map
+      resource_map.keys.count.should == 75
+      resource_map['FuelType'].should == 'fuel'
     end
     it 'should exclude data_miner files' do
-      Earth.resources.keys.should_not include('DataMiner')
+      Earth.resource_map.keys.should_not include('DataMiner')
     end
   end
 
-  describe '.resource_names' do
+  describe '.search' do
     it 'should get a list of all resource names' do
-      Earth.resource_names.count.should == 76
-      Earth.resource_names.should include('Aircraft')
-      Earth.resource_names.should include('Industry')
+      Earth.search.length.should == 75
+      Earth.search.should include('Aircraft')
+      Earth.search.should include('Industry')
     end
-    it 'should filter resources by domain' do
-      Earth.resource_names(['fuel']).count.should == 5
-      Earth.resource_names(['fuel']).should include('FuelType')
+    it 'should filter resource_map by domain' do
+      Earth.search('fuel').length.should == 5
+      Earth.search('fuel').should include('FuelType')
     end
   end
 

data/spec/spec_helper.rb

@@ -1,18 +1,21 @@
 require 'rubygems'
 require 'bundler'
 Bundler.setup
-
+require 'logger'
 require 'active_record'
-require 'sqlite3'
-ActiveRecord::Base.establish_connection :adapter => 'sqlite3',
-  :database => ':memory:'
+require 'data_miner'
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'earth'
 
-$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
-require 'earth' # we do require Earth to live
+logger = Logger.new 'log/test.log'
+logger.level = Logger::DEBUG
 
-require 'data_miner'
-require 'logger'
-DataMiner.logger = Logger.new(nil)
+ActiveRecord::Base.establish_connection :adapter => 'sqlite3', :database => ':memory:'
+
+ActiveRecord::Base.logger = logger
+
+DataMiner.logger = logger
 
 RSpec.configure do |c|
   c.filter_run_excluding :slow => true