earth 0.0.40 → 0.0.41

data/lib/earth.rb

@@ -24,62 +24,18 @@ module Earth
   def taps_server=(val)
     @taps_server = val
   end
-
-  def classes
-    [
-      AirConditionerUse,
-      Aircraft,
-      AircraftClass,
-      AircraftManufacturer,
-      Airline,
-      Airport,
-      AutomobileFuelType,
-      AutomobileMake,
-      AutomobileMakeFleetYear,
-      AutomobileMakeYear,
-      AutomobileModel,
-      AutomobileModelYear,
-      AutomobileSizeClass,
-      AutomobileVariant,
-      Breed,
-      BreedGender,
-      BusClass,
-      CensusDivision,
-      CensusRegion,
-      ClimateDivision,
-      ClothesMachineUse,
-      Country,
-      DietClass,
-      DishwasherUse,
-      EgridRegion,
-      EgridSubregion,
-      FlightConfiguration,
-      FlightDistanceClass,
-      FlightDomesticity,
-      FlightFuelType,
-      FlightPropulsion,
-      FlightSeatClass,
-      FlightSegment,
-      FlightService,
-      FoodGroup,
-      Gender,
-      LodgingClass,
-      PetroleumAdministrationForDefenseDistrict,
-      RailClass,
-      ResidenceAppliance,
-      ResidenceClass,
-      ResidenceFuelPrice,
-      ResidenceFuelType,
-      ResidentialEnergyConsumptionSurveyResponse,
-      Species,
-      State,
-      Urbanity,
-      ZipCode
-    ]
+  
+  # Takes argument like Earth.resource_names(['air'])
+  # Default is search all domains
+  # For example, <tt>[ 'Aircraft', 'Airline' ]</tt>
+  def resource_names(search_domains = nil)
+    (search_domains || domains).map do |domain|
+      Dir[File.join(Earth.gem_root, 'lib', 'earth', domain, '*.rb')]
+    end.flatten.uniq.map { |p| File.basename(p, '.rb').camelcase } - %w{ DataMiner }
   end
 
-  def root 
-    File.join(File.dirname(__FILE__), '..')
+  def gem_root 
+    File.expand_path File.join(File.dirname(__FILE__), '..')
   end
 
   def domains
@@ -121,6 +77,7 @@ module Earth
 private
   def load_domains(domains, options)
     if domains.empty? or domains.include?(:all)
+      # sabshere 9/16/10 why maintain this separately?
       require 'earth/all'
       require 'earth/data_miner' if options[:apply_schemas] or options[:load_data_miner]
     elsif !domains.include?(:none)
@@ -133,24 +90,33 @@ private
 
   def load_plugins
     require 'earth/active_record_ext'
-    Dir[File.join(Earth.root, 'vendor', '**', 'init.rb')].each do |pluginit|
+    Dir[File.join(Earth.gem_root, 'vendor', '**', 'init.rb')].each do |pluginit|
       $:.unshift File.join(File.dirname(pluginit), 'lib')
       load pluginit
     end
   end
 
   def load_schemas
-    load_ar_schema
+    force_fallback_table
     load_data_miner_schemas
   end
-
-  def load_ar_schema
-    orig_std_out = STDOUT.clone
-    STDOUT.reopen File.open(File.join('/tmp', 'schema_output'), 'w') 
-
-    load File.join(File.dirname(__FILE__), 'earth', 'schema.rb')
-  ensure
-    STDOUT.reopen(orig_std_out)
+  
+  # sabshere 9/17/10 this sucks. the falls_back_on gem sucks.
+  def force_fallback_table
+    c = ActiveRecord::Base.connection
+    if c.table_exists? 'fallbacks'
+      raise "The earth gem expects a different schema for the fallbacks table." unless  c.column_exists?('fallbacks', 'name') and
+                                                                                        c.column_exists?('fallbacks', 'values') and
+                                                                                        c.column_exists?('fallbacks', 'created_at') and
+                                                                                        c.column_exists?('fallbacks', 'updated_at')
+    else
+      c.create_table 'fallbacks' do |t|
+        t.string   'name'
+        t.text     'values'
+        t.datetime 'created_at'
+        t.datetime 'updated_at'
+      end
+    end
   end
 
   def load_data_miner_schemas

data/lib/earth/air/aircraft/data_miner.rb

@@ -1,4 +1,24 @@
 Aircraft.class_eval do
+  
+  # TODO use http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_AIRCRAFT_TYPE
+  def self.bts_name_dictionary
+    @_bts_dictionary ||= LooseTightDictionary.new RemoteTable.new(:url => 'http://www.transtats.bts.gov/Download_Lookup.asp?Lookup=L_AIRCRAFT_TYPE', :select => lambda { |record| record['Code'].to_i.between?(1, 998) }),
+                                                  :tightenings  => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false),
+                                                  :identities   => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false),
+                                                  :blockings    => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false),
+                                                  :blocking_only => true,
+                                                  :right_reader => lambda { |record| record['Description'] }
+  end
+  
+  # warning: self-referential, assumes it will be used once first import step is done
+  def self.icao_name_dictionary
+    @_icao_dictionary ||= LooseTightDictionary.new Aircraft.all,
+                                                   :tightenings  => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false),
+                                                   :identities   => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false),
+                                                   :blockings    => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false),
+                                                   :right_reader => lambda { |record| record.manufacturer_name.to_s + ' ' + record.name.to_s }
+  end
+
   class Aircraft::BtsMatcher
     attr_reader :wants
     def initialize(wants)
@@ -71,10 +91,6 @@ Aircraft.class_eval do
   end
 
   data_miner do
-    process "Don't re-import too often" do
-      raise DataMiner::Skip unless DataMiner::Run.allowed? Aircraft
-    end
-    
     schema Earth.database_options do
       string   'icao_code'
       string   'manufacturer_name'
@@ -161,11 +177,8 @@ Aircraft.class_eval do
       update_all "weighting = (#{segments.project(segments[:passengers].sum).where(aircraft[:bts_aircraft_type_code].eq(segments[:bts_aircraft_type_code])).to_sql})"
     end
     
-    [ AircraftManufacturer ].each do |synthetic_resource|
-      process "Synthesize #{synthetic_resource}" do
-        synthetic_resource.run_data_miner!
-      end
+    process "Synthesize AircraftManufacturer" do
+      AircraftManufacturer.run_data_miner!
     end
-    
   end
 end

data/lib/earth/air/flight_segment/data_miner.rb

@@ -159,10 +159,6 @@ FlightSegment.class_eval do
   }.gsub /[\s]+/,''
 
   data_miner do
-    process "Don't re-import too often" do
-      raise DataMiner::Skip unless DataMiner::Run.allowed? FlightSegment
-    end
-    
     schema Earth.database_options do
       string   'row_hash'
       string   'propulsion_id'

data/lib/earth/automobile/automobile_variant/data_miner.rb

@@ -273,10 +273,6 @@ AutomobileVariant.class_eval do
   end
   
   data_miner do
-    process "Don't re-import too often" do
-      raise DataMiner::Skip unless DataMiner::Run.allowed? AutomobileVariant
-    end
-    
     schema Earth.database_options do
       string   'row_hash'
       string   'name' # short name!
@@ -454,9 +450,9 @@ AutomobileVariant.class_eval do
       update_all 'fuel_efficiency_highway = 1 / ((0.001376 / 0.425143707) + (1.3466 / raw_fuel_efficiency_highway))'
     end
     
-    [ AutomobileMake, AutomobileModelYear, AutomobileModel ].each do |synthetic_resource|
+    %w{ AutomobileMake AutomobileModelYear AutomobileModel }.each do |synthetic_resource|
       process "Synthesize #{synthetic_resource}" do
-        synthetic_resource.run_data_miner!
+        synthetic_resource.constantize.run_data_miner!
       end
     end
   end

data/lib/earth/industry/sector/data_miner.rb

@@ -6,8 +6,7 @@ Sector.class_eval do
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdHV2dzdjdGVnLUJCdDA4TURXNThROVE&hl=en&single=true&gid=1&output=csv' do
       key 'io_code'
       store 'description'
-      store 'emission_factor'
-      store 'emission_factor_units'
+      store 'emission_factor', :units_field_name => 'emission_factor_units'
     end
   end
 end

data/lib/earth/locality/census_division.rb

@@ -7,6 +7,11 @@ class CensusDivision < ActiveRecord::Base
   has_many :climate_divisions, :through => :states
   has_many :residential_energy_consumption_survey_responses, :foreign_key => 'census_division_number'
   
+  falls_back_on :meeting_building_natural_gas_intensity => 0.011973,
+                :meeting_building_fuel_oil_intensity => 0.0037381,
+                :meeting_building_electricity_intensity => 0.072444,
+                :meeting_building_district_heat_intensity => 3458.7
+  
   data_miner do
     tap "Brighter Planet's sanitized census divisions", Earth.taps_server
 

data/lib/earth/locality/census_division/data_miner.rb

@@ -22,11 +22,16 @@ CensusDivision.class_eval do
       float   'lodging_building_district_heat_intensity'
       string  'lodging_building_district_heat_intensity_units'
     end
-    
-    falls_back_on :meeting_building_natural_gas_intensity => 0.011973,
-                  :meeting_building_fuel_oil_intensity => 0.0037381,
-                  :meeting_building_electricity_intensity => 0.072444,
-                  :meeting_building_district_heat_intensity => 3458.7
+
+    process 'Define some unit conversions' do
+      Conversions.register :hundred_cubic_feet_per_square_foot_hour, :cubic_metres_per_square_metre_hour,   30.48
+      Conversions.register :gallons_per_square_foot_hour,            :litres_per_square_metre_hour,         40.745833
+      Conversions.register :kilowatt_hours_per_square_foot_hour,     :kilowatt_hours_per_square_metre_hour, 10.76391
+      Conversions.register :thousand_btu_per_square_foot_hour,       :joules_per_square_metre_hour,         11_356_527
+      Conversions.register :hundred_cubic_feet_per_room_night,       :cubic_metres_per_room_night,          2.8317
+      Conversions.register :gallons_per_room_night,                  :litres_per_room_night,                3.7854
+      Conversions.register :thousand_btu_per_room_night,             :joules_per_room_night,                1_055_056
+    end
     
     import 'the U.S. Census Geographic Terms and Definitions',
            :url => 'http://www.census.gov/popest/geographic/codes02.csv',
@@ -38,16 +43,6 @@ CensusDivision.class_eval do
       store 'census_region_name', :field_name => 'Region', :dictionary => { :input => 'number', :output => 'name', :url => 'http://data.brighterplanet.com/census_regions.csv' }
     end
     
-    process 'Define some unit conversions' do
-      Conversions.register :hundred_cubic_feet_per_square_foot_hour, :cubic_metres_per_square_metre_hour,   30.48
-      Conversions.register :gallons_per_square_foot_hour,            :litres_per_square_metre_hour,         40.745833
-      Conversions.register :kilowatt_hours_per_square_foot_hour,     :kilowatt_hours_per_square_metre_hour, 10.76391
-      Conversions.register :thousand_btu_per_square_foot_hour,       :joules_per_square_metre_hour, 11_356_527
-      Conversions.register :hundred_cubic_feet_per_room_night,       :cubic_metres_per_room_night,           2.8317
-      Conversions.register :gallons_per_room_night,                  :litres_per_room_night,                 3.7854
-      Conversions.register :thousand_btu_per_room_night,             :joules_per_room_night,         1_055_056
-    end
-    
     import 'meeting building fuel intensities calculated from CBECS 2003',
            :url => 'https://spreadsheets.google.com/pub?key=0AoQJbWqPrREqdGotdjgtUEk0eUU0R3lxM1hHdlF1V0E&hl=en&output=csv' do
       key   'number', :field_name => 'census_division'

data/lib/earth/residence/residence_fuel_type.rb

@@ -1,7 +1,6 @@
 class ResidenceFuelType < ActiveRecord::Base
   set_primary_key :name
   
-  
   has_many :prices, :class_name => 'ResidenceFuelPrice', :foreign_key => 'residence_fuel_type_name'
   
   data_miner do

data/lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb

@@ -1,9 +1,5 @@
 ResidentialEnergyConsumptionSurveyResponse.class_eval do
   data_miner do
-    process "Don't re-import too often" do
-      raise DataMiner::Skip unless DataMiner::Run.allowed? ResidentialEnergyConsumptionSurveyResponse
-    end
-    
     schema Earth.database_options do
       integer  'department_of_energy_identifier'
       string   'residence_class_id'

data/spec/lib/earth_spec.rb

@@ -7,13 +7,13 @@ describe Earth do
 
   it 'should require all Earth models' do
     lambda do
-      Earth.classes.each { |k| k }
+      Earth.resource_names.each { |k| k.constantize }
     end.should_not raise_error(NameError)
   end
 
   it 'should include data_miner definitions' do
     lambda do
-      Earth.classes.each { |k| k.should_receive(:data_miner) }
+      Earth.resource_names.each { |k| k.constantize.should_receive(:data_miner) }
     end
     require 'earth/data_miner'
   end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: earth
 version: !ruby/object:Gem::Version 
-  hash: 79
+  hash: 77
   prerelease: false
   segments: 
   - 0
   - 0
-  - 40
-  version: 0.0.40
+  - 41
+  version: 0.0.41
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -17,7 +17,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-09-15 00:00:00 -07:00
+date: 2010-09-17 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -28,7 +28,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: -1848230024
+        hash: 299253624
         segments: 
         - 3
         - 0
@@ -366,7 +366,6 @@ files:
 - lib/earth/residence/residence_fuel_type/data_miner.rb
 - lib/earth/residence/residential_energy_consumption_survey_response.rb
 - lib/earth/residence/residential_energy_consumption_survey_response/data_miner.rb
-- lib/earth/schema.rb
 - vendor/geokit-rails/CHANGELOG.rdoc
 - vendor/geokit-rails/MIT-LICENSE
 - vendor/geokit-rails/README.markdown

data/lib/earth/schema.rb

@@ -1,8 +0,0 @@
-ActiveRecord::Schema.define(:version => 1) do
-  create_table "fallbacks", :force => true do |t|
-    t.string   "name"
-    t.text     "values"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-  end
-end