earth 0.11.7 → 0.11.8

data/README.markdown

@@ -11,6 +11,36 @@ The data that these models represent can be pulled from http://data.brighterplan
     ft = AutomobileFuel.first
     ...
 
+`Earth.init` loads desired "data domains" as well as any supporting classes and plugins that each data model needs. A "data domain" is a grouping of related data models. For instance, all automobile-related data is in the `:automobile` domain.
+
+### Data storage
+
+You can store Earth data in any relational database. On your very first run, you will need to create the tables for data each model. This is done using minirecord with the `apply_schemas` option:
+
+    require 'activerecord'
+    ActiveRecord::Base.establish_connection :adapter => ...   # Not needed if using Rails
+
+    require 'earth'
+    Earth.init :all, :apply_schemas => true
+
+### Pulling data from data.brighterplanet.com
+
+By default, Earth will pull data from data.brighterplanet.com. Simply call `run_data_miner!` on whichever data model class you need. If there are any Earth classes that the chosen class depends on, they will be downloaded as well automatically:
+
+    require 'earth'
+    Earth.init :locality
+    ZipCode.run_data_miner!
+
+### Pulling data from the original sources
+
+If you'd like to pull data directly from the source, e.g. Automobile data from EPA's sources, simply require the data\_miner file for the desired domain:
+
+    require 'earth'
+    Earth.init :automobile
+
+    require 'earth/automobile/data_miner'
+    AutomobileMake.run_data_miner!
+
 ## Collaboration cycle 
 Brighter Planet vigorously encourages collaborative improvement of its emitter libraries. Collaboration requires a (free) GitHub account.
 

data/bin/earth_tester.rb

@@ -79,7 +79,7 @@ if (resource = ARGV[1].to_s.camelcase).present?
   show_resource resource
 else
   DataMiner.run
-  Earth.search(domain).each do |resource|
+  Earth.resources(domain).each do |resource|
     show_resource resource
   end
 end

data/earth.gemspec

@@ -31,11 +31,11 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'to_regexp'
   s.add_runtime_dependency 'cohort_scope'
   s.add_runtime_dependency 'table_warnings', '>=0.0.6'
-  s.add_runtime_dependency 'remote_table', '>=1.2.3'
+  s.add_runtime_dependency 'remote_table', '>=1.3.0'
   s.add_runtime_dependency 'falls_back_on'
   s.add_runtime_dependency 'fixed_width-multibyte'
-  s.add_runtime_dependency 'geokit-rails'
-  s.add_runtime_dependency 'loose_tight_dictionary', '>=0.2.3'
+  s.add_runtime_dependency 'geokit'
+  s.add_runtime_dependency 'fuzzy_match', '>=1.1.0'
   s.add_runtime_dependency 'weighted_average', '>=1.0.2'
   s.add_runtime_dependency 'mini_record-compat' # need https://github.com/DAddYE/mini_record/pull/7
   s.add_runtime_dependency 'activesupport'

data/lib/earth.rb

@@ -10,55 +10,44 @@ require 'fixed_width'
 require 'errata'
 require 'mini_record'
 require 'table_warnings'
-require 'loose_tight_dictionary'
-require 'loose_tight_dictionary/cached_result'
+require 'fuzzy_match'
+require 'fuzzy_match/cached_result'
 
 # The earth module is an interface for loading data models from various domains.
 module Earth
-  autoload :Utils, 'earth/utils'
-  
+  extend ::ActiveSupport::Memoizable
   extend self
 
-  # Takes argument like Earth.search(['air'])
-  # Default is search all domains
-  # For example, <tt>[ 'Aircraft', 'Airline' ]</tt>
-  def search(*search_domains)
-    search_domains = search_domains.empty? ? [:all] : search_domains.flatten.map(&:to_sym)
-    if search_domains.include? :all
-      resources
-    else
-      resource_map.select do |resource, domain|
-        search_domains.include? domain.to_sym
-      end.map do |resource, domain|
-        resource
+  def domains
+    ::Dir[::File.join(lib_dir, '*')].map do |path|
+      if ::File.directory? path
+        ::File.basename path
       end
-    end
+    end.compact.uniq.sort
   end
-
-  def gem_root 
-    File.expand_path File.join(File.dirname(__FILE__), '..')
+  memoize :domains
+  
+  def resources(*search_domains)
+    search_domains = search_domains.flatten.compact.map(&:to_s)
+    if search_domains.empty?
+      search_domains = domains
+    end
+    search_domains.map do |domain|
+      ::Dir[::File.join(lib_dir, domain, '**', '*.rb')].map do |possible_resource|
+        unless possible_resource.include?('data_miner')
+          ::File.basename(possible_resource, '.rb').camelcase
+        end
+      end
+    end.flatten.compact.sort
   end
+  memoize :resources
 
-  def domains
-    resource_map.values.uniq.sort
+  def vendor_dir
+    ::File.expand_path '../../vendor', __FILE__
   end
   
-  def resources
-    resource_map.keys.sort
-  end
-
-  def resource_map
-    @resource_map ||= Dir[File.join(gem_root, 'lib', 'earth', '*')].select do |path|
-      File.directory? path
-    end.inject({}) do |memo, domain_path|
-      Dir[File.join(domain_path, '*.rb')].each do |resource_path|
-        resource = File.basename(resource_path, '.rb').camelcase
-        unless resource == 'DataMiner'
-          memo[resource] = File.basename domain_path
-        end
-      end
-      memo
-    end
+  def lib_dir
+    ::File.expand_path '../earth', __FILE__
   end
 
   # Earth.init will load any specified domains, any needed ActiveRecord plugins, 
@@ -69,51 +58,94 @@ module Earth
   # Earth.init should be performed after a connection is made to the database and 
   # before any domain models are referenced.
   def init(*args)
-    options = args.last.is_a?(Hash) ? args.pop.symbolize_keys : {}
-    domains = args.empty? ? [ :all ] : args.map(&:to_sym)
+    options = args.extract_options!
+    domains = args
+
+    options[:load_data_miner] = true if options[:apply_schemas]
 
     _warn_unless_mysql_ansi_mode
     _load_plugins
-    _load_domains domains, options
-    _decorate_resources options
-    _load_schemas search(domains), options
+    
+    if domains.include?(:none)
+      # don't load anything
+    elsif domains.include?(:all) or domains.empty?
+      require_all options
+    else
+      domains.each do |domain|
+        require_domain domain, options
+      end
+    end
+    
+    # be sure to look at both explicitly and implicitly loaded resources
+    resources.each do |resource|
+      next unless ::Object.const_defined?(resource)
+      resource_model = resource.constantize
+      
+      _append_pull_dependencies_step_to_data_miner resource_model
+
+      if options[:load_data_miner]
+        _prepend_auto_upgrade_step_to_data_miner resource_model
+      else
+        _prepend_taps_step_to_data_miner resource_model
+      end
+
+      if options[:apply_schemas]
+        resource_model.auto_upgrade!
+      end
+    end
+  end
+
+  # internal use
+  def require_related(path)
+    path = ::File.expand_path path
+    raise ::ArgumentError, %{[earth gem] #{path} is not in #{lib_dir}} unless path.start_with?(lib_dir)
+    domain = %r{#{lib_dir}/([^\./]+)}.match(path).captures.first
+    require_domain domain, :load_data_miner => path.include?('data_miner')
+  end
+  
+  # internal use
+  def require_all(options = {})
+    require_glob ::File.join(lib_dir, '**', '*.rb'), options
   end
 
   private
   
-  def _warn_unless_mysql_ansi_mode
-    if ActiveRecord::Base.connection.adapter_name =~ /mysql/i
-      sql_mode = ActiveRecord::Base.connection.select_value("SELECT @@GLOBAL.sql_mode") + ActiveRecord::Base.connection.select_value("SELECT @@SESSION.sql_mode")
-      $stderr.puts "[earth gem] Warning: MySQL detected, but PIPES_AS_CONCAT not set. Importing from scratch will fail. Consider setting sql-mode=ANSI in my.cnf." unless sql_mode =~ /pipes_as_concat/i
-    end
+  def require_domain(domain, options = {})
+    require_glob ::File.join(lib_dir, domain.to_s, '**', '*.rb'), options 
   end
   
-  # TODO sabshere don't use directories to specify domains
-  # * you have 20 million data_miner.rb files which are easy to confuse
-  # * you have to go all over the filesystem to figure things out
-  def _load_domains(domains, options)
-    return if domains.include? :none
-    if domains.empty? or domains.include?(:all)
-      # sabshere 9/16/10 why maintain this separately?
-      require 'earth/all'
-      require 'earth/data_miner' if options[:apply_schemas] or options[:load_data_miner]
-    else
-      domains.each do |domain| 
-        require "earth/#{domain}"
-        require "earth/#{domain}/data_miner" if options[:apply_schemas] or options[:load_data_miner]
+  def require_glob(glob, options = {})
+    data_miner_paths = []
+    ::Dir[glob].each do |path|
+      if path.include?('data_miner')
+        data_miner_paths << path
+      else
+        require path
       end
     end
+    # load data_miner blocks second to make sure they override
+    data_miner_paths.each do |path|
+      require path
+    end if options[:load_data_miner]
+    nil
   end
-
+  memoize :require_glob
+  
+  def _warn_unless_mysql_ansi_mode
+    if ::ActiveRecord::Base.connection.adapter_name =~ /mysql/i
+      sql_mode = ::ActiveRecord::Base.connection.select_value("SELECT @@GLOBAL.sql_mode") + ::ActiveRecord::Base.connection.select_value("SELECT @@SESSION.sql_mode")
+      $stderr.puts "[earth gem] Warning: MySQL detected, but PIPES_AS_CONCAT not set. Importing from scratch will fail. Consider setting sql-mode=ANSI in my.cnf." unless sql_mode =~ /pipes_as_concat/i
+    end
+  end
+  
   def _load_plugins
-    Dir[File.join(Earth.gem_root, 'vendor', '**', 'init.rb')].each do |pluginit|
-      $:.unshift File.join(File.dirname(pluginit), 'lib')
-      load pluginit
+    ::Dir[::File.expand_path('../../vendor/**/init.rb', __FILE__)].each do |pluginit|
+      $LOAD_PATH.unshift ::File.join(::File.dirname(pluginit), 'lib')
+      ::Kernel.load pluginit
     end
   end
   
-  def _append_pull_dependencies_step_to_data_miner(resource)
-    resource_model = resource.constantize
+  def _append_pull_dependencies_step_to_data_miner(resource_model)
     return if resource_model.data_miner_config.steps.any? { |step| step.description == :run_data_miner_on_parent_associations! }
 
     pull_dependencies_step = DataMiner::Process.new resource_model.data_miner_config, :run_data_miner_on_parent_associations!
@@ -121,8 +153,7 @@ module Earth
     resource_model.data_miner_config.steps.push pull_dependencies_step
   end
   
-  def _prepend_auto_upgrade_step_to_data_miner(resource)
-    resource_model = resource.constantize
+  def _prepend_auto_upgrade_step_to_data_miner(resource_model)
     return if resource_model.data_miner_config.steps.any? { |step| step.description == :auto_upgrade! }
 
     auto_upgrade_step = DataMiner::Process.new resource_model.data_miner_config, :auto_upgrade!
@@ -132,31 +163,15 @@ module Earth
   
   TAPS_STEP = 'Tap the Brighter Planet data server'
   TAPS_SOURCE = 'http://carbon:neutral@data.brighterplanet.com:5000'
-  def _prepend_taps_step_to_data_miner(resource)
-    resource_model = resource.constantize
+  def _prepend_taps_step_to_data_miner(resource_model)
     return if resource_model.data_miner_config.steps.any? { |step| step.description == TAPS_STEP }
     
     taps_step = DataMiner::Tap.new resource_model.data_miner_config, TAPS_STEP, TAPS_SOURCE
     
     resource_model.data_miner_config.steps.unshift taps_step
   end
-  
-  def _decorate_resources(options)
-    resources.each do |resource|
-      next unless ::Object.const_defined?(resource)
-      _append_pull_dependencies_step_to_data_miner resource
-      if options[:apply_schemas] or options[:load_data_miner]
-        _prepend_auto_upgrade_step_to_data_miner resource
-      else
-        _prepend_taps_step_to_data_miner resource
-      end
-    end
-  end
-
-  def _load_schemas(selected_resources, options)
-    return unless options[:apply_schemas]
-    selected_resources.each do |resource|
-      resource.constantize.auto_upgrade!
-    end
-  end
 end
+
+require 'earth/utils'
+# TODO move this into a gem or into its own namespace in this gem
+require ::File.join(Earth.vendor_dir, 'clean_find_in_batches', 'init')

data/lib/earth/air.rb

@@ -1,12 +1,3 @@
-require 'earth/air/aircraft'
-require 'earth/air/aircraft_instance'
-require 'earth/air/aircraft_instance_seat_class'
-require 'earth/air/airline'
-require 'earth/air/airport'
-require 'earth/air/bts_aircraft'
-require 'earth/air/flight_distance_class'
-require 'earth/air/flight_distance_class_seat_class'
-require 'earth/air/flight_seat_class'
-require 'earth/air/flight_segment'
-
-require 'earth/locality'
+# lib/earth/air.rb
+require 'earth'
+Earth.require_related __FILE__

data/lib/earth/air/aircraft.rb

@@ -2,12 +2,12 @@ class Aircraft < ActiveRecord::Base
   set_primary_key :icao_code
   
   # Fuzzy association with FlightSegment
-  cache_loose_tight_dictionary_matches_with :flight_segments, :primary_key => :description, :foreign_key => :aircraft_description
+  cache_fuzzy_match_with :flight_segments, :primary_key => :description, :foreign_key => :aircraft_description
   
   class << self
-    # set up a loose_tight_dictionary for matching Aircraft description with FlightSegment aircraft_description
-    def loose_tight_dictionary
-      @loose_tight_dictionary ||= LooseTightDictionary.new(Aircraft.all,
+    # set up a fuzzy_match for matching Aircraft description with FlightSegment aircraft_description
+    def fuzzy_match
+      @fuzzy_match ||= FuzzyMatch.new(Aircraft.all,
           :haystack_reader => lambda { |record| record.description },
           :blockings  => RemoteTable.new(:url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDlRR2NmdzE2ZjZwTy1ucjh4cWFYOFE&gid=0&output=csv').map { |record| record['blocking'] },
           :identities => RemoteTable.new(:url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdDlRR2NmdzE2ZjZwTy1ucjh4cWFYOFE&gid=1&output=csv').map { |record| record['identity'] },
@@ -61,7 +61,7 @@ class Aircraft < ActiveRecord::Base
     # Cache fuzzy matches between FlightSegment aircraft_description and Aircraft description
     def manually_cache_flight_segments!
       FlightSegment.run_data_miner!
-      LooseTightDictionary::CachedResult.setup
+      FuzzyMatch::CachedResult.setup
       connection.select_values("SELECT DISTINCT(aircraft_description) FROM flight_segments WHERE aircraft_description IS NOT NULL").each do |original_description|
         # If the flight segment's aircraft_description contains '/' then it describes multiple aircraft.
         # We need to synthesize descriptions for those aircraft, find all Aircraft that fuzzily match the
@@ -86,20 +86,20 @@ class Aircraft < ActiveRecord::Base
           suffixes.map{ |suffix| root + suffix }.each do |synthesized_description|
             # Look up the Aircraft that match each synthesized description and associate
             # them with the original flight segment aircraft_description
-            Aircraft.loose_tight_dictionary.find_all(synthesized_description).each do |aircraft|
+            Aircraft.fuzzy_match.find_all(synthesized_description).each do |aircraft|
               attrs = {
                 :a_class => "Aircraft",
                 :a => aircraft.description,
                 :b_class => "FlightSegment",
                 :b => original_description
               }
-              unless ::LooseTightDictionary::CachedResult.exists? attrs
-                ::LooseTightDictionary::CachedResult.create! attrs
+              unless ::FuzzyMatch::CachedResult.exists? attrs
+                ::FuzzyMatch::CachedResult.create! attrs
               end
             end
           end
         # If the flight segment's aircraft_description doesn't contain '/' we can use
-        # a method provided by loose_tight_dictionary to associate it with Aircraft
+        # a method provided by fuzzy_match to associate it with Aircraft
         else
           FlightSegment.find_by_aircraft_description(original_description).cache_aircraft!
         end

data/lib/earth/air/airport.rb

@@ -1,3 +1,6 @@
+require ::File.join(Earth.vendor_dir, 'geokit-rails', 'lib', 'geokit-rails')
+require 'earth/locality'
+
 class Airport < ActiveRecord::Base
   set_primary_key :iata_code
   

data/lib/earth/air/airport/data_miner.rb

@@ -1,3 +1,5 @@
+require 'earth/locality/data_miner'
+
 Airport.class_eval do
   class Airport::Guru
     def method_missing(method_id, *args, &block)

data/lib/earth/air/data_miner.rb

@@ -1,10 +1,3 @@
-require 'earth/air/aircraft/data_miner'
-require 'earth/air/airline/data_miner'
-require 'earth/air/airport/data_miner'
-require 'earth/air/bts_aircraft/data_miner'
-require 'earth/air/flight_distance_class/data_miner'
-require 'earth/air/flight_distance_class_seat_class/data_miner'
-require 'earth/air/flight_seat_class/data_miner'
-require 'earth/air/flight_segment/data_miner'
-
-require 'earth/locality/data_miner'
+# lib/earth/air/data_miner.rb
+require 'earth'
+Earth.require_related __FILE__

data/lib/earth/air/flight_segment.rb

@@ -1,3 +1,5 @@
+require 'earth/locality'
+
 class FlightSegment < ActiveRecord::Base
   set_primary_key :row_hash
   
@@ -12,7 +14,7 @@ class FlightSegment < ActiveRecord::Base
   # has_many :destination_city_airports, :foreign_key => 'city', :primary_key => 'destination_airport_city', :class_name => 'Airport'
   
   # Enable flight_segment.aircraft
-  cache_loose_tight_dictionary_matches_with :aircraft, :primary_key => :aircraft_description, :foreign_key => :description
+  cache_fuzzy_match_with :aircraft, :primary_key => :aircraft_description, :foreign_key => :description
   
   class << self
     def update_averages!

data/lib/earth/air/flight_segment/data_miner.rb

@@ -1,3 +1,5 @@
+require 'earth/locality/data_miner'
+
 FlightSegment.class_eval do
   # For import errata
   class FlightSegment::Guru

data/lib/earth/all.rb

@@ -1,13 +1,3 @@
-require 'earth/air'
-require 'earth/automobile'
-require 'earth/bus'
-require 'earth/computation'
-require 'earth/diet'
-require 'earth/fuel'
-require 'earth/hospitality'
-require 'earth/industry'
-require 'earth/locality'
-require 'earth/pet'
-require 'earth/rail'
-require 'earth/residence'
-require 'earth/shipping'
+# lib/earth/all.rb
+require 'earth'
+Earth.require_all