earth 0.12.4 → 1.0.0

data/.gitignore

@@ -22,4 +22,7 @@ log
 
 ## PROJECT::SPECIFIC
 Gemfile.lock
-*.gem
+*.gem
+earth_tester.sqlite3
+
+doc

data/Gemfile

@@ -2,11 +2,6 @@ source :rubygems
 
 gemspec
 
-group :development do
-  gem 'guard'
-  gem 'pry'
-end
-
 unless RUBY_VERSION >= '1.9'
   gem 'fastercsv'
 end

data/README.markdown

@@ -1,26 +1,28 @@
 # earth
 
-Earth is a collection of data models that represent various things found here on Earth, such as pet breeds, kinds of rail travel, zip codes, and Petroleum Administration for Defense Districts.
+Earth is a collection of *data models* that represent various things found here on Earth, such as countries, automobiles, aircraft, zip codes, and pet breeds.
 
-The data that these models represent can be pulled from preconfigured authoritative sources.
+By default the data that these models represent is pulled from [Brighter Planet's open reference data site](http://data.brighterplanet.com) using the [taps gem](http://rubygems.org/gems/taps). The data can also be imported directly from preconfigured authoritative sources. 
 
 ## Usage
 
 ``` ruby
 require 'earth'
-Earth.init :automobile, :locality
+require 'earth/automobile/automobile_fuel'
+
+Earth.init
 ft = AutomobileFuel.first
 # ...
 ```
 
-`Earth.init` loads desired "data domains" as well as any supporting classes and plugins that each data model needs. A "data domain" is a grouping of related data models. For instance, all automobile-related data is in the `:automobile` domain.
+`Earth.init` prepares the environment to load and download data for each data model. You can load all data models at once with `Earth.init :all`. There are several other options to `init` that configure data mining sources and database connections. See the [rdocs](http://rdoc.info/github/brighterplanet/earth) for more details on the Earth module.
 
-### Domains
+### Data model categories
 
 <table>
   <thead>
   <tr>
-    <th>Domain</th>
+    <th>Category</th>
     <th>Models</th>
   </tr>
   </thead>
@@ -83,15 +85,7 @@ ft = AutomobileFuel.first
 
 ### Data storage
 
-You can store Earth data in any relational database. On your very first run, you will need to create the tables for data each model. This is done using [the `active_record_inline_schema` library](https://github.com/seamusabshere/active_record_inline_schema) if you pass the `apply_schemas` option:
-
-``` ruby
-require 'activerecord'
-ActiveRecord::Base.establish_connection :adapter => ...   # Not needed if using Rails
-
-require 'earth'
-Earth.init :all, :apply_schemas => true
-```
+You can store Earth data in any relational database. On your very first run, you will need to create the tables for data each model. You can either use the Rails standard rake tasks (see below) or with a call to `Earth.reset_schemas!`
 
 ### Pulling data from data.brighterplanet.com
 
@@ -99,22 +93,41 @@ By default, Earth will pull data from [data.brighterplanet.com](http://data.brig
 
 ``` ruby
 require 'earth'
-Earth.init :locality
+require 'earth/locality/zip_code'
+
+Earth.init
 ZipCode.run_data_miner!
 ```
 
 ### Pulling data from the original sources
 
-If you'd like to bypass the [data.brighterplanet.com](http://data.brighterplanet.com) proxy and pull data directly from authoritative sources (*e.g.,* automobile data from EPA), simply require the `data\_miner` file for the desired domain:
+If you'd like to bypass the [data.brighterplanet.com](http://data.brighterplanet.com) proxy and pull data directly from authoritative sources (*e.g.,* automobile data from EPA), simply specify the :mine_original_sources option to `Earth.init`
 
 ``` ruby
 require 'earth'
-Earth.init :automobile
+Earth.init :mine_original_sources => true
 
-require 'earth/automobile/data_miner'
+require 'earth/automobile'
 AutomobileMake.run_data_miner!
 ```
 
+### Rake tasks
+
+Earth provides handy rails tasks for creating, migrating, and data mining models whether you're using it from a Rails app or a standalone Ruby app.
+
+In your Rakefile, add:
+
+    require 'earth/tasks'
+    Earth::Tasks.new
+
+If you're using Earth outside of Rails, all of the default `rake db:*` tasks will now be available. Within rails, certain tasks are augmented to 
+help manage your Earth models using data_miner and active_record_inline_schema in addition to standard migrations.
+
+Of note are the following tasks:
+
+* `rake db:migrate` runs `.create_table!` on each Earth resource model.
+* `rake db:seed` runs `.run_data_miner!` on each Earth resource model.
+
 ## Collaboration cycle 
 Brighter Planet vigorously encourages collaborative improvement.
 

data/Rakefile

@@ -62,3 +62,7 @@ Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_files.include('README*')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
+
+ENV['DATABASE_URL'] ||= 'mysql2://root:password@localhost/test_earth'
+require 'earth/tasks'
+Earth::Tasks.new

data/bin/earth_tester.rb

@@ -16,66 +16,78 @@ if File.exist?(File.join(Dir.pwd, 'earth.gemspec'))
   $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 end
 
-require 'active_support/all'
-require 'active_record'
+require 'thor'
 
-case ENV['DATABASE']
-when /postgr/i
-  system %{dropdb test_earth}
-  system %{createdb test_earth}
-  ActiveRecord::Base.establish_connection(
-    'adapter' => 'postgresql',
-    'encoding' => 'utf8',
-    'database' => 'test_earth',
-    'username' => `whoami`.chomp
-  )
-when /sqlite/i
-  ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":memory:")
-else
-  system %{mysql -u root -ppassword -e "DROP DATABASE test_earth"} if ENV['RESET_DB'] == 'true'
-  system %{mysql -u root -ppassword -e "CREATE DATABASE test_earth CHARSET utf8"}
-  ActiveRecord::Base.establish_connection(
-    'adapter' => (RUBY_PLATFORM == 'java' ? 'mysql' : 'mysql2'),
-    'encoding' => 'utf8',
-    'database' => 'test_earth',
-    'username' => 'root',
-    'password' => 'password'
-  )
-end
+class EarthTester < Thor
+  desc :console, "Get a console"
+  method_option :domains, :type => :array, :default => []
+  method_option :database, :type => :string, :default => 'mysql'
+  method_option :reset, :type => :boolean, :desc => 'Fully reset databases'
+  method_option :mine_original_sources, :type => :boolean, :desc => "Load full data_miner blocks"
+  def console
+    earth_options = options.inject({}) do |memo, (k, v)|
+      k = k.to_sym
+      if [:mine_original_sources].include?(k)
+        memo[k] = v
+      end
+      memo
+    end
+    environment
+    Earth.init(*(options.domains.map(&:to_sym)+[earth_options]))
+    require 'pry'
+    Pry.color = false
+    binding.pry
+  end
 
-require 'earth'
+  private
+  
+  def environment
+    require 'active_support/all'
+    require 'active_record'
 
-DataMiner.unit_converter = :conversions
+    # TODO convert to @dkastner's Earth.database_configurations
+    case options.database
+    when /postgr/i
+      if options.reset
+        system %{dropdb test_earth}
+        system %{createdb test_earth}
+      end
+      ActiveRecord::Base.establish_connection(
+        'adapter' => 'postgresql',
+        'encoding' => 'utf8',
+        'database' => 'test_earth',
+        'username' => `whoami`.chomp
+      )
+    when /sqlite/i
+      if options.reset
+        FileUtils.rm_f 'earth_tester.sqlite3'
+      end
+      ActiveRecord::Base.establish_connection('adapter' => 'sqlite3', 'database' => 'earth_tester.sqlite3')
+    when /mysql/i
+      if options.reset
+        system %{mysql -u root -ppassword -e "DROP DATABASE test_earth"}
+        system %{mysql -u root -ppassword -e "CREATE DATABASE test_earth CHARSET utf8"}
+      end
+      ActiveRecord::Base.establish_connection(
+        'adapter' => (RUBY_PLATFORM == 'java' ? 'mysql' : 'mysql2'),
+        'encoding' => 'utf8',
+        'database' => 'test_earth',
+        'username' => 'root',
+        'password' => 'password'
+      )
+    else
+      raise "not sure what database to connect to - #{options.database.inspect}"
+    end
 
-DataMiner::Run.auto_upgrade!
+    require 'earth'
 
-ActiveRecord::Base.logger = Logger.new $stderr
-ActiveRecord::Base.logger.level = Logger::INFO
+    DataMiner.unit_converter = :conversions
 
-def show_resource(resource)
-  resource_model = resource.constantize
-  if (warnings = resource_model.table_warnings).any?
-    $stderr.puts
-    $stderr.puts '#'*50
-    $stderr.puts "# #{resource}"
-    $stderr.puts '#'*50
-    $stderr.puts
-    warnings.each do |warning|
-      $stderr.puts "* #{warning}"
-    end
-  end
-end
-
-def init(domain)
-  Earth.init domain, :load_data_miner => true, :apply_schemas => true
-end
+    DataMiner::Run.auto_upgrade!
 
-if ARGV[0]
-  init ARGV[0]
+    ActiveRecord::Base.logger = Logger.new $stdout
+    ActiveRecord::Base.logger.level = Logger::DEBUG
+  end
 end
 
-require 'pry'
-Pry.color = false
-
-# you prob want to init() something
-binding.pry
+EarthTester.start

data/earth.gemspec

@@ -20,12 +20,11 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'activesupport'
   s.add_runtime_dependency 'cohort_analysis'
   s.add_runtime_dependency 'conversions'
-  s.add_runtime_dependency 'data_miner', '>=2.2.0'
+  s.add_runtime_dependency 'data_miner', '>=2.4.0'
   s.add_runtime_dependency 'falls_back_on'
   s.add_runtime_dependency 'fixed_width-multibyte'
   s.add_runtime_dependency 'fuzzy_match', '>=1.3.3'
   s.add_runtime_dependency 'geokit'
-  s.add_runtime_dependency 'active_record_inline_schema'
   s.add_runtime_dependency 'remote_table', '>=2.0.2'
   s.add_runtime_dependency 'table_warnings', '>=1.0.1'
   s.add_runtime_dependency 'to_regexp'
@@ -35,10 +34,14 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'charisma'
   s.add_development_dependency 'cucumber'
+  s.add_development_dependency 'factory_girl'
   s.add_development_dependency 'mysql2' # for bin/earth_tester.rb; use mysql2 for utf-8 compatibility
   s.add_development_dependency 'pg'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'rdoc'
   s.add_development_dependency 'rspec'
-  s.add_development_dependency 'sqlite3-ruby'
+  s.add_development_dependency 'sandbox'
+  s.add_development_dependency 'sqlite3'
+  s.add_development_dependency 'thor'
+  s.add_development_dependency 'pry'
 end

data/lib/earth.rb

@@ -1,147 +1,100 @@
 require 'active_support/core_ext'
+require 'active_support/string_inquirer'
 require 'active_record'
 require 'data_miner'
-require 'falls_back_on'
 require 'weighted_average'
 require 'fixed_width'
 require 'errata'
-require 'active_record_inline_schema'
-require 'table_warnings'
 require 'fuzzy_match'
 
-require 'earth/utils'
 require 'earth/conversions_ext'
 require 'earth/inflectors'
+require 'earth/loader'
+require 'earth/model'
+require 'earth/utils'
+require 'earth/warnings'
 
-require 'earth/active_record_base_class_methods'
-ActiveRecord::Base.extend Earth::ActiveRecordBaseClassMethods
-
-# The earth module is an interface for loading data models from various domains.
+# The earth module is an interface for loading data models
 module Earth
-  TAPS_SOURCE = 'http://carbon:neutral@data.brighterplanet.com:5000'
-  TAPS_DESCRIPTION = "Brighter Planet's reference data web service"
   VENDOR_DIR = ::File.expand_path '../../vendor', __FILE__
   LIB_DIR = ::File.expand_path '../earth', __FILE__
   DATA_DIR = ::File.expand_path '../../data', __FILE__
   ERRATA_DIR = ::File.expand_path '../../errata', __FILE__
 
-  def Earth.domains
-    @domains ||= ::Dir[::File.join(LIB_DIR, '*')].map do |path|
-      if ::File.directory? path
-        ::File.basename path
-      end
-    end.compact.uniq.sort
-  end
-  
-  def Earth.resources(*search_domains)
-    search_domains = search_domains.flatten.compact.map(&:to_s)
-    if search_domains.empty?
-      search_domains = domains
-    end
-    search_domains.map do |domain|
-      ::Dir[::File.join(LIB_DIR, domain, '**', '*.rb')].map do |possible_resource|
-        unless possible_resource.include?('data_miner')
-          ::File.basename(possible_resource, '.rb').camelcase
-        end
-      end
-    end.flatten.compact.sort
-  end
+  mattr_accessor :mine_original_sources
+  mattr_accessor :database_configurations
 
-  # Earth.init will load any specified domains, any needed ActiveRecord plugins, 
-  # and will apply each domain model's schema to the database if the 
-  # :apply_schemas option is given. See Earth.domains for the list of allowable 
-  # domains.
+  # Earth.init is the gateway to using Earth. It can load all models at
+  # once, connect to the database using Rails conventions, and set up
+  # the models to pull data from original sources instead of Brighter 
+  # Planet's pre-processed data service.
   #
-  # Earth.init should be performed after a connection is made to the database and 
-  # before any domain models are referenced.
+  # @param [Symbol] load_directive use `:all` to load all models at once (optional)
+  # @param [Hash] options load options
+  # * :mine_original_sources, if true, will load files necessary to data mine from scratch rather than downloading from data.brighterplanet.com. Note that you must run Earth.init before requiring models in order for this option to work properly.
+  # * :connect will connect to the database for you
   def Earth.init(*args)
     options = args.extract_options!
-    domains = args
 
-    warn_unless_mysql_ansi_mode
-    load_plugins
-    
-    if domains.include?(:none)
-      # don't load anything
-    elsif domains.include?(:all) or domains.empty?
-      require_all options
-    else
-      domains.each do |domain|
-        require_domain domain, options
-      end
-    end
+    connect if options[:connect]
+
+    Warnings.check_mysql_ansi_mode
+
+    Earth.mine_original_sources = options[:load_data_miner] || options[:mine_original_sources]
     
-    # be sure to look at both explicitly and implicitly loaded resources
-    resources.select do |resource|
-      ::Object.const_defined?(resource)
-    end.each do |resource|
-      resource_model = resource.constantize
-      unless options[:skip_parent_associations]
-        resource_model.data_miner_script.append_once :process, :run_data_miner_on_parent_associations!
-      end
-      if options[:load_data_miner]
-        resource_model.data_miner_script.prepend_once :process, :auto_upgrade!
-      else
-        resource_model.data_miner_script.prepend_once :tap, TAPS_DESCRIPTION, TAPS_SOURCE
-      end
-      if options[:apply_schemas]
-        resource_model.auto_upgrade!
+    if args.include? :all
+      require 'earth/all'
+    elsif args.length > 0
+      Kernel.warn "Deprecation Warning: `Earth.init :domain` will be removed. Use `require 'earth/domain'` instead"
+      args.each do |argh|
+        require "earth/#{argh}"
       end
     end
   end
-
-  # internal use
-  def Earth.require_related(path)
-    path = ::File.expand_path path
-    raise ::ArgumentError, %{[earth gem] #{path} is not in #{LIB_DIR}} unless path.start_with?(LIB_DIR)
-    domain = %r{#{LIB_DIR}/([^\./]+)}.match(path).captures.first
-    require_domain domain, :load_data_miner => path.include?('data_miner')
-  end
   
-  # internal use
-  def Earth.require_all(options = {})
-    require_glob ::File.join(LIB_DIR, '**', '*.rb'), options
+  # List the currently loaded data model class names.
+  #
+  # @return [Array] a list of camelized resource names
+  def Earth.resources
+    @resources ||= Earth.resource_models.map(&:to_s).sort
   end
-  
-  private
-  
-  def Earth.require_domain(domain, options = {})
-    require_glob ::File.join(LIB_DIR, domain.to_s, '**', '*.rb'), options 
+
+  # List the currently loaded data model classes
+  #
+  # @return [Array] a list of resource classes
+  def Earth.resource_models
+    Earth::Model.registry
   end
-  
-  def Earth.require_glob(glob, options = {})
-    @require_glob ||= []
-    args = [glob, options]
-    return if @require_glob.include?(args)
-    @require_glob << args
-    data_miner_paths = []
-    ::Dir[glob].each do |path|
-      if path.include?('data_miner')
-        data_miner_paths << path
-      else
-        require path
-      end
-    end
-    # load data_miner blocks second to make sure they override
-    data_miner_paths.each do |path|
-      require path
-    end if options[:load_data_miner]
-    nil
+
+  # Connect to the database using ActiveRecord's default behavior
+  def Earth.connect
+    ActiveRecord::Base.establish_connection
+    ActiveRecord::Base.connection
   end
-  
-  def Earth.warn_unless_mysql_ansi_mode
-    if ::ActiveRecord::Base.connection.adapter_name =~ /mysql/i
-      sql_mode = ::ActiveRecord::Base.connection.select_value("SELECT @@GLOBAL.sql_mode") + ::ActiveRecord::Base.connection.select_value("SELECT @@SESSION.sql_mode")
-      unless sql_mode.downcase.include? 'pipes_as_concat'
-        ::Kernel.warn "[earth gem] Warning: MySQL detected, but PIPES_AS_CONCAT not set. Importing from scratch will fail. Consider setting sql-mode=ANSI in my.cnf."
-      end
-    end
+
+  # The current environment. Earth detects the following environment variables:
+  #
+  # * EARTH_ENV (for CLI apps and daemons)
+  # * RAILS_ENV
+  # * RACK_ENV
+  #
+  # Default is `development`
+  def Earth.env
+    @env ||= ActiveSupport::StringInquirer.new(ENV['EARTH_ENV'] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development')
   end
-  
-  def Earth.load_plugins
-    ::Dir[::File.expand_path('../../vendor/**/init.rb', __FILE__)].each do |pluginit|
-      $LOAD_PATH.unshift ::File.join(::File.dirname(pluginit), 'lib')
-      ::Kernel.load pluginit
-    end
+
+  # Drop and recreate tables for all currently loaded data models.
+  #
+  def Earth.reset_schemas!
+    Earth.resource_models.each(&:create_table!)
+  end
+
+  # Run data miner on all currently loaded data models.
+  #
+  # @note By default, data is mined from data.brighterplanet.com 
+  # via taps. In order to mine from scratch, call Earth.init 
+  # with the :mine_original_sources option.
+  def Earth.run_data_miner!
+    DataMiner.run(Earth.resources)
   end
 end