earth 0.12.4 → 1.0.0

data/lib/earth/locality/zip_code/data_miner.rb

@@ -1,9 +1,5 @@
 ZipCode.class_eval do
   data_miner do
-    process "Start from scratch" do
-      delete_all
-    end
-    
     import 'the Mapping Hacks zipcode database',
            :url => 'http://mappinghacks.com/data/zipcode.zip', # http://archive.data.brighterplanet.com/zipcode.zip
            :filename => 'zipcode.csv' do

data/lib/earth/model.rb

@@ -0,0 +1,90 @@
+require 'earth'
+require 'table_warnings'
+
+module Earth
+  module Model
+    def Model.extended(target)
+      target.extend SafeFinders
+      target.extend Schema
+      target.mattr_accessor :source_file
+      self.registry << target
+      target.source_file = caller.first.split(':').first
+
+      extend_mining(target)
+    end
+
+    def Model.registry
+      @registry ||= []
+    end
+
+    def Model.extend_mining(target)
+      if Earth.mine_original_sources
+        path = File.join(File.dirname(target.source_file), File.basename(target.source_file, '.rb'), 'data_miner')
+        require path if File.exist?(path + '.rb')
+        target.data_miner_script.prepend_once :process, :create_table!
+      else
+        target.data_miner_script.prepend_once :sql, "Brighter Planet's reference data", "http://data.brighterplanet.com/#{to_s.underscore.pluralize}.sql"
+      end
+      target.data_miner_script.append_once :process, :run_data_miner_on_parent_associations!
+    end
+
+    module Schema
+      # http://ostermiller.org/findcomment.html
+      COMMENT = %r{/\*(?:.|[\r\n])*?\*/}
+      WHITESPACE = /\s+/
+      SEMICOLON = / ?; ?/
+
+      def create_table!(force = true)
+        c = ActiveRecord::Base.connection_pool.checkout
+
+        if c.table_exists?(table_name)
+          return unless force
+          c.drop_table table_name
+        end
+
+        statements = const_get(:TABLE_STRUCTURE).gsub(COMMENT, '').gsub(WHITESPACE, ' ').split(SEMICOLON).select(&:present?)
+
+        statements.each do |sql|
+          c.execute sql
+        end
+
+        # safely reset column information
+        if c.respond_to?(:schema_cache)
+          c.schema_cache.clear!
+        end
+        reset_column_information
+        descendants.each do |descendant|
+          descendant.reset_column_information
+        end
+
+        nil
+      ensure
+        ActiveRecord::Base.connection_pool.checkin c
+      end
+    end
+
+    module SafeFinders
+      # http://www.seejohncode.com/tag/rails/
+      # Override due to implementation of regular find_in_batches conflicting using UUIDs
+      def safe_find_in_batches(&block)
+        return find_in_batches({}, &block) if relation.primary_key.is_a?(Arel::Attributes::Integer)
+
+        batch_size = 1000
+        offset = 0
+        # Get the relation and keep going over it until there's nothing left
+        relation = order("#{quoted_table_name}.#{quoted_primary_key} ASC").limit(batch_size)
+        while (results = relation.offset(offset).limit(batch_size).all).any?
+          unscoped { block.call(results) }
+          offset += batch_size
+        end
+        nil
+      end
+
+      def safe_find_each
+        safe_find_in_batches do |records|
+          records.each { |record| yield record }
+        end
+      end
+    end
+  end
+end

data/lib/earth/pet.rb

@@ -1,3 +1,3 @@
 # lib/earth/pet.rb
 require 'earth'
-Earth.require_related __FILE__
+Earth::Loader.require_related __FILE__

data/lib/earth/pet/breed.rb

@@ -1,13 +1,28 @@
+require 'earth/model'
+
+require 'earth/pet/breed_gender'
+require 'earth/pet/species'
+
 class Breed < ActiveRecord::Base
+  extend Earth::Model
+
+  TABLE_STRUCTURE = <<-EOS
+
+CREATE TABLE breeds
+  (
+     name         CHARACTER VARYING(255) NOT NULL PRIMARY KEY,
+     species_name CHARACTER VARYING(255),
+     weight       FLOAT,
+     weight_units CHARACTER VARYING(255)
+  );
+
+EOS
+
   self.primary_key = "name"
   
   has_many :breed_genders, :foreign_key => 'breed_name'
   belongs_to :species, :foreign_key => 'species_name'
   
-  col :name
-  col :species_name
-  col :weight, :type => :float
-  col :weight_units
 
   warn_if_nulls_except(
     :weight,

data/lib/earth/pet/breed_gender.rb

@@ -1,14 +1,29 @@
+require 'earth/model'
+
+require 'earth/pet/breed'
+require 'earth/pet/gender'
+
 class BreedGender < ActiveRecord::Base
+  extend Earth::Model
+
+  TABLE_STRUCTURE = <<-EOS
+
+CREATE TABLE breed_genders
+  (
+     name         CHARACTER VARYING(255) NOT NULL PRIMARY KEY,
+     breed_name   CHARACTER VARYING(255),
+     gender_name  CHARACTER VARYING(255),
+     weight       FLOAT,
+     weight_units CHARACTER VARYING(255)
+  );
+
+EOS
+
   self.primary_key = "name"
   
   belongs_to :breed, :foreign_key => 'breed_name'
   belongs_to :gender, :foreign_key => 'gender_name'
   
-  col :name
-  col :breed_name
-  col :gender_name
-  col :weight, :type => :float
-  col :weight_units
 
   warn_unless_size 586
 end

data/lib/earth/pet/gender.rb

@@ -1,9 +1,23 @@
+require 'earth/model'
+
+require 'earth/pet/breed_gender'
+
 class Gender < ActiveRecord::Base
+  extend Earth::Model
+
+  TABLE_STRUCTURE = <<-EOS
+
+CREATE TABLE genders
+  (
+     name CHARACTER VARYING(255) NOT NULL PRIMARY KEY
+  );
+
+EOS
+
   self.primary_key = "name"
   
   has_many :breed_genders, :foreign_key => 'gender_name'
   
-  col :name
 
   warn_unless_size 2
 end

data/lib/earth/pet/species.rb

@@ -1,20 +1,36 @@
+require 'falls_back_on'
+
+require 'earth/model'
+
+require 'earth/pet/breed'
+
 class Species < ActiveRecord::Base
+  extend Earth::Model
+
+  TABLE_STRUCTURE = <<-EOS
+
+CREATE TABLE species
+  (
+     name                               CHARACTER VARYING(255) NOT NULL PRIMARY KEY,
+     population                         INTEGER,
+     diet_emission_intensity            FLOAT,
+     diet_emission_intensity_units      CHARACTER VARYING(255),
+     weight                             FLOAT,
+     weight_units                       CHARACTER VARYING(255),
+     marginal_dietary_requirement       FLOAT,
+     marginal_dietary_requirement_units CHARACTER VARYING(255),
+     fixed_dietary_requirement          FLOAT,
+     fixed_dietary_requirement_units    CHARACTER VARYING(255),
+     minimum_weight                     FLOAT,
+     minimum_weight_units               CHARACTER VARYING(255),
+     maximum_weight                     FLOAT,
+     maximum_weight_units               CHARACTER VARYING(255)
+  );
+
+EOS
+
   self.primary_key = "name"
   
-  col :name
-  col :population, :type => :integer
-  col :diet_emission_intensity, :type => :float
-  col :diet_emission_intensity_units
-  col :weight, :type => :float
-  col :weight_units
-  col :marginal_dietary_requirement, :type => :float
-  col :marginal_dietary_requirement_units
-  col :fixed_dietary_requirement, :type => :float
-  col :fixed_dietary_requirement_units
-  col :minimum_weight, :type => :float
-  col :minimum_weight_units
-  col :maximum_weight, :type => :float
-  col :maximum_weight_units
 
   has_many :breeds, :foreign_key => 'species_name'
   

data/lib/earth/rail.rb

@@ -1,3 +1,3 @@
 # lib/earth/rail.rb
 require 'earth'
-Earth.require_related __FILE__
+Earth::Loader.require_related __FILE__

data/lib/earth/rail/country_rail_class.rb

@@ -1,21 +1,32 @@
-require 'earth/locality'
+require 'earth/model'
+
 class CountryRailClass < ActiveRecord::Base
+  extend Earth::Model
+
+  TABLE_STRUCTURE = <<-EOS
+
+CREATE TABLE country_rail_classes
+  (
+     name                        CHARACTER VARYING(255) NOT NULL PRIMARY KEY,
+     country_iso_3166_code       CHARACTER VARYING(255),
+     rail_class_name             CHARACTER VARYING(255),
+     passengers                  FLOAT,
+     speed                       FLOAT,
+     speed_units                 CHARACTER VARYING(255),
+     trip_distance               FLOAT,
+     trip_distance_units         CHARACTER VARYING(255),
+     electricity_intensity       FLOAT,
+     electricity_intensity_units CHARACTER VARYING(255),
+     diesel_intensity            FLOAT,
+     diesel_intensity_units      CHARACTER VARYING(255),
+     co2_emission_factor         FLOAT,
+     co2_emission_factor_units   CHARACTER VARYING(255)
+  );
+
+EOS
+
   self.primary_key = "name"
   
-  col :name
-  col :country_iso_3166_code
-  col :rail_class_name
-  col :passengers, :type => :float
-  col :speed, :type => :float
-  col :speed_units
-  col :trip_distance, :type => :float
-  col :trip_distance_units
-  col :electricity_intensity, :type => :float
-  col :electricity_intensity_units
-  col :diesel_intensity, :type => :float
-  col :diesel_intensity_units
-  col :co2_emission_factor, :type => :float
-  col :co2_emission_factor_units
 
   warn_unless_size 4
 end

data/lib/earth/rail/country_rail_class/data_miner.rb

@@ -1,4 +1,3 @@
-require 'earth/locality/data_miner'
 CountryRailClass.class_eval do
   data_miner do
     import "a list of country-specific rail classes and their pre-calculated characteristics",

data/lib/earth/rail/country_rail_traction.rb

@@ -1,17 +1,27 @@
-require 'earth/fuel'
-require 'earth/locality'
+require 'earth/model'
+
 class CountryRailTraction < ActiveRecord::Base
+  extend Earth::Model
+
+  TABLE_STRUCTURE = <<-EOS
+
+CREATE TABLE country_rail_tractions
+  (
+     name                        CHARACTER VARYING(255) NOT NULL PRIMARY KEY,
+     country_iso_3166_code       CHARACTER VARYING(255),
+     rail_traction_name          CHARACTER VARYING(255),
+     electricity_intensity       FLOAT,
+     electricity_intensity_units CHARACTER VARYING(255),
+     diesel_intensity            FLOAT,
+     diesel_intensity_units      CHARACTER VARYING(255),
+     co2_emission_factor         FLOAT,
+     co2_emission_factor_units   CHARACTER VARYING(255)
+  );
+
+EOS
+
   self.primary_key = "name"
   
-  col :name
-  col :country_iso_3166_code
-  col :rail_traction_name
-  col :electricity_intensity, :type => :float
-  col :electricity_intensity_units
-  col :diesel_intensity, :type => :float
-  col :diesel_intensity_units
-  col :co2_emission_factor, :type => :float
-  col :co2_emission_factor_units
 
   warn_unless_size 50
 end

data/lib/earth/rail/country_rail_traction/data_miner.rb

@@ -1,5 +1,5 @@
-require 'earth/fuel/data_miner'
-require 'earth/locality/data_miner'
+require 'earth/rail/rail_fuel'
+
 CountryRailTraction.class_eval do
   data_miner do
     import "european rail traction data from the UIC",

data/lib/earth/rail/country_rail_traction_class.rb

@@ -1,18 +1,28 @@
-require 'earth/fuel'
-require 'earth/locality'
+require 'earth/model'
+
 class CountryRailTractionClass < ActiveRecord::Base
+  extend Earth::Model
+
+  TABLE_STRUCTURE = <<-EOS
+
+CREATE TABLE country_rail_traction_classes
+  (
+     name                        CHARACTER VARYING(255) NOT NULL PRIMARY KEY,
+     country_iso_3166_code       CHARACTER VARYING(255),
+     rail_traction_name          CHARACTER VARYING(255),
+     rail_class_name             CHARACTER VARYING(255),
+     electricity_intensity       FLOAT,
+     electricity_intensity_units CHARACTER VARYING(255),
+     diesel_intensity            FLOAT,
+     diesel_intensity_units      CHARACTER VARYING(255),
+     co2_emission_factor         FLOAT,
+     co2_emission_factor_units   CHARACTER VARYING(255)
+  );
+
+EOS
+
   self.primary_key = "name"
   
-  col :name
-  col :country_iso_3166_code
-  col :rail_traction_name
-  col :rail_class_name
-  col :electricity_intensity, :type => :float
-  col :electricity_intensity_units
-  col :diesel_intensity, :type => :float
-  col :diesel_intensity_units
-  col :co2_emission_factor, :type => :float
-  col :co2_emission_factor_units
 
   warn_unless_size 125
 end

data/lib/earth/rail/country_rail_traction_class/data_miner.rb

@@ -1,5 +1,5 @@
-require 'earth/fuel/data_miner'
-require 'earth/locality/data_miner'
+require 'earth/rail/rail_fuel'
+
 CountryRailTractionClass.class_eval do
   data_miner do
     import "european rail traction class data from the UIC",

data/lib/earth/rail/national_transit_database_company.rb

@@ -1,5 +1,24 @@
-require 'earth/locality'
+require 'earth/model'
+
+require 'earth/rail/national_transit_database_mode'
+require 'earth/rail/national_transit_database_record'
+
 class NationalTransitDatabaseCompany < ActiveRecord::Base
+  extend Earth::Model
+
+  TABLE_STRUCTURE = <<-EOS
+
+CREATE TABLE ntd_companies
+  (
+     id            CHARACTER VARYING(255) NOT NULL PRIMARY KEY,
+     name          CHARACTER VARYING(255),
+     acronym       CHARACTER VARYING(255),
+     zip_code_name CHARACTER VARYING(255),
+     duns_number   CHARACTER VARYING(255)
+  );
+
+EOS
+
   self.primary_key = "id"
   self.table_name = :ntd_companies
   
@@ -31,11 +50,6 @@ class NationalTransitDatabaseCompany < ActiveRecord::Base
     end
   end
   
-  col :id
-  col :name
-  col :acronym
-  col :zip_code_name
-  col :duns_number
 
   warn_if_nulls_except(
     :acronym,