earth 0.5.4 → 0.6.0

data/Gemfile

@@ -1,4 +1,5 @@
 gem 'data_miner', :path => ENV['LOCAL_DATA_MINER'] if ENV['LOCAL_DATA_MINER']
+gem 'mini_record', :path => ENV['LOCAL_MINI_RECORD'] if ENV['LOCAL_MINI_RECORD']
 gem 'force_schema', :path => ENV['LOCAL_FORCE_SCHEMA'] if ENV['LOCAL_FORCE_SCHEMA']
 
 source :rubygems

data/TODO

@@ -0,0 +1,2 @@
+* convert former/current verify blocks into table_warnings warn {...} blocks
+* convert tests to minitest

data/bin/earth_tester.rb

@@ -14,19 +14,72 @@ end
 require 'active_support/all'
 require 'active_record'
 
-ActiveRecord::Base.establish_connection(
-  'adapter' => 'mysql',
-  'database' => 'test_earth',
-  'username' => 'root',
-  'password' => 'password',
-  'encoding' => 'utf8' # very, very important
-)
+case ENV['EARTH_DB_ADAPTER']
+when 'mysql'
+  adapter = 'mysql'
+  database = 'test_earth'
+  username = 'root'
+  password = 'password'
+  
+  # system %{mysql -u #{username} -p#{password} -e "DROP DATABASE #{database}"}
+  # system %{mysql -u #{username} -p#{password} -e "CREATE DATABASE #{database}"}
+when 'sqlite'
+  adapter = 'sqlite3'
+  database = ':memory:'
+  username = nil
+  password = nil
+else
+  adapter = 'postgresql'
+  username = ENV['EARTH_POSTGRES_USERNAME'] || `whoami`.chomp
+  password = ENV['EARTH_POSTGRES_PASSWORD']
+  database = ENV['EARTH_POSTGRES_DATABASE'] || 'test_earth'
+
+  createdb_bin = ENV['EARTH_CREATEDB_BIN'] || 'createdb'
+  dropdb_bin = ENV['EARTH_DROPDB_BIN'] || 'dropdb'
+  # system %{#{dropdb_bin} #{database}}
+  # system %{#{createdb_bin} #{database}}
+end
+
+config = {
+  'encoding' => 'utf8',
+  'adapter' => adapter,
+  'database' => database,
+}
+config['username'] = username if username
+config['password'] = password if password
+
+ActiveRecord::Base.establish_connection config
 
 require 'earth'
 
-Earth.init ARGV[0], :load_data_miner => true, :apply_schemas => true
+domain = ARGV[0]
+
+Earth.init domain, :load_data_miner => true, :apply_schemas => true
 
 ActiveRecord::Base.logger = Logger.new $stderr
 ActiveRecord::Base.logger.level = Logger::INFO
 
-ARGV[1].camelcase.constantize.run_data_miner!
+def show_resource(resource)
+  resource_model = resource.constantize
+  if (warnings = resource_model.table_warnings).any?
+    $stderr.puts
+    $stderr.puts '#'*50
+    $stderr.puts "# #{resource}"
+    $stderr.puts '#'*50
+    $stderr.puts
+    warnings.each do |warning|
+      $stderr.puts "* #{warning}"
+    end
+  end
+end
+
+if (resource = ARGV[1].to_s.camelcase).present?
+  resource.constantize.run_data_miner!
+  show_resource resource
+else
+  DataMiner.run
+  Earth.search(domain).each do |resource|
+    show_resource resource
+  end
+end
+# ARGV[1].split(/[^a-z_]/i).each { |underscore| underscore.camelcase.constantize.run_data_miner! }

data/earth.gemspec

@@ -27,23 +27,26 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.3.7}
   
-  s.add_runtime_dependency 'data_miner'
+  s.add_runtime_dependency 'data_miner', '>=1.3'
   s.add_runtime_dependency 'to_regexp'
   s.add_runtime_dependency 'cohort_scope'
+  s.add_runtime_dependency 'table_warnings', '>=0.0.6'
   s.add_runtime_dependency 'remote_table', '>=1.2.3'
   s.add_runtime_dependency 'falls_back_on'
   s.add_runtime_dependency 'fixed_width-multibyte'
   s.add_runtime_dependency 'geokit-rails'
   s.add_runtime_dependency 'loose_tight_dictionary', '>=0.2.3'
-  s.add_runtime_dependency 'weighted_average'
-  s.add_runtime_dependency 'force_schema', '>=0.0.2'
+  s.add_runtime_dependency 'weighted_average', '>=1'
+  # s.add_runtime_dependency 'mini_record' # need https://github.com/DAddYE/mini_record/pull/7 - for now install manually from https://github.com/seamusabshere/mini_record
+  s.add_runtime_dependency 'activesupport'
+  s.add_runtime_dependency 'activerecord'
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'bueller'
   s.add_development_dependency 'cucumber'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'rdoc'
-  s.add_development_dependency 'rdoc'
   s.add_development_dependency 'rspec'
   s.add_development_dependency 'sqlite3-ruby'
   s.add_development_dependency 'mysql' # for bin/earth_tester.rb
+  s.add_development_dependency 'pg'
 end

data/lib/earth.rb

@@ -1,3 +1,4 @@
+require 'active_support/core_ext'
 require 'active_record'
 require 'cohort_scope'
 require 'earth/conversions_ext'
@@ -7,22 +8,15 @@ require 'falls_back_on'
 require 'weighted_average'
 require 'fixed_width'
 require 'errata'
-require 'force_schema'
+require 'mini_record'
+require 'table_warnings'
 require 'loose_tight_dictionary'
 require 'loose_tight_dictionary/cached_result'
 
-# hackety hack
-def INSERT_IGNORE(cmd)
-  if ActiveRecord::Base.connection.adapter_name.downcase == 'sqlite'
-    prefix = 'INSERT'
-  else
-    prefix = 'INSERT IGNORE'
-  end
-  ActiveRecord::Base.connection.execute "#{prefix} #{cmd}"
-end
-
 # The earth module is an interface for loading data models from various domains.
 module Earth
+  autoload :Utils, 'earth/utils'
+  
   extend self
 
   # Takes argument like Earth.search(['air'])
@@ -78,6 +72,7 @@ module Earth
     options = args.last.is_a?(Hash) ? args.pop.symbolize_keys : {}
     domains = args.empty? ? [ :all ] : args.map(&:to_sym)
 
+    _configure_database_connection
     _load_plugins
     _load_domains domains, options
     _decorate_resources options
@@ -86,6 +81,12 @@ module Earth
 
   private
   
+  def _configure_database_connection
+    if ActiveRecord::Base.connection.adapter_name =~ /mysql/i
+      ActiveRecord::Base.connection.execute("SET SQL_MODE = 'PIPES_AS_CONCAT'")    
+    end
+  end
+  
   # TODO sabshere don't use directories to specify domains
   # * you have 20 million data_miner.rb files which are easy to confuse
   # * you have to go all over the filesystem to figure things out
@@ -119,13 +120,13 @@ module Earth
     resource_model.data_miner_config.steps.push pull_dependencies_step
   end
   
-  def _prepend_force_schema_step_to_data_miner(resource)
+  def _prepend_auto_upgrade_step_to_data_miner(resource)
     resource_model = resource.constantize
-    return if resource_model.data_miner_config.steps.any? { |step| step.description == :force_schema! }
+    return if resource_model.data_miner_config.steps.any? { |step| step.description == :auto_upgrade! }
 
-    force_schema_step = DataMiner::Process.new resource_model.data_miner_config, :force_schema!
+    auto_upgrade_step = DataMiner::Process.new resource_model.data_miner_config, :auto_upgrade!
 
-    resource_model.data_miner_config.steps.unshift force_schema_step
+    resource_model.data_miner_config.steps.unshift auto_upgrade_step
   end
   
   TAPS_STEP = 'Tap the Brighter Planet data server'
@@ -144,7 +145,7 @@ module Earth
       next unless ::Object.const_defined?(resource)
       _append_pull_dependencies_step_to_data_miner resource
       if options[:apply_schemas] or options[:load_data_miner]
-        _prepend_force_schema_step_to_data_miner resource
+        _prepend_auto_upgrade_step_to_data_miner resource
       else
         _prepend_taps_step_to_data_miner resource
       end
@@ -154,7 +155,7 @@ module Earth
   def _load_schemas(selected_resources, options)
     return unless options[:apply_schemas]
     selected_resources.each do |resource|
-      resource.constantize.force_schema!
+      resource.constantize.auto_upgrade!
     end
   end
 end

data/lib/earth/air.rb

@@ -1,6 +1,8 @@
 require 'earth/air/aircraft'
 require 'earth/air/aircraft_class'
 require 'earth/air/aircraft_fuel_use_equation'
+require 'earth/air/aircraft_instance'
+require 'earth/air/aircraft_instance_seat_class'
 require 'earth/air/airline'
 require 'earth/air/airport'
 require 'earth/air/bts_aircraft'

data/lib/earth/air/aircraft.rb

@@ -4,8 +4,24 @@ class Aircraft < ActiveRecord::Base
   belongs_to :aircraft_class,    :foreign_key => 'class_code',    :primary_key => 'code'
   belongs_to :fuel_use_equation, :foreign_key => 'fuel_use_code', :primary_key => 'code', :class_name => 'AircraftFuelUseEquation'
   
-  # set up a loose_tight_dictionary for matching Aircraft description with FlightSegment aircraft_description
+  col :icao_code
+  col :manufacturer_name
+  col :model_name
+  col :description
+  col :aircraft_type
+  col :engine_type
+  col :engines, :type => :integer
+  col :weight_class
+  col :class_code
+  col :fuel_use_code
+  col :seats, :type => :float
+  col :passengers, :type => :float
+  
+  # Enable aircraft.flight_segments
+  cache_loose_tight_dictionary_matches_with :flight_segments, :primary_key => :description, :foreign_key => :aircraft_description
+
   class << self
+    # set up a loose_tight_dictionary for matching Aircraft description with FlightSegment aircraft_description
     def loose_tight_dictionary
       @loose_tight_dictionary ||= LooseTightDictionary.new(Aircraft.all,
           :haystack_reader => lambda { |record| record.description },
@@ -15,23 +31,64 @@ class Aircraft < ActiveRecord::Base
           :must_match_blocking => true,
           :first_blocking_decides => true)
     end
-  end
   
-  force_schema do
-    string  'icao_code'
-    string  'manufacturer_name'
-    string  'model_name'
-    string  'description'
-    string  'aircraft_type'
-    string  'engine_type'
-    integer 'engines'
-    string  'weight_class'
-    string  'class_code'
-    string  'fuel_use_code'
-    float   'seats'
-    float   'passengers'
-  end
+    # FIXME TODO do we want to restrict this to certain years?
+    # Derive some average characteristics from flight segments
+    def update_averages!
+      manually_cache_flight_segments!
+      find_each do |aircraft|
+        aircraft.seats = aircraft.flight_segments.weighted_average :seats_per_flight, :weighted_by => :passengers
+        aircraft.passengers = aircraft.flight_segments.sum :passengers
+        aircraft.save!
+      end
+    end
   
-  # Enable aircraft.flight_segments
-  cache_loose_tight_dictionary_matches_with :flight_segments, :primary_key => :description, :foreign_key => :aircraft_description
-end
+    # Cache fuzzy matches between FlightSegment aircraft_description and Aircraft description
+    def manually_cache_flight_segments!
+      FlightSegment.run_data_miner!
+      LooseTightDictionary::CachedResult.setup
+      connection.select_values("SELECT DISTINCT(aircraft_description) FROM flight_segments WHERE aircraft_description IS NOT NULL").each do |original_description|
+        # If the flight segment's aircraft_description contains '/' then it describes multiple aircraft.
+        # We need to synthesize descriptions for those aircraft, find all Aircraft that fuzzily match the
+        # synthesized descriptions, and associate those Aircraft with the original aircraft_description.
+        # e.g. boeing 747-100/200
+        if original_description.include?('/')
+          # Pull out the complete first aircraft description
+          # e.g. 'boeing 747-100'
+          first_description = original_description.split('/')[0]
+        
+          # Pull out the root of the description - the text up to and including the last ' ' or '-'
+          # e.g. 'boeing 747-'
+          root_length = first_description.rindex(/[ \-]/)
+          root = first_description.slice(0..root_length)
+        
+          # Pull out the suffixes - the text separated by forward slashes
+          # e.g. ['100', '200']
+          suffixes = original_description.split(root)[1].split('/')
+        
+          # Create an array of synthesized descriptions by appending each suffix to the root
+          # e.g. ['boeing 747-100', 'boeing 747-200']
+          suffixes.map{ |suffix| root + suffix }.each do |synthesized_description|
+            # Look up the Aircraft that match each synthesized description and associate
+            # them with the original flight segment aircraft_description
+            Aircraft.loose_tight_dictionary.find_all(synthesized_description).each do |aircraft|
+              attrs = {
+                :a_class => "Aircraft",
+                :a => aircraft.description,
+                :b_class => "FlightSegment",
+                :b => original_description
+              }
+              unless ::LooseTightDictionary::CachedResult.exists? attrs
+                ::LooseTightDictionary::CachedResult.create! attrs
+              end
+            end
+          end
+        # If the flight segment's aircraft_description doesn't contain '/' we can use
+        # a method provided by loose_tight_dictionary to associate it with Aircraft
+        else
+          FlightSegment.find_by_aircraft_description(original_description).cache_aircraft!
+        end
+      end
+    end
+  end
+end

data/lib/earth/air/aircraft/data_miner.rb

@@ -21,68 +21,6 @@ Aircraft.class_eval do
     @manufacturer_whitelist.any? { |manufacturer_regexp| manufacturer_regexp.match candidate }
   end
   
-  # FIXME TODO do we want to restrict this to certain years?
-  # Derive some average characteristics from flight segments
-  def self.update_averages!
-    FlightSegment.run_data_miner!
-    manually_cache_flight_segments!
-    find_each do |aircraft|
-      aircraft.seats = aircraft.flight_segments.weighted_average :seats_per_flight, :weighted_by => :passengers
-      aircraft.passengers = aircraft.flight_segments.sum :passengers
-      aircraft.save
-    end
-  end
-  
-  # Cache fuzzy matches between FlightSegment aircraft_description and Aircraft description
-  def self.manually_cache_flight_segments!
-    FlightSegment.run_data_miner!
-    LooseTightDictionary::CachedResult.setup
-    FlightSegment.find_by_sql("SELECT * FROM flight_segments GROUP BY aircraft_description HAVING aircraft_description IS NOT NULL").each do |flight_segment|
-      original_description = flight_segment.aircraft_description
-      
-      # If the flight segment's aircraft_description contains '/' then it describes multiple aircraft.
-      # We need to synthesize descriptions for those aircraft, find all Aircraft that fuzzily match the
-      # synthesized descriptions, and associate those Aircraft with the original aircraft_description.
-      # e.g. boeing 747-100/200
-      if original_description.include?("/")
-        # Pull out the complete first aircraft description
-        # e.g. 'boeing 747-100'
-        first_description = original_description.split('/')[0]
-        
-        # Pull out the root of the description - the text up to and including the last ' ' or '-'
-        # e.g. 'boeing 747-'
-        root_length = first_description.rindex(/[ \-]/)
-        root = first_description.slice(0..root_length)
-        
-        # Pull out the suffixes - the text separated by forward slashes
-        # e.g. ['100', '200']
-        suffixes = original_description.split(root)[1].split('/')
-        
-        # Create an array of synthesized descriptions by appending each suffix to the root
-        # e.g. ['boeing 747-100', 'boeing 747-200']
-        suffixes.map{ |suffix| root + suffix }.each do |synthesized_description|
-          # Look up the Aircraft that match each synthesized description and associate
-          # them with the original flight segment aircraft_description
-          Aircraft.loose_tight_dictionary.find_all(synthesized_description).each do |aircraft|
-            attrs = {
-              :a_class => "Aircraft",
-              :a => aircraft.description,
-              :b_class => "FlightSegment",
-              :b => original_description
-            }
-            unless ::LooseTightDictionary::CachedResult.exists? attrs
-              ::LooseTightDictionary::CachedResult.create! attrs
-            end
-          end
-        end
-      # If the flight segment's aircraft_description doesn't contain '/' we can use
-      # a method provided by loose_tight_dictionary to associate it with Aircraft
-      else
-        flight_segment.cache_aircraft!
-      end
-    end
-  end
-
   data_miner do
     ('A'..'Z').each do |letter|
       import("aircraft made by whitelisted manufacturers whose ICAO code starts with '#{letter}' from the FAA",
@@ -140,7 +78,7 @@ Aircraft.class_eval do
     end
     
     process :update_averages!
-
+    
     # FIXME TODO verify this
   end
 end

data/lib/earth/air/aircraft_class.rb

@@ -2,6 +2,17 @@ class AircraftClass < ActiveRecord::Base
   set_primary_key :code
   
   has_many :aircraft, :foreign_key => 'class_code', :primary_key => 'code'
+
+  col :code
+  col :m3, :type => :float
+  col :m3_units
+  col :m2, :type => :float
+  col :m2_units
+  col :m1, :type => :float
+  col :m1_units
+  col :b, :type => :float
+  col :b_units
+  col :seats, :type => :float
   
   def fuel_use_coefficients
     [m3, m2, m1, b]
@@ -10,17 +21,42 @@ class AircraftClass < ActiveRecord::Base
   def valid_fuel_use_equation?
     fuel_use_coefficients.all?(&:present?) and fuel_use_coefficients.any?(&:nonzero?)
   end
-
-  force_schema do
-    string 'code'
-    float  'm3'
-    string 'm3_units'
-    float  'm2'
-    string 'm2_units'
-    float  'm1'
-    string 'm1_units'
-    float  'b'
-    string 'b_units'
-    float  'seats'
+  
+  class << self
+    def update_averages!
+      Aircraft.run_data_miner!
+      AircraftFuelUseEquation.run_data_miner!
+      find_each do |aircraft_class|
+        cumulative_passengers = 0
+        aircraft_class.m3 = 0
+        aircraft_class.m2 = 0
+        aircraft_class.m1 = 0
+        aircraft_class.b = 0
+      
+        aircraft_class.aircraft.where('passengers > 0 AND fuel_use_code IS NOT NULL').each do |a|
+          cumulative_passengers += a.passengers
+          aircraft_class.m3 += a.fuel_use_equation.m3 * a.passengers
+          aircraft_class.m2 += a.fuel_use_equation.m2 * a.passengers
+          aircraft_class.m1 += a.fuel_use_equation.m1 * a.passengers
+          aircraft_class.b += a.fuel_use_equation.b * a.passengers
+        end
+      
+        if cumulative_passengers > 0
+          aircraft_class.m3 /= cumulative_passengers
+          aircraft_class.m2 /= cumulative_passengers
+          aircraft_class.m1 /= cumulative_passengers
+          aircraft_class.b /= cumulative_passengers
+        end
+      
+        aircraft_class.seats = aircraft_class.aircraft.weighted_average(:seats, :weighted_by => :passengers)
+      
+        aircraft_class.m3_units = 'kilograms_per_cubic_nautical_mile'
+        aircraft_class.m2_units = 'kilograms_per_square_nautical_mile'
+        aircraft_class.m1_units = 'kilograms_per_nautical_mile'
+        aircraft_class.b_units  = 'kilograms'
+      
+        aircraft_class.save!
+      end
+    end
   end
 end