earth 0.5.2 → 0.5.3

data/Gemfile

@@ -1,5 +1,5 @@
 gem 'data_miner', :path => ENV['LOCAL_DATA_MINER'] if ENV['LOCAL_DATA_MINER']
-gem 'create_table', :path => ENV['LOCAL_CREATE_TABLE'] if ENV['LOCAL_CREATE_TABLE']
+gem 'force_schema', :path => ENV['LOCAL_FORCE_SCHEMA'] if ENV['LOCAL_FORCE_SCHEMA']
 
 source :rubygems
 

data/LICENSE-PREAMBLE

@@ -9,7 +9,7 @@ IDENTITY OF THE SOFTWARE MODULE COVERED BY LICENSE:
 
 This license covers the Brighter Planet reference data import module,
 referred to as 'earth,' officially distributed from
-http://github.com/brighterplanet/earth.
+https://github.com/brighterplanet/earth.
 
 SCOPE OF THE SOFTWARE MODULE COVERED BY LICENSE:
 

data/Rakefile

@@ -24,6 +24,9 @@ Cucumber::Rake::Task.new(:features) do |t|
   end
 end
 
+directory 'log/'
+task :features => 'log/'
+
 if RUBY_VERSION =~ /^1\.8/
   desc "Run cucumber tests with RCov"
   Cucumber::Rake::Task.new(:features_with_coverage) do |t|

data/earth.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'geokit-rails'
   s.add_runtime_dependency 'loose_tight_dictionary', '>=0.2.3'
   s.add_runtime_dependency 'weighted_average'
-  s.add_runtime_dependency 'create_table', '>=0.0.2'
+  s.add_runtime_dependency 'force_schema', '>=0.0.2'
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'bueller'
   s.add_development_dependency 'cucumber'

data/lib/earth.rb

@@ -7,7 +7,7 @@ require 'falls_back_on'
 require 'weighted_average'
 require 'fixed_width'
 require 'errata'
-require 'create_table'
+require 'force_schema'
 require 'loose_tight_dictionary'
 require 'loose_tight_dictionary/cached_result'
 
@@ -119,13 +119,13 @@ module Earth
     resource_model.data_miner_config.steps.push pull_dependencies_step
   end
   
-  def _prepend_create_table_step_to_data_miner(resource)
+  def _prepend_force_schema_step_to_data_miner(resource)
     resource_model = resource.constantize
-    return if resource_model.data_miner_config.steps.any? { |step| step.description == :create_table! }
+    return if resource_model.data_miner_config.steps.any? { |step| step.description == :force_schema! }
 
-    create_table_step = DataMiner::Process.new resource_model.data_miner_config, :create_table!
+    force_schema_step = DataMiner::Process.new resource_model.data_miner_config, :force_schema!
 
-    resource_model.data_miner_config.steps.unshift create_table_step
+    resource_model.data_miner_config.steps.unshift force_schema_step
   end
   
   TAPS_STEP = 'Tap the Brighter Planet data server'
@@ -144,7 +144,7 @@ module Earth
       next unless ::Object.const_defined?(resource)
       _append_pull_dependencies_step_to_data_miner resource
       if options[:apply_schemas] or options[:load_data_miner]
-        _prepend_create_table_step_to_data_miner resource
+        _prepend_force_schema_step_to_data_miner resource
       else
         _prepend_taps_step_to_data_miner resource
       end
@@ -154,7 +154,7 @@ module Earth
   def _load_schemas(selected_resources, options)
     return unless options[:apply_schemas]
     selected_resources.each do |resource|
-      resource.constantize.create_table!
+      resource.constantize.force_schema!
     end
   end
 end

data/lib/earth/air/aircraft.rb

@@ -17,6 +17,21 @@ class Aircraft < ActiveRecord::Base
     end
   end
   
+  force_schema do
+    string  'icao_code'
+    string  'manufacturer_name'
+    string  'model_name'
+    string  'description'
+    string  'aircraft_type'
+    string  'engine_type'
+    integer 'engines'
+    string  'weight_class'
+    string  'class_code'
+    string  'fuel_use_code'
+    float   'seats'
+    float   'passengers'
+  end
+  
   # Enable aircraft.flight_segments
   cache_loose_tight_dictionary_matches_with :flight_segments, :primary_key => :description, :foreign_key => :aircraft_description
 end

data/lib/earth/air/aircraft/data_miner.rb

@@ -20,22 +20,69 @@ Aircraft.class_eval do
     @manufacturer_whitelist ||= RemoteTable.new(:url => 'https://spreadsheets.google.com/spreadsheet/pub?key=0AoQJbWqPrREqdFRFalpOdlg1cnF6amlSM1dDc1lya2c&output=csv').map { |record| record['Manufacturer'].to_regexp }
     @manufacturer_whitelist.any? { |manufacturer_regexp| manufacturer_regexp.match candidate }
   end
-
-  create_table do
-    string  'icao_code'
-    string  'manufacturer_name'
-    string  'model_name'
-    string  'description'
-    string  'aircraft_type'
-    string  'engine_type'
-    integer 'engines'
-    string  'weight_class'
-    string  'class_code'
-    string  'fuel_use_code'
-    float   'seats'
-    float   'passengers'
+  
+  # FIXME TODO do we want to restrict this to certain years?
+  # Derive some average characteristics from flight segments
+  def self.update_averages!
+    FlightSegment.run_data_miner!
+    manually_cache_flight_segments!
+    find_each do |aircraft|
+      aircraft.seats = aircraft.flight_segments.weighted_average :seats_per_flight, :weighted_by => :passengers
+      aircraft.passengers = aircraft.flight_segments.sum :passengers
+      aircraft.save
+    end
   end
   
+  # Cache fuzzy matches between FlightSegment aircraft_description and Aircraft description
+  def self.manually_cache_flight_segments!
+    FlightSegment.run_data_miner!
+    LooseTightDictionary::CachedResult.setup
+    FlightSegment.find_by_sql("SELECT * FROM flight_segments GROUP BY aircraft_description HAVING aircraft_description IS NOT NULL").each do |flight_segment|
+      original_description = flight_segment.aircraft_description
+      
+      # If the flight segment's aircraft_description contains '/' then it describes multiple aircraft.
+      # We need to synthesize descriptions for those aircraft, find all Aircraft that fuzzily match the
+      # synthesized descriptions, and associate those Aircraft with the original aircraft_description.
+      # e.g. boeing 747-100/200
+      if original_description.include?("/")
+        # Pull out the complete first aircraft description
+        # e.g. 'boeing 747-100'
+        first_description = original_description.split('/')[0]
+        
+        # Pull out the root of the description - the text up to and including the last ' ' or '-'
+        # e.g. 'boeing 747-'
+        root_length = first_description.rindex(/[ \-]/)
+        root = first_description.slice(0..root_length)
+        
+        # Pull out the suffixes - the text separated by forward slashes
+        # e.g. ['100', '200']
+        suffixes = original_description.split(root)[1].split('/')
+        
+        # Create an array of synthesized descriptions by appending each suffix to the root
+        # e.g. ['boeing 747-100', 'boeing 747-200']
+        suffixes.map{ |suffix| root + suffix }.each do |synthesized_description|
+          # Look up the Aircraft that match each synthesized description and associate
+          # them with the original flight segment aircraft_description
+          Aircraft.loose_tight_dictionary.find_all(synthesized_description).each do |aircraft|
+            attrs = {
+              :a_class => "Aircraft",
+              :a => aircraft.description,
+              :b_class => "FlightSegment",
+              :b => original_description
+            }
+            unless ::LooseTightDictionary::CachedResult.exists? attrs
+              ::LooseTightDictionary::CachedResult.create! attrs
+            end
+          end
+        end
+      # If the flight segment's aircraft_description doesn't contain '/' we can use
+      # a method provided by loose_tight_dictionary to associate it with Aircraft
+      else
+        flight_segment.cache_aircraft!
+      end
+    end
+  end
+
   data_miner do
     ('A'..'Z').each do |letter|
       import("aircraft made by whitelisted manufacturers whose ICAO code starts with '#{letter}' from the FAA",
@@ -73,13 +120,13 @@ Aircraft.class_eval do
     end
     
     process "Synthesize description from manufacturer name and model name" do
-      Aircraft.find_each do |aircraft|
+      find_each do |aircraft|
         aircraft.update_attribute :description, [aircraft.manufacturer_name, aircraft.model_name].join(' ').downcase
       end
     end
     
     process "Synthesize class code from engine type and weight class" do
-      Aircraft.find_each do |aircraft|
+      find_each do |aircraft|
         size = case aircraft.weight_class
         when 'Small', 'Small+', 'Light'
           'Light'
@@ -92,67 +139,8 @@ Aircraft.class_eval do
       end
     end
     
-    process "Ensure FlightSegment is populated" do
-      FlightSegment.run_data_miner!
-    end
-    
-    process "Cache fuzzy matches between FlightSegment aircraft_description and Aircraft description" do
-      LooseTightDictionary::CachedResult.setup
-      FlightSegment.find_by_sql("SELECT * FROM flight_segments GROUP BY aircraft_description HAVING aircraft_description IS NOT NULL").each do |flight_segment|
-        original_description = flight_segment.aircraft_description
-        
-        # If the flight segment's aircraft_description contains '/' then it describes multiple aircraft.
-        # We need to synthesize descriptions for those aircraft, find all Aircraft that fuzzily match the
-        # synthesized descriptions, and associate those Aircraft with the original aircraft_description.
-        # e.g. boeing 747-100/200
-        if original_description.include?("/")
-          # Pull out the complete first aircraft description
-          # e.g. 'boeing 747-100'
-          first_description = original_description.split('/')[0]
-          
-          # Pull out the root of the description - the text up to and including the last ' ' or '-'
-          # e.g. 'boeing 747-'
-          root_length = first_description.rindex(/[ \-]/)
-          root = first_description.slice(0..root_length)
-          
-          # Pull out the suffixes - the text separated by forward slashes
-          # e.g. ['100', '200']
-          suffixes = original_description.split(root)[1].split('/')
-          
-          # Create an array of synthesized descriptions by appending each suffix to the root
-          # e.g. ['boeing 747-100', 'boeing 747-200']
-          suffixes.map{ |suffix| root + suffix }.each do |synthesized_description|
-            # Look up the Aircraft that match each synthesized description and associate
-            # them with the original flight segment aircraft_description
-            Aircraft.loose_tight_dictionary.find_all(synthesized_description).each do |aircraft|
-              attrs = {
-                :a_class => "Aircraft",
-                :a => aircraft.description,
-                :b_class => "FlightSegment",
-                :b => original_description
-              }
-              unless ::LooseTightDictionary::CachedResult.exists? attrs
-                ::LooseTightDictionary::CachedResult.create! attrs
-              end
-            end
-          end
-        # If the flight segment's aircraft_description doesn't contain '/' we can use
-        # a method provided by loose_tight_dictionary to associate it with Aircraft
-        else
-          flight_segment.cache_aircraft!
-        end
-      end
-    end
-    
-    # FIXME TODO do we want to restrict this to certain years?
-    process "Derive some average characteristics from flight segments" do
-      Aircraft.find_each do |aircraft|
-        aircraft.seats = aircraft.flight_segments.weighted_average :seats_per_flight, :weighted_by => :passengers
-        aircraft.passengers = aircraft.flight_segments.sum :passengers
-        aircraft.save
-      end
-    end
-    
+    process :update_averages!
+
     # FIXME TODO verify this
   end
 end

data/lib/earth/air/aircraft_class.rb

@@ -11,7 +11,7 @@ class AircraftClass < ActiveRecord::Base
     fuel_use_coefficients.all?(&:present?) and fuel_use_coefficients.any?(&:nonzero?)
   end
 
-  create_table do
+  force_schema do
     string 'code'
     float  'm3'
     string 'm3_units'

data/lib/earth/air/aircraft_fuel_use_equation.rb

@@ -20,7 +20,7 @@ class AircraftFuelUseEquation < ActiveRecord::Base
     fuel_use_coefficients.all?(&:present?) and fuel_use_coefficients.any?(&:nonzero?)
   end
 
-  create_table do
+  force_schema do
     string 'code'
     string 'aircraft_description'
     float  'm3'

data/lib/earth/air/airline.rb

@@ -1,7 +1,7 @@
 class Airline < ActiveRecord::Base
   set_primary_key :name
 
-  create_table do
+  force_schema do
     string 'name'
     string 'bts_code'
     string 'iata_code'

data/lib/earth/air/airport.rb

@@ -7,7 +7,7 @@ class Airport < ActiveRecord::Base
                    :lat_column_name => :latitude,
                    :lng_column_name => :longitude
 
-  create_table do
+  force_schema do
     string   'iata_code'
     string   'name'
     string   'city'

data/lib/earth/air/bts_aircraft.rb

@@ -1,6 +1,6 @@
 class BtsAircraft < ActiveRecord::Base
   set_primary_key :bts_code
-  create_table do
+  force_schema do
     string 'bts_code'
     string 'description'
   end

data/lib/earth/air/flight_distance_class.rb

@@ -1,6 +1,6 @@
 class FlightDistanceClass < ActiveRecord::Base
   set_primary_key :name
-  create_table do
+  force_schema do
     string 'name'
     float  'distance'
     string 'distance_units'

data/lib/earth/air/flight_seat_class.rb

@@ -3,7 +3,7 @@ class FlightSeatClass < ActiveRecord::Base
   
   falls_back_on :multiplier => 1
 
-  create_table do
+  force_schema do
     string   'name'
     string   'distance_class_name'
     string   'seat_class_name'

data/lib/earth/air/flight_segment.rb

@@ -19,7 +19,7 @@ class FlightSegment < ActiveRecord::Base
                 :load_factor      => lambda { weighted_average(:load_factor,      :weighted_by => :passengers) }, # 0.78073233770097  data1 10-12-2010
                 :freight_share    => lambda { weighted_average(:freight_share,    :weighted_by => :passengers) }  # 0.022567224170157 data1 10-12-2010
   
-  create_table do
+  force_schema do
     string  'row_hash'                          # auto-generated primary key
     string  'origin_airport_iata_code'          # iata code
     string  'origin_airport_city'               # city

data/lib/earth/air/flight_segment/data_miner.rb

@@ -10,6 +10,24 @@ FlightSegment.class_eval do
     end
   end
   
+  def self.update_averages!
+    # Derive load factor, which is passengers divided by available seats
+    update_all 'load_factor = passengers / seats', 'seats > 0'
+
+    # Assume a load factor of 1 where passengers > available seats
+    update_all 'load_factor = 1', 'passengers > seats AND seats > 0'
+  
+    # TODO: what is 90.718474
+    # Derive freight share as a fraction of the total weight carried
+    update_all 'freight_share = (freight + mail) / (freight + mail + (passengers * 90.718474))', '(freight + mail + passengers) > 0'
+  
+    # Derive average seats per flight
+    update_all 'seats_per_flight = seats / flights', 'flights > 0'
+    
+    # Add a useful date field
+    update_all 'approximate_date = DATE(CONCAT_WS("-", year, month, "14"))', 'month IS NOT NULL'
+  end
+  
   URL = 'http://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=293&Has_Group=3&Is_Zipped=0'
   FORM_DATA = %{
     UserTableName=T_100_Segment__All_Carriers&
@@ -201,10 +219,10 @@ FlightSegment.class_eval do
         store 'flights',                           :field_name => 'DEPARTURES_PERFORMED'
         store 'passengers',                        :field_name => 'PASSENGERS'
         store 'seats',                             :field_name => 'SEATS'
-        store 'payload_capacity',                  :field_name => 'PAYLOAD',  :units => 'pounds'
-        store 'freight',                           :field_name => 'FREIGHT',  :units => 'pounds'
-        store 'mail',                              :field_name => 'MAIL',     :units => 'pounds'
-        store 'distance',                          :field_name => 'DISTANCE', :units => 'miles'
+        store 'payload_capacity',                  :field_name => 'PAYLOAD',  :from_units => :pounds, :to_units => :kilograms
+        store 'freight',                           :field_name => 'FREIGHT',  :from_units => :pounds, :to_units => :kilograms
+        store 'mail',                              :field_name => 'MAIL',     :from_units => :pounds, :to_units => :kilograms
+        store 'distance',                          :field_name => 'DISTANCE', :from_units => :miles, :to_units => :kilometres
         store 'month',                             :field_name => 'MONTH'
         store 'year',                              :field_name => 'YEAR'
         store 'source',                            :static => 'BTS T100'
@@ -232,49 +250,9 @@ FlightSegment.class_eval do
       end
     end
     
-    %w{ payload_capacity freight mail }.each do |field|
-      process "Convert #{field} from pounds to kilograms" do
-        conversion_factor = 1.pounds.to(:kilograms)
-        connection.execute %{
-          UPDATE flight_segments
-          SET #{field} = #{field} * #{conversion_factor},
-              #{field + '_units'} = 'kilograms'
-          WHERE #{field + '_units'} = 'pounds'
-        }
-      end
-    end
-    
-    process "Convert distance from miles to kilometres" do
-      conversion_factor = 1.miles.to(:kilometres)
-      connection.execute %{
-        UPDATE flight_segments
-        SET distance = distance * #{conversion_factor},
-            distance_units = 'kilometres'
-        WHERE distance_units = 'miles'
-      }
-    end
-    
-    process "Derive load factor, which is passengers divided by available seats" do
-      update_all 'load_factor = passengers / seats', 'seats > 0'
-    end
-    
-    process "Assume a load factor of 1 where passengers > available seats" do
-      update_all 'load_factor = 1', 'passengers > seats AND seats > 0'
-    end
-    
-    process "Derive freight share as a fraction of the total weight carried" do
-      update_all 'freight_share = (freight + mail) / (freight + mail + (passengers * 90.718474))', '(freight + mail + passengers) > 0'
-    end
-    
-    process "Derive average seats per flight" do
-      update_all 'seats_per_flight = seats / flights', 'flights > 0'
-    end
-    
-    process "Add a useful date field" do
-      update_all 'approximate_date = DATE(CONCAT_WS("-", year, month, "14"))', 'month IS NOT NULL'
-    end
+    process :update_averages!
     
-    process "Data mine Aircraft to cache fuzzy matches" do
+    process "Data mine Aircraft because it's like a belongs-to association" do
       Aircraft.run_data_miner!
     end
     

data/lib/earth/automobile/automobile_fuel.rb

@@ -98,7 +98,7 @@ class AutomobileFuel < ActiveRecord::Base
                 :hfc_emission_factor => lambda { AutomobileFuel.fallback_hfc_emission_factor },
                 :hfc_emission_factor_units => lambda { AutomobileFuel.fallback_hfc_emission_factor_units }
 
-  create_table do
+  force_schema do
     string 'name'
     string 'code'
     string 'base_fuel_name'