earth 0.12.4 → 1.0.0

data/lib/earth/shipping/carrier_mode.rb

@@ -1,16 +1,31 @@
+require 'earth/model'
+
+require 'earth/shipping/carrier'
+require 'earth/shipping/shipment_mode'
+
 class CarrierMode < ActiveRecord::Base
+  extend Earth::Model
+
+  TABLE_STRUCTURE = <<-EOS
+
+CREATE TABLE carrier_modes
+  (
+     name                            CHARACTER VARYING(255) NOT NULL PRIMARY KEY,
+     carrier_name                    CHARACTER VARYING(255),
+     mode_name                       CHARACTER VARYING(255),
+     package_volume                  FLOAT,
+     route_inefficiency_factor       FLOAT,
+     transport_emission_factor       FLOAT,
+     transport_emission_factor_units CHARACTER VARYING(255)
+  );
+
+EOS
+
   self.primary_key = "name"
   
   belongs_to :carrier, :foreign_key => 'carrier_name', :primary_key => 'name'
   belongs_to :mode,    :foreign_key => 'mode_name',    :primary_key => 'name', :class_name => 'ShipmentMode'
 
-  col :name
-  col :carrier_name
-  col :mode_name
-  col :package_volume, :type => :float
-  col :route_inefficiency_factor, :type => :float
-  col :transport_emission_factor, :type => :float
-  col :transport_emission_factor_units
   
   # Don't need to check that carrier_name appears in carriers b/c carriers is derived from carrier_modes.carrier_name
   # Don't need to check that mode_name appears in shipment_modes b/c shipment_modes is derived from carrier_modes.mode_name

data/lib/earth/shipping/shipment_mode.rb

@@ -1,12 +1,26 @@
+require 'earth/model'
+
+require 'earth/shipping/carrier_mode'
+
 class ShipmentMode < ActiveRecord::Base
+  extend Earth::Model
+
+  TABLE_STRUCTURE = <<-EOS
+
+CREATE TABLE shipment_modes
+  (
+     name                            CHARACTER VARYING(255) NOT NULL PRIMARY KEY,
+     route_inefficiency_factor       FLOAT,
+     transport_emission_factor       FLOAT,
+     transport_emission_factor_units CHARACTER VARYING(255)
+  );
+
+EOS
+
   self.primary_key = "name"
   
   has_many :carrier_modes, :foreign_key => 'mode_name', :primary_key => 'name'
   
-  col :name
-  col :route_inefficiency_factor, :type => :float
-  col :transport_emission_factor, :type => :float
-  col :transport_emission_factor_units
   
   # verify "Route inefficiency factor should be one or more" do
   #   ShipmentMode.all.each do |mode|

data/lib/earth/tasks.rb

@@ -0,0 +1,62 @@
+require 'earth'
+require 'rake'
+require 'active_record/connection_adapters/abstract/connection_specification'
+
+module Earth
+  class Tasks
+    include Rake::DSL
+
+    def initialize
+      init_earth_tasks
+      init_bare unless Object.const_defined?('Rails')
+
+      namespace :db do
+        task :create => 'earth:db:create'
+        task :drop => 'earth:db:drop'
+        task :migrate => 'earth:db:migrate'
+        task :seed => 'earth:db:seed'
+      end
+    end
+
+    def init_bare
+      Object.const_set 'Rails', Earth
+
+      require 'active_record'
+      load 'active_record/railties/databases.rake'
+
+      Rake::Task['db:load_config'].clear
+      Rake::Task['db:create'].clear
+      Rake::Task['db:drop'].clear
+      Rake::Task['db:migrate'].clear
+      Rake::Task['db:seed'].clear
+    end
+
+    def config
+      spec = ENV['DATABASE_URL']
+      resolver = ActiveRecord::Base::ConnectionSpecification::Resolver.new spec, {}
+      resolver.spec.config.stringify_keys
+    end
+        
+    def init_earth_tasks
+      namespace :earth do
+        namespace :db do
+          task :create do
+            create_database(config)
+          end
+          task :drop do
+            drop_database_and_rescue(config)
+          end
+          task :load_config do
+            Earth.connect
+          end
+          task :migrate => :load_config do
+            Earth.reset_schemas!
+          end
+          task :seed => :load_config do
+            Earth.run_data_miner!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/earth/version.rb

@@ -1,3 +1,3 @@
 module Earth
-  VERSION = "0.12.4"
+  VERSION = "1.0.0"
 end

data/lib/earth/warnings.rb

@@ -0,0 +1,12 @@
+module Earth
+  module Warnings
+    def Warnings.check_mysql_ansi_mode
+      if ::ActiveRecord::Base.connection.adapter_name =~ /mysql/i
+        sql_mode = ::ActiveRecord::Base.connection.select_value("SELECT @@GLOBAL.sql_mode") + ::ActiveRecord::Base.connection.select_value("SELECT @@SESSION.sql_mode")
+        unless sql_mode.downcase.include? 'pipes_as_concat'
+          ::Kernel.warn "[earth gem] Warning: MySQL detected, but PIPES_AS_CONCAT not set. Importing from scratch will fail. Consider setting sql-mode=ANSI in my.cnf."
+        end
+      end
+    end
+  end
+end

data/spec/data_mining_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+require 'fileutils'
+require 'earth'
+
+describe 'Data Mining', :data_miner => true do
+  it 'is successful for each individual model' do
+    Earth.init :all
+    Earth.resource_models.each do |resource|
+      begin
+        puts resource.to_s
+        script_file = File.join(Dir.pwd, 'miner.rb')
+        File.open script_file, 'w' do |f|
+          f.puts <<-RUBY
+require 'earth'
+require 'conversions'
+DataMiner.unit_converter = :conversions
+Earth.init :mine_original_sources => true, :connect => true
+require #{File.join(File.dirname(resource.source_file), File.basename(resource.source_file,'.rb')).inspect}
+#{resource.to_s}.run_data_miner!
+          RUBY
+        end
+
+        `bundle exec ruby #{script_file}`
+        #$?.to_i.should == 0
+      ensure
+        FileUtils.rm_f script_file
+      end
+    end
+  end
+end
+

data/spec/earth/air/aircraft_spec.rb

@@ -2,16 +2,6 @@ require 'spec_helper'
 require 'earth/air/aircraft'
 
 describe Aircraft do
-  describe 'import', :data_miner => true do
-    before do
-      Earth.init :air, :load_data_miner => true, :skip_parent_associations => :true
-    end
-    
-    it 'should import data' do
-      Aircraft.run_data_miner!
-    end
-  end
-  
   describe "verify imported data", :sanity => true do
     it "should have all the data" do
       Aircraft.count.should == 437

data/spec/earth/air/airline_spec.rb

@@ -2,19 +2,9 @@ require 'spec_helper'
 require 'earth/air/airline'
 
 describe Airline do
-  before :all do
-    Earth.init :air, :load_data_miner => true, :skip_parent_associations => :true
-  end
-  
-  describe 'import', :data_miner => true do
-    it 'should import data' do
-      Airline.run_data_miner!
-    end
-  end
-  
   describe "verify imported data", :sanity => true do
     it "should have all the data" do
-      Airline.count.should == 1523
+      Airline.count.should be_within(10).of(1523)
     end
   end
 end

data/spec/earth/air/airport_spec.rb

@@ -4,16 +4,6 @@ require 'spec_helper'
 require 'earth/air/airport'
 
 describe Airport do
-  describe "when importing data", :data_miner => true do
-    before do
-      Earth.init :air, :load_data_miner => true, :skip_parent_associations => :true
-    end
-    
-    it "imports all airports" do
-      Airport.run_data_miner!
-    end
-  end
-  
   describe "verify imported data", :sanity => true do
     it "should have all the data" do
       Airport.count.should == 5324

data/spec/earth/air/bts_aircraft_spec.rb

@@ -2,16 +2,6 @@ require 'spec_helper'
 require 'earth/air/bts_aircraft'
 
 describe BtsAircraft do
-  describe 'import', :data_miner => true do
-    before do
-      Earth.init :air, :load_data_miner => true, :skip_parent_associations => :true
-    end
-    
-    it 'should import data' do
-      BtsAircraft.run_data_miner!
-    end
-  end
-  
   describe 'verify imported data', :sanity => true do
     it 'should have all the data' do
       BtsAircraft.count.should == 379

data/spec/earth/air/flight_distance_class_spec.rb

@@ -2,16 +2,6 @@ require 'spec_helper'
 require 'earth/air/flight_distance_class'
 
 describe FlightDistanceClass do
-  describe "when importing data", :data_miner => true do
-    before do
-      Earth.init :air, :load_data_miner => true, :skip_parent_associations => :true
-    end
-    
-    it "imports all naics codes" do
-      FlightDistanceClass.run_data_miner!
-    end
-  end
-  
   describe "verify imported data", :sanity => true do
     it "should have all the data" do
       FlightDistanceClass.count.should == 2

data/spec/earth/air/flight_segment_spec.rb

@@ -1,17 +1,9 @@
 require 'spec_helper'
 require 'earth/air/flight_segment'
+require 'factories/flight_segment'
+require 'factories/airline'
 
 describe FlightSegment do
-  describe 'import', :data_miner => true do
-    before do
-      Earth.init :air, :load_data_miner => true, :skip_parent_associations => :true
-    end
-    
-    it 'should import data' do
-      FlightSegment.run_data_miner!
-    end
-  end
-  
   describe "verify imported data", :sanity => true, :flight_segment => true do
     it "should have all the data" do
       FlightSegment.where(:year => 2009).count.should == 403_980
@@ -62,9 +54,34 @@ describe FlightSegment do
         Airline.exists?(:name => name).should == true
       end
     end
+
+    it 'should relate to airline' do
+      FlightSegment.any? { |fs| !fs.airline.nil? }.should be_true
+    end
     
     it "should have aircraft description" do
       FlightSegment.where(:aircraft_description => nil).count.should == 0
     end
   end
+
+  describe '#airline' do
+    it 'finds an airline by BTS code' do
+      FactoryGirl.create :airline, :delta
+      FactoryGirl.create :airline, :united
+
+      fs = FactoryGirl.build :flight_segment, :delta
+      fs.airline.bts_code.should == 'DL'
+    end
+    it 'finds an airline by ICAO code' do
+      FactoryGirl.create :airline, :delta
+      FactoryGirl.create :airline, :united
+
+      fs = FactoryGirl.build :flight_segment, :delta_icao
+      fs.airline.bts_code.should == 'DL'
+    end
+    it 'returns nil if there is no airline' do
+      fs = FactoryGirl.build :flight_segment
+      fs.airline.should be_nil
+    end
+  end
 end

data/spec/earth/automobile/automobile_activity_year_spec.rb

@@ -3,16 +3,9 @@ require 'earth/automobile/automobile_activity_year'
 
 describe AutomobileActivityYear do
   before :all do
-    Earth.init :automobile, :load_data_miner => true
     require 'earth/acronyms'
   end
   
-  describe 'import', :data_miner => true do
-    it 'should import data' do
-      AAY.run_data_miner!
-    end
-  end
-  
   describe 'verify', :sanity => true do
     it { AAY.count.should == 15 }
     it { AAY.where("hfc_emission_factor > 0").count.should == AAY.count }

data/spec/earth/automobile/automobile_activity_year_type_fuel_spec.rb

@@ -3,16 +3,9 @@ require 'earth/automobile/automobile_activity_year_type_fuel'
 
 describe AutomobileActivityYearTypeFuel do
   before :all do
-    Earth.init :automobile, :load_data_miner => true
     require 'earth/acronyms'
   end
   
-  describe 'import', :data_miner => true do
-    it 'should import data' do
-      AAYTF.run_data_miner!
-    end
-  end
-  
   describe 'verify', :sanity => true do
     it { AAYTF.count.should == 120 }
     it { AAYTF.where(:distance => nil).count.should == 0 }

data/spec/earth/automobile/automobile_activity_year_type_spec.rb

@@ -3,16 +3,9 @@ require 'earth/automobile/automobile_activity_year_type'
 
 describe AutomobileActivityYearType do
   before :all do
-    Earth.init :automobile, :load_data_miner => true
     require 'earth/acronyms'
   end
   
-  describe 'import', :data_miner => true do
-    it 'should import data' do
-      AAYT.run_data_miner!
-    end
-  end
-  
   describe 'verify', :sanity => true do
     it { AAYT.count.should == 30 }
     it { AAYT.where("hfc_emissions > 0").count.should == AAYT.count }

data/spec/earth/automobile/automobile_fuel_spec.rb

@@ -4,16 +4,6 @@ require 'earth/automobile'
 describe AutomobileFuel do
   let(:fallback) { AutomobileFuel.fallback }
   
-  describe 'import', :data_miner => true do
-    before do
-      Earth.init :automobile, :load_data_miner => true
-    end
-    
-    it 'should import data without problems' do
-      AutomobileFuel.run_data_miner!
-    end
-  end
-  
   describe 'verify imported data', :sanity => true do
     it { AutomobileFuel.count.should == 12 }
     it { AutomobileFuel.where(:distance_key => nil).count.should == 0 }

data/spec/earth/automobile/automobile_make_model_spec.rb

@@ -12,17 +12,9 @@ describe AutomobileMakeModel do
   let(:f150_ffv_years) { AMMY.where(:make_name => 'Ford', :model_name => 'F150 FFV') }
   
   before :all do
-    Earth.init :automobile, :load_data_miner => true, :skip_parent_associations => true
     require 'earth/acronyms'
   end
   
-  describe 'import', :data_miner => true do
-    it 'should import data' do
-      AMM.run_data_miner!
-      AMM.count.should == AMM.connection.select_value("SELECT COUNT(DISTINCT make_name, model_name) FROM #{AMMY.quoted_table_name}")
-    end
-  end
-  
   describe 'verify', :sanity => true do
     it { AMM.where(:fuel_code => nil).count.should == 0 }
     it { AMM.where("fuel_efficiency_city > 0").count.should == AMM.count }

data/spec/earth/automobile/automobile_make_model_year_spec.rb

@@ -3,17 +3,9 @@ require 'earth/automobile/automobile_make_model_year'
 
 describe AutomobileMakeModelYear do
   before :all do
-    Earth.init :automobile, :load_data_miner => true, :skip_parent_associations => :true
     require 'earth/acronyms'
   end
   
-  describe 'import', :data_miner => true do
-    it 'should import data' do
-      AMMY.run_data_miner!
-      AMMY.count.should == AMMY.connection.select_value("SELECT COUNT(DISTINCT make_name, model_name, year) FROM #{AMMYV.quoted_table_name}")
-    end
-  end
-  
   describe 'verify imported data', :sanity => true do
     it 'has data from all years' do
       (1985..2012).each do |year|

data/spec/earth/automobile/automobile_make_model_year_variant_spec.rb

@@ -4,17 +4,9 @@ require 'earth/automobile/automobile_make_model_year_variant'
 
 describe AutomobileMakeModelYearVariant do
   before :all do
-    Earth.init :automobile, :load_data_miner => true
     require 'earth/acronyms'
   end
   
-  describe 'import', :data_miner => true do
-    it 'should import data' do
-      AMMYV.run_data_miner!
-      AMMYV.count.should == 28433
-    end
-  end
-  
   describe 'verify imported data', :sanity => true do
     # confirm we have right data from all years
     it { AMMYV.where(:year => 1985).count.should == 1152 }