data_miner 0.4.25 → 0.4.26

data/Rakefile

@@ -17,7 +17,7 @@ begin
     gem.add_dependency 'conversions', '>=1.4.4'
     gem.add_dependency 'blockenspiel', '>=0.3.2'
     gem.add_dependency 'log4r', '>=1.1.7'
-    gem.add_development_dependency 'errata', '>=0.2.1'
+    gem.add_dependency 'errata', '>=0.2.1'
     gem.add_development_dependency "loose_tight_dictionary", ">=0.0.5"
     gem.require_path = "lib"
     gem.files.include %w(lib/data_miner) unless gem.files.empty? # seems to fail once it's in the wild

data/VERSION

@@ -1 +1 @@
-0.4.25
+0.4.26

data/data_miner.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.4.25"
+  s.version = "0.4.26"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
@@ -59,7 +59,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<conversions>, [">= 1.4.4"])
       s.add_runtime_dependency(%q<blockenspiel>, [">= 0.3.2"])
       s.add_runtime_dependency(%q<log4r>, [">= 1.1.7"])
-      s.add_development_dependency(%q<errata>, [">= 0.2.1"])
+      s.add_runtime_dependency(%q<errata>, [">= 0.2.1"])
       s.add_development_dependency(%q<loose_tight_dictionary>, [">= 0.0.5"])
     else
       s.add_dependency(%q<remote_table>, [">= 0.2.20"])

data/lib/data_miner/import.rb

@@ -14,6 +14,11 @@ module DataMiner
       @configuration = configuration
       @position_in_run = position_in_run
       @description = description
+      
+      if table_options[:errata].is_a?(String)
+        table_options[:errata] = Errata.new :url => table_options[:errata], :responder => resource
+      end
+        
       if table_options[:table].present?
         DataMiner.log_or_raise "You should specify :table or :url, but not both" if table_options[:url].present?
         @table = table_options[:table]

data/lib/data_miner.rb

@@ -11,6 +11,7 @@ end if ActiveSupport::VERSION::MAJOR == 3
 require 'active_record'
 require 'blockenspiel'
 require 'conversions'
+require 'errata'
 require 'remote_table'
 require 'andand'
 require 'log4r'

data/test/data_miner_test.rb

@@ -1022,6 +1022,61 @@ class Aircraft < ActiveRecord::Base
   end
 end
 
+# note that this depends on stuff in Aircraft
+class AircraftDeux < ActiveRecord::Base
+  set_primary_key :icao_code
+  
+  # defined on the class because we defined the errata with a shorthand
+  class << self
+    def is_not_attributed_to_aerospatiale?(row)
+      not row['Manufacturer'] =~ /AEROSPATIALE/i
+    end
+    
+    def is_not_attributed_to_cessna?(row)
+      not row['Manufacturer'] =~ /CESSNA/i
+    end
+    
+    def is_not_attributed_to_learjet?(row)
+      not row['Manufacturer'] =~ /LEAR/i
+    end
+    
+    def is_not_attributed_to_dehavilland?(row)
+      not row['Manufacturer'] =~ /DE ?HAVILLAND/i
+    end
+    
+    def is_not_attributed_to_mcdonnell_douglas?(row)
+      not row['Manufacturer'] =~ /MCDONNELL DOUGLAS/i
+    end
+    
+    def is_not_a_dc_plane?(row)
+      not row['Model'] =~ /DC/i
+    end
+    
+    def is_a_crj_900?(row)
+      row['Designator'].downcase == 'crj9'
+    end
+  end
+  
+  data_miner do
+    # ('A'..'Z').each do |letter|
+    # Note: for the purposes of testing, only importing "D"
+    %w{ D }.each do |letter|
+      import("ICAO codes starting with letter #{letter} used by the FAA",
+              :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
+              :encoding => 'US-ASCII',
+              :errata => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
+              :row_xpath => '//table/tr[2]/td/table/tr',
+              :column_xpath => 'td') do
+        key 'icao_code', :field_name => 'Designator'
+        store 'bts_name', :matcher => Aircraft::BtsNameMatcher.new
+        store 'bts_aircraft_type_code', :matcher => Aircraft::BtsAircraftTypeCodeMatcher.new
+        store 'manufacturer_name', :field_name => 'Manufacturer'
+        store 'name', :field_name => 'Model'
+      end
+    end
+  end
+end
+
 # todo: have somebody properly organize these
 class DataMinerTest < Test::Unit::TestCase
   if ENV['ALL'] == 'true' or ENV['NEW'] == 'true'
@@ -1140,6 +1195,11 @@ class DataMinerTest < Test::Unit::TestCase
   end
   
   if ENV['ALL'] == 'true' or ENV['SLOW'] == 'true'
+    should "allow errata to be specified with a shorthand, assuming the responder is the resource class itself" do
+      AircraftDeux.run_data_miner!
+      assert AircraftDeux.exists? :icao_code => 'DC91', :bts_aircraft_type_code => '630'
+    end
+    
     should "mine aircraft" do
       Aircraft.run_data_miner!
       assert Aircraft.exists? :icao_code => 'DC91', :bts_aircraft_type_code => '630'

data/test/test_helper.rb

@@ -3,8 +3,6 @@ require 'test/unit'
 require 'shoulda'
 require 'ruby-debug'
 
-require 'errata'
-
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'data_miner'
@@ -17,7 +15,7 @@ ActiveRecord::Base.establish_connection(
 )
 
 ActiveSupport::Inflector.inflections do |inflect|
-  inflect.uncountable 'aircraft'
+  inflect.uncountable %w{ aircraft aircraft_deux }
 end
 
 class Test::Unit::TestCase
@@ -295,6 +293,26 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.integer  'data_miner_last_run_id'
   end
   execute 'ALTER TABLE aircraft ADD PRIMARY KEY (icao_code);'
+  
+  create_table 'aircraft_deux', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.string 'icao_code'
+    t.string   'manufacturer_name'
+    t.string   'name'
+    
+    t.string   "bts_name"
+    t.string "bts_aircraft_type_code"
+
+    # t.string   'brighter_planet_aircraft_class_code'
+    # t.float    'm3'
+    # t.float    'm2'
+    # t.float    'm1'
+    # t.float    'endpoint_fuel'
+    t.datetime 'updated_at'
+    t.datetime 'created_at'
+    t.integer  'data_miner_touch_count'
+    t.integer  'data_miner_last_run_id'
+  end
+  execute 'ALTER TABLE aircraft_deux ADD PRIMARY KEY (icao_code);'
 end
 
 DataMiner::Run.create_tables

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 4
-  - 25
-  version: 0.4.25
+  - 26
+  version: 0.4.26
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -128,7 +128,7 @@ dependencies:
         - 2
         - 1
         version: 0.2.1
-  type: :development
+  type: :runtime
   version_requirements: *id008
 - !ruby/object:Gem::Dependency 
   name: loose_tight_dictionary