data_miner 0.4.21 → 0.4.22

data/README.rdoc

@@ -81,6 +81,11 @@ Now you should have
   >> Airport.first.country_name
   => "Papua New Guinea"
 
+==Wishlist
+
+* each_record do |record| ... which would use find_in_batches
+* when proxying add_column, rename_column, etc. automatically include the table name
+
 ==Authors
 
 * Seamus Abshere <seamus@abshere.net>

data/Rakefile

@@ -10,14 +10,16 @@ begin
     gem.email = "seamus@abshere.net"
     gem.homepage = "http://github.com/seamusabshere/data_miner"
     gem.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-    gem.add_dependency 'remote_table', '>=0.2.10'
+    gem.add_dependency 'remote_table', '>=0.2.17'
     gem.add_dependency 'activerecord', '>=2.3.4'
     gem.add_dependency 'activesupport', '>=2.3.4'
     gem.add_dependency 'andand', '>=1.3.1'
     gem.add_dependency 'errata', '>=0.1.7'
     gem.add_dependency 'conversions', '>=1.4.4'
     gem.add_dependency 'blockenspiel', '>=0.3.2'
+    gem.add_dependency 'text', '>=0.2.0'
     gem.add_dependency 'log4r', '>=1.1.7'
+    gem.add_development_dependency "loose_tight_dictionary", ">=0.0.3"
     gem.require_path = "lib"
     gem.files.include %w(lib/data_miner) unless gem.files.empty? # seems to fail once it's in the wild
     gem.rdoc_options << '--line-numbers' << '--inline-source'

data/VERSION

@@ -1 +1 @@
-0.4.21
+0.4.22

data/data_miner.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.4.21"
+  s.version = "0.4.22"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-  s.date = %q{2010-04-16}
+  s.date = %q{2010-04-28}
   s.description = %q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
   s.email = %q{seamus@abshere.net}
   s.extra_rdoc_files = [
@@ -52,33 +52,39 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<remote_table>, [">= 0.2.10"])
+      s.add_runtime_dependency(%q<remote_table>, [">= 0.2.17"])
       s.add_runtime_dependency(%q<activerecord>, [">= 2.3.4"])
       s.add_runtime_dependency(%q<activesupport>, [">= 2.3.4"])
       s.add_runtime_dependency(%q<andand>, [">= 1.3.1"])
       s.add_runtime_dependency(%q<errata>, [">= 0.1.7"])
       s.add_runtime_dependency(%q<conversions>, [">= 1.4.4"])
       s.add_runtime_dependency(%q<blockenspiel>, [">= 0.3.2"])
+      s.add_runtime_dependency(%q<text>, [">= 0.2.0"])
       s.add_runtime_dependency(%q<log4r>, [">= 1.1.7"])
+      s.add_development_dependency(%q<loose_tight_dictionary>, [">= 0.0.3"])
     else
-      s.add_dependency(%q<remote_table>, [">= 0.2.10"])
+      s.add_dependency(%q<remote_table>, [">= 0.2.17"])
       s.add_dependency(%q<activerecord>, [">= 2.3.4"])
       s.add_dependency(%q<activesupport>, [">= 2.3.4"])
       s.add_dependency(%q<andand>, [">= 1.3.1"])
       s.add_dependency(%q<errata>, [">= 0.1.7"])
       s.add_dependency(%q<conversions>, [">= 1.4.4"])
       s.add_dependency(%q<blockenspiel>, [">= 0.3.2"])
+      s.add_dependency(%q<text>, [">= 0.2.0"])
       s.add_dependency(%q<log4r>, [">= 1.1.7"])
+      s.add_dependency(%q<loose_tight_dictionary>, [">= 0.0.3"])
     end
   else
-    s.add_dependency(%q<remote_table>, [">= 0.2.10"])
+    s.add_dependency(%q<remote_table>, [">= 0.2.17"])
     s.add_dependency(%q<activerecord>, [">= 2.3.4"])
     s.add_dependency(%q<activesupport>, [">= 2.3.4"])
     s.add_dependency(%q<andand>, [">= 1.3.1"])
     s.add_dependency(%q<errata>, [">= 0.1.7"])
     s.add_dependency(%q<conversions>, [">= 1.4.4"])
     s.add_dependency(%q<blockenspiel>, [">= 0.3.2"])
+    s.add_dependency(%q<text>, [">= 0.2.0"])
     s.add_dependency(%q<log4r>, [">= 1.1.7"])
+    s.add_dependency(%q<loose_tight_dictionary>, [">= 0.0.3"])
   end
 end
 

data/lib/data_miner.rb

@@ -2,6 +2,8 @@ require 'active_support'
 require 'active_support/version'
 %w{
   active_support/core_ext/array/conversions
+  active_support/core_ext/string/access
+  active_support/core_ext/string/multibyte
 }.each do |active_support_3_requirement|
   require active_support_3_requirement
 end if ActiveSupport::VERSION::MAJOR == 3
@@ -15,6 +17,7 @@ require 'andand'
 require 'log4r'
 require 'fileutils'
 require 'tmpdir'
+require 'amatch'
 
 require 'data_miner/attribute'
 require 'data_miner/configuration'

data/lib/data_miner/attribute.rb

@@ -11,6 +11,7 @@ module DataMiner
       :to_units,
       :static,
       :dictionary,
+      :matcher,
       :field_name,
       :delimiter,
       :split,
@@ -72,7 +73,12 @@ module DataMiner
       value
     end
     
+    def match_row(row)
+      matcher.lookup row
+    end
+    
     def value_from_row(row)
+      return match_row row if wants_matcher?
       value = value_in_source row
       return value if value.is_a? ActiveRecord::Base # carry through trapdoor
       value = value_in_dictionary value if wants_dictionary?
@@ -159,6 +165,9 @@ module DataMiner
     def wants_dictionary?
       options[:dictionary].present?
     end
+    def wants_matcher?
+      options[:matcher].present?
+    end
 
     # Options that always have values
     def field_name
@@ -209,5 +218,8 @@ module DataMiner
     def dictionary
       @_dictionary ||= Dictionary.new options[:dictionary]
     end
+    def matcher
+      @_matcher ||= options[:matcher].new
+    end
   end
 end

data/lib/data_miner/import.rb

@@ -16,7 +16,7 @@ module DataMiner
       @position_in_run = position_in_run
       @description = description
       @errata = Errata.new(:url => options[:errata], :klass => resource) if options[:errata]
-      @table = RemoteTable.new(options.slice(:url, :filename, :form_data, :format, :skip, :cut, :schema, :schema_name, :trap, :select, :reject, :sheet, :delimiter, :headers, :transform, :crop, :encoding, :compression, :glob))
+      @table = RemoteTable.new options
     end
 
     def inspect

data/test/data_miner_test.rb

@@ -940,9 +940,91 @@ class T100FlightSegment < ActiveRecord::Base
   end
 end
 
+require 'loose_tight_dictionary'
+class Aircraft < ActiveRecord::Base
+  set_primary_key :icao_code
+
+  def self.bts_dictionary
+    @_dictionary ||= LooseTightDictionary.new RemoteTable.new(:url => 'http://www.bts.gov/programs/airline_information/accounting_and_reporting_directives/csv/number_260.csv', :select => lambda { |record| record['Aircraft Type'].to_i.between?(1, 998) and record['Manufacturer'].present? }),
+                                              :tightenings  => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false),
+                                              :identities   => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=3&output=csv', :headers => false),
+                                              :blockings    => RemoteTable.new(:url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=4&output=csv', :headers => false),
+                                              :left_reader  => lambda { |record| record['Manufacturer'] + ' ' + record['Model'] },
+                                              :right_reader => lambda { |record| record['Manufacturer'] + ' ' + record['Long Name'] }
+  end
+
+  class BtsAircraftTypeCodeMatcher
+    def lookup(left_record)
+      right_record = Aircraft.bts_dictionary.left_to_right left_record
+      right_record['Aircraft Type'] if right_record
+    end
+  end
+  
+  class BtsNameMatcher
+    def lookup(left_record)
+      right_record = Aircraft.bts_dictionary.left_to_right left_record
+      right_record['Manufacturer'] + ' ' + right_record['Long Name'] if right_record
+    end
+  end
+  
+  class << self
+    # for errata
+    def is_not_attributed_to_aerospatiale?(row)
+      not row['Manufacturer'] =~ /AEROSPATIALE/i
+    end
+    
+    def is_not_attributed_to_cessna?(row)
+      not row['Manufacturer'] =~ /CESSNA/i
+    end
+    
+    def is_not_attributed_to_learjet?(row)
+      not row['Manufacturer'] =~ /LEAR/i
+    end
+    
+    def is_not_attributed_to_dehavilland?(row)
+      not row['Manufacturer'] =~ /DE ?HAVILLAND/i
+    end
+    
+    def is_not_attributed_to_mcdonnell_douglas?(row)
+      not row['Manufacturer'] =~ /MCDONNELL DOUGLAS/i
+    end
+    
+    def is_not_a_dc_plane?(row)
+      not row['Model'] =~ /DC/i
+    end
+    
+    def is_a_crj_900?(row)
+      row['Designator'].downcase == 'crj9'
+    end
+  end
+  
+  data_miner do
+    # ('A'..'Z').each do |letter|
+    # Note: for the purposes of testing, only importing "D"
+    %w{ D }.each do |letter|
+      import("ICAO codes starting with letter #{letter} used by the FAA",
+              :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
+              :encoding => 'US-ASCII',
+              :row_xpath => '//table/tr[2]/td/table/tr',
+              :errata => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
+              :column_xpath => 'td') do
+        key 'icao_code', :field_name => 'Designator'
+        store 'bts_name', :matcher => Aircraft::BtsNameMatcher
+        store 'bts_aircraft_type_code', :matcher => Aircraft::BtsAircraftTypeCodeMatcher
+        store 'manufacturer_name', :field_name => 'Manufacturer'
+        store 'name', :field_name => 'Model'
+      end
+    end
+  end
+end
+
 # todo: have somebody properly organize these
 class DataMinerTest < Test::Unit::TestCase
   if ENV['NEW'] == 'true'
+    should "mine aircraft" do
+      Aircraft.run_data_miner!
+      assert Aircraft.exists? :icao_code => 'DC91', :bts_aircraft_type_code => '630'
+    end
   end
     
   if ENV['FAST'] == 'true'

data/test/test_helper.rb

@@ -14,6 +14,10 @@ ActiveRecord::Base.establish_connection(
   'password' => ''
 )
 
+ActiveSupport::Inflector.inflections do |inflect|
+  inflect.uncountable 'aircraft'
+end
+
 class Test::Unit::TestCase
 end
 
@@ -269,6 +273,26 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.integer 'data_miner_last_run_id'
   end
   execute "ALTER TABLE residential_energy_consumption_survey_responses ADD PRIMARY KEY (department_of_energy_identifier);"
+  
+  create_table 'aircraft', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.string 'icao_code'
+    t.string   'manufacturer_name'
+    t.string   'name'
+    
+    t.string   "bts_name"
+    t.string "bts_aircraft_type_code"
+
+    # t.string   'brighter_planet_aircraft_class_code'
+    # t.float    'm3'
+    # t.float    'm2'
+    # t.float    'm1'
+    # t.float    'endpoint_fuel'
+    t.datetime 'updated_at'
+    t.datetime 'created_at'
+    t.integer  'data_miner_touch_count'
+    t.integer  'data_miner_last_run_id'
+  end
+  execute 'ALTER TABLE aircraft ADD PRIMARY KEY (icao_code);'
 end
 
 DataMiner::Run.create_tables

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 4
-  - 21
-  version: 0.4.21
+  - 22
+  version: 0.4.22
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-16 00:00:00 -04:00
+date: 2010-04-28 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -28,8 +28,8 @@ dependencies:
         segments: 
         - 0
         - 2
-        - 10
-        version: 0.2.10
+        - 17
+        version: 0.2.17
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -117,9 +117,23 @@ dependencies:
   type: :runtime
   version_requirements: *id007
 - !ruby/object:Gem::Dependency 
-  name: log4r
+  name: text
   prerelease: false
   requirement: &id008 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 2
+        - 0
+        version: 0.2.0
+  type: :runtime
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: log4r
+  prerelease: false
+  requirement: &id009 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
@@ -129,7 +143,21 @@ dependencies:
         - 7
         version: 1.1.7
   type: :runtime
-  version_requirements: *id008
+  version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: loose_tight_dictionary
+  prerelease: false
+  requirement: &id010 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 0
+        - 3
+        version: 0.0.3
+  type: :development
+  version_requirements: *id010
 description: Mine remote data into your ActiveRecord models. You can also perform associations and convert units.
 email: seamus@abshere.net
 executables: []