data_miner 0.4.24 → 0.4.25

data/Rakefile

@@ -10,15 +10,15 @@ begin
     gem.email = "seamus@abshere.net"
     gem.homepage = "http://github.com/seamusabshere/data_miner"
     gem.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-    gem.add_dependency 'remote_table', '>=0.2.19'
+    gem.add_dependency 'remote_table', '>=0.2.20'
     gem.add_dependency 'activerecord', '>=2.3.4'
     gem.add_dependency 'activesupport', '>=2.3.4'
     gem.add_dependency 'andand', '>=1.3.1'
-    gem.add_dependency 'errata', '>=0.1.7'
     gem.add_dependency 'conversions', '>=1.4.4'
     gem.add_dependency 'blockenspiel', '>=0.3.2'
     gem.add_dependency 'log4r', '>=1.1.7'
-    gem.add_development_dependency "loose_tight_dictionary", ">=0.0.3"
+    gem.add_development_dependency 'errata', '>=0.2.1'
+    gem.add_development_dependency "loose_tight_dictionary", ">=0.0.5"
     gem.require_path = "lib"
     gem.files.include %w(lib/data_miner) unless gem.files.empty? # seems to fail once it's in the wild
     gem.rdoc_options << '--line-numbers' << '--inline-source'

data/VERSION

@@ -1 +1 @@
-0.4.24
+0.4.25

data/data_miner.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.4.24"
+  s.version = "0.4.25"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-  s.date = %q{2010-04-29}
+  s.date = %q{2010-05-06}
   s.description = %q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
   s.email = %q{seamus@abshere.net}
   s.extra_rdoc_files = [
@@ -52,36 +52,36 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<remote_table>, [">= 0.2.19"])
+      s.add_runtime_dependency(%q<remote_table>, [">= 0.2.20"])
       s.add_runtime_dependency(%q<activerecord>, [">= 2.3.4"])
       s.add_runtime_dependency(%q<activesupport>, [">= 2.3.4"])
       s.add_runtime_dependency(%q<andand>, [">= 1.3.1"])
-      s.add_runtime_dependency(%q<errata>, [">= 0.1.7"])
       s.add_runtime_dependency(%q<conversions>, [">= 1.4.4"])
       s.add_runtime_dependency(%q<blockenspiel>, [">= 0.3.2"])
       s.add_runtime_dependency(%q<log4r>, [">= 1.1.7"])
-      s.add_development_dependency(%q<loose_tight_dictionary>, [">= 0.0.3"])
+      s.add_development_dependency(%q<errata>, [">= 0.2.1"])
+      s.add_development_dependency(%q<loose_tight_dictionary>, [">= 0.0.5"])
     else
-      s.add_dependency(%q<remote_table>, [">= 0.2.19"])
+      s.add_dependency(%q<remote_table>, [">= 0.2.20"])
       s.add_dependency(%q<activerecord>, [">= 2.3.4"])
       s.add_dependency(%q<activesupport>, [">= 2.3.4"])
       s.add_dependency(%q<andand>, [">= 1.3.1"])
-      s.add_dependency(%q<errata>, [">= 0.1.7"])
       s.add_dependency(%q<conversions>, [">= 1.4.4"])
       s.add_dependency(%q<blockenspiel>, [">= 0.3.2"])
       s.add_dependency(%q<log4r>, [">= 1.1.7"])
-      s.add_dependency(%q<loose_tight_dictionary>, [">= 0.0.3"])
+      s.add_dependency(%q<errata>, [">= 0.2.1"])
+      s.add_dependency(%q<loose_tight_dictionary>, [">= 0.0.5"])
     end
   else
-    s.add_dependency(%q<remote_table>, [">= 0.2.19"])
+    s.add_dependency(%q<remote_table>, [">= 0.2.20"])
     s.add_dependency(%q<activerecord>, [">= 2.3.4"])
     s.add_dependency(%q<activesupport>, [">= 2.3.4"])
     s.add_dependency(%q<andand>, [">= 1.3.1"])
-    s.add_dependency(%q<errata>, [">= 0.1.7"])
     s.add_dependency(%q<conversions>, [">= 1.4.4"])
     s.add_dependency(%q<blockenspiel>, [">= 0.3.2"])
     s.add_dependency(%q<log4r>, [">= 1.1.7"])
-    s.add_dependency(%q<loose_tight_dictionary>, [">= 0.0.3"])
+    s.add_dependency(%q<errata>, [">= 0.2.1"])
+    s.add_dependency(%q<loose_tight_dictionary>, [">= 0.0.5"])
   end
 end
 

data/lib/data_miner.rb

@@ -12,7 +12,6 @@ require 'active_record'
 require 'blockenspiel'
 require 'conversions'
 require 'remote_table'
-require 'errata'
 require 'andand'
 require 'log4r'
 require 'fileutils'

data/lib/data_miner/attribute.rb

@@ -74,7 +74,7 @@ module DataMiner
     end
     
     def match_row(row)
-      matcher.lookup row
+      matcher.match row
     end
     
     def value_from_row(row)
@@ -219,7 +219,7 @@ module DataMiner
       @_dictionary ||= Dictionary.new options[:dictionary]
     end
     def matcher
-      @_matcher ||= options[:matcher].new
+      @_matcher ||= (options[:matcher].is_a?(String) ? options[:matcher].constantize.new : options[:matcher])
     end
   end
 end

data/lib/data_miner/import.rb

@@ -3,20 +3,23 @@ module DataMiner
     include Blockenspiel::DSL
     
     attr_reader :attributes
-    attr_accessor :configuration, :position_in_run, :options, :table, :errata
+    attr_accessor :configuration, :position_in_run, :table
     attr_accessor :description
     delegate :resource, :to => :configuration
     
-    def initialize(configuration, position_in_run, description, options = {})
-      options.symbolize_keys!
-      @options = options
+    def initialize(configuration, position_in_run, description, table_options = {})
+      table_options.symbolize_keys!
 
       @attributes = ActiveSupport::OrderedHash.new
       @configuration = configuration
       @position_in_run = position_in_run
       @description = description
-      @errata = Errata.new(:url => options[:errata], :klass => resource) if options[:errata]
-      @table = RemoteTable.new options
+      if table_options[:table].present?
+        DataMiner.log_or_raise "You should specify :table or :url, but not both" if table_options[:url].present?
+        @table = table_options[:table]
+      else
+        @table = RemoteTable.new table_options
+      end
     end
 
     def inspect
@@ -46,11 +49,6 @@ module DataMiner
       test_counter = 0
 
       table.each_row do |row|
-        if errata
-          next if errata.rejects?(row)
-          errata.correct!(row)
-        end
-        
         if ENV['DUMP'] == 'true'
           raise "[data_miner gem] Stopping after 5 rows because TEST=true" if test_counter > 5
           test_counter += 1

data/test/data_miner_test.rb

@@ -220,6 +220,70 @@ class AutomobileVariant < ActiveRecord::Base
     end
   end
   
+  class Guru
+    # the following matching methods are needed by the errata
+    # per https://brighterplanet.sifterapp.com/projects/30/issues/750/comments
+
+    def transmission_is_blank?(row)
+      row['transmission'].blank?
+    end
+
+    def is_a_2007_gmc_or_chevrolet?(row)
+      row['year'] == 2007 and %w(GMC CHEVROLET).include? row['MFR'].upcase
+    end
+
+    def is_a_porsche?(row)
+      row['make'].upcase == 'PORSCHE'
+    end
+
+    def is_not_a_porsche?(row)
+      !is_a_porsche? row
+    end
+
+    def is_a_mercedes_benz?(row)
+      row['make'] =~ /MERCEDES/i
+    end
+
+    def is_a_lexus?(row)
+      row['make'].upcase == 'LEXUS'
+    end
+
+    def is_a_bmw?(row)
+      row['make'].upcase == 'BMW'
+    end
+
+    def is_a_ford?(row)
+      row['make'].upcase == 'FORD'
+    end
+
+    def is_a_rolls_royce_and_model_contains_bentley?(row)
+      is_a_rolls_royce?(row) and model_contains_bentley?(row)
+    end
+
+    def is_a_bentley?(row)
+      row['make'].upcase == 'BENTLEY'
+    end
+
+    def is_a_rolls_royce?(row)
+      row['make'] =~ /ROLLS/i
+    end
+
+    def is_a_turbo_brooklands?(row)
+      row['model'] =~ /TURBO R\/RL BKLDS/i
+    end
+
+    def model_contains_maybach?(row)
+      row['model'] =~ /MAYBACH/i
+    end
+    
+    def model_contains_bentley?(row)
+      row['model'] =~ /BENTLEY/i
+    end
+  end
+  
+  errata = Errata.new :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv',
+                      :responder => AutomobileVariant::Guru.new
+  
   data_miner do
     # 1985---1997
     (85..97).each do |yy|
@@ -227,7 +291,7 @@ class AutomobileVariant < ActiveRecord::Base
       import(:url => "http://www.fueleconomy.gov/FEG/epadata/#{yy}mfgui.zip",
              :filename => filename,
              :transform => { :class => FuelEconomyGuide::ParserB, :year => "19#{yy}".to_i },
-             :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
+             :errata => errata) do
         key 'row_hash'
         store 'make_name', :field_name => 'make'
         store 'model_name', :field_name => 'model'
@@ -264,7 +328,7 @@ class AutomobileVariant < ActiveRecord::Base
       2005 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/05data.zip', :filename => 'guide2005-2004oct15.csv' }
     }.sort { |a, b| a.first <=> b.first }.each do |year, options|
       import options.merge(:transform => { :class => FuelEconomyGuide::ParserC, :year => year },
-                           :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
+                           :errata => errata) do
         key 'row_hash'
         store 'make_name', :field_name => 'make'
         store 'model_name', :field_name => 'model'
@@ -296,7 +360,7 @@ class AutomobileVariant < ActiveRecord::Base
       # 2010 => { :url => 'http://www.fueleconomy.gov/FEG/epadata/10data.zip', :filename => '2010FEguide-for DOE-rel dates before 10-16-09-no-sales10-8-09public.xls' }
     }.sort { |a, b| a.first <=> b.first }.each do |year, options|
       import options.merge(:transform => { :class => FuelEconomyGuide::ParserD, :year => year },
-                           :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/fuel_economy_guide/errata.csv') do
+                           :errata => errata) do
         key 'row_hash'
         store 'make_name', :field_name => 'make'
         store 'model_name', :field_name => 'model'
@@ -344,67 +408,6 @@ class AutomobileVariant < ActiveRecord::Base
   def fuel_economy_description
     [ fuel_efficiency_city, fuel_efficiency_highway ].map { |f| f.kilometres_per_litre.to(:miles_per_gallon).round }.join('/')
   end
-  
-  class << self
-    # the following matching methods are needed by the errata
-    # per https://brighterplanet.sifterapp.com/projects/30/issues/750/comments
-
-    def transmission_is_blank?(row)
-      row['transmission'].blank?
-    end
-
-    def is_a_2007_gmc_or_chevrolet?(row)
-      row['year'] == 2007 and %w(GMC CHEVROLET).include? row['MFR'].upcase
-    end
-
-    def is_a_porsche?(row)
-      row['make'].upcase == 'PORSCHE'
-    end
-
-    def is_not_a_porsche?(row)
-      !is_a_porsche? row
-    end
-
-    def is_a_mercedes_benz?(row)
-      row['make'] =~ /MERCEDES/i
-    end
-
-    def is_a_lexus?(row)
-      row['make'].upcase == 'LEXUS'
-    end
-
-    def is_a_bmw?(row)
-      row['make'].upcase == 'BMW'
-    end
-
-    def is_a_ford?(row)
-      row['make'].upcase == 'FORD'
-    end
-
-    def is_a_rolls_royce_and_model_contains_bentley?(row)
-      is_a_rolls_royce?(row) and model_contains_bentley?(row)
-    end
-
-    def is_a_bentley?(row)
-      row['make'].upcase == 'BENTLEY'
-    end
-
-    def is_a_rolls_royce?(row)
-      row['make'] =~ /ROLLS/i
-    end
-
-    def is_a_turbo_brooklands?(row)
-      row['model'] =~ /TURBO R\/RL BKLDS/i
-    end
-
-    def model_contains_maybach?(row)
-      row['model'] =~ /MAYBACH/i
-    end
-    
-    def model_contains_bentley?(row)
-      row['model'] =~ /BENTLEY/i
-    end
-  end
 end
 
 class Country < ActiveRecord::Base
@@ -954,20 +957,20 @@ class Aircraft < ActiveRecord::Base
   end
 
   class BtsAircraftTypeCodeMatcher
-    def lookup(left_record)
+    def match(left_record)
       right_record = Aircraft.bts_dictionary.left_to_right left_record
       right_record['Aircraft Type'] if right_record
     end
   end
   
   class BtsNameMatcher
-    def lookup(left_record)
+    def match(left_record)
       right_record = Aircraft.bts_dictionary.left_to_right left_record
       right_record['Manufacturer'] + ' ' + right_record['Long Name'] if right_record
     end
   end
   
-  class << self
+  class Guru
     # for errata
     def is_not_attributed_to_aerospatiale?(row)
       not row['Manufacturer'] =~ /AEROSPATIALE/i
@@ -1005,12 +1008,13 @@ class Aircraft < ActiveRecord::Base
       import("ICAO codes starting with letter #{letter} used by the FAA",
               :url => "http://www.faa.gov/air_traffic/publications/atpubs/CNT/5-2-#{letter}.htm",
               :encoding => 'US-ASCII',
+              :errata => Errata.new(:url => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
+                                    :responder => Aircraft::Guru.new),
               :row_xpath => '//table/tr[2]/td/table/tr',
-              :errata => 'http://spreadsheets.google.com/pub?key=tObVAGyqOkCBtGid0tJUZrw',
               :column_xpath => 'td') do
         key 'icao_code', :field_name => 'Designator'
-        store 'bts_name', :matcher => Aircraft::BtsNameMatcher
-        store 'bts_aircraft_type_code', :matcher => Aircraft::BtsAircraftTypeCodeMatcher
+        store 'bts_name', :matcher => Aircraft::BtsNameMatcher.new
+        store 'bts_aircraft_type_code', :matcher => Aircraft::BtsAircraftTypeCodeMatcher.new
         store 'manufacturer_name', :field_name => 'Manufacturer'
         store 'name', :field_name => 'Model'
       end
@@ -1020,14 +1024,10 @@ end
 
 # todo: have somebody properly organize these
 class DataMinerTest < Test::Unit::TestCase
-  if ENV['NEW'] == 'true'
-    should "mine aircraft" do
-      Aircraft.run_data_miner!
-      assert Aircraft.exists? :icao_code => 'DC91', :bts_aircraft_type_code => '630'
-    end
+  if ENV['ALL'] == 'true' or ENV['NEW'] == 'true'
   end
     
-  if ENV['FAST'] == 'true'
+  if ENV['ALL'] == 'true' or ENV['FAST'] == 'true'
     should "keep a call stack so that you can call run_data_miner! on a child" do
       CrosscallingCensusDivision.run_data_miner!
       assert CrosscallingCensusDivision.exists? :name => 'Mountain Division', :number => 8, :census_region_number => 4, :census_region_name => 'West Region'
@@ -1139,7 +1139,12 @@ class DataMinerTest < Test::Unit::TestCase
     end
   end
   
-  if ENV['SLOW'] == 'true'
+  if ENV['ALL'] == 'true' or ENV['SLOW'] == 'true'
+    should "mine aircraft" do
+      Aircraft.run_data_miner!
+      assert Aircraft.exists? :icao_code => 'DC91', :bts_aircraft_type_code => '630'
+    end
+    
     should "mine automobile variants" do
       AutomobileVariant.run_data_miner!
       assert AutomobileVariant.count('make_name LIKE "%tesla"') > 0
@@ -1150,7 +1155,7 @@ class DataMinerTest < Test::Unit::TestCase
       assert T100FlightSegment.count('dest_country_name LIKE "%United States"') > 0
     end
     
-    should "mine residence survey day" do
+    should "mine residence survey responses" do
       ResidentialEnergyConsumptionSurveyResponse.run_data_miner!
       assert ResidentialEnergyConsumptionSurveyResponse.find(6).residence_class.starts_with?('Single-family detached house')
     end

data/test/test_helper.rb

@@ -3,6 +3,8 @@ require 'test/unit'
 require 'shoulda'
 require 'ruby-debug'
 
+require 'errata'
+
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'data_miner'
@@ -11,7 +13,7 @@ ActiveRecord::Base.establish_connection(
   'adapter' => 'mysql',
   'database' => 'data_miner_test',
   'username' => 'root',
-  'password' => ''
+  'password' => 'password'
 )
 
 ActiveSupport::Inflector.inflections do |inflect|

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 4
-  - 24
-  version: 0.4.24
+  - 25
+  version: 0.4.25
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-29 00:00:00 -04:00
+date: 2010-05-06 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -28,8 +28,8 @@ dependencies:
         segments: 
         - 0
         - 2
-        - 19
-        version: 0.2.19
+        - 20
+        version: 0.2.20
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -74,24 +74,10 @@ dependencies:
         version: 1.3.1
   type: :runtime
   version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: errata
-  prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 1
-        - 7
-        version: 0.1.7
-  type: :runtime
-  version_requirements: *id005
 - !ruby/object:Gem::Dependency 
   name: conversions
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: &id005 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
@@ -101,11 +87,11 @@ dependencies:
         - 4
         version: 1.4.4
   type: :runtime
-  version_requirements: *id006
+  version_requirements: *id005
 - !ruby/object:Gem::Dependency 
   name: blockenspiel
   prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
+  requirement: &id006 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
@@ -115,11 +101,11 @@ dependencies:
         - 2
         version: 0.3.2
   type: :runtime
-  version_requirements: *id007
+  version_requirements: *id006
 - !ruby/object:Gem::Dependency 
   name: log4r
   prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
+  requirement: &id007 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
@@ -129,6 +115,20 @@ dependencies:
         - 7
         version: 1.1.7
   type: :runtime
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: errata
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 2
+        - 1
+        version: 0.2.1
+  type: :development
   version_requirements: *id008
 - !ruby/object:Gem::Dependency 
   name: loose_tight_dictionary
@@ -140,8 +140,8 @@ dependencies:
         segments: 
         - 0
         - 0
-        - 3
-        version: 0.0.3
+        - 5
+        version: 0.0.5
   type: :development
   version_requirements: *id009
 description: Mine remote data into your ActiveRecord models. You can also perform associations and convert units.