data_miner 0.4.29 → 0.4.30

data/VERSION

@@ -1 +1 @@
-0.4.29
+0.4.30

data/data_miner.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.4.29"
+  s.version = "0.4.30"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-  s.date = %q{2010-05-17}
+  s.date = %q{2010-05-18}
   s.description = %q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
   s.email = %q{seamus@abshere.net}
   s.extra_rdoc_files = [

data/lib/data_miner/attribute.rb

@@ -224,7 +224,7 @@ module DataMiner
       options[:static]
     end
     def dictionary
-      @_dictionary ||= Dictionary.new options[:dictionary]
+      @_dictionary ||= (options[:dictionary].is_a?(Dictionary) ? options[:dictionary] : Dictionary.new(options[:dictionary]))
     end
     def matcher
       @_matcher ||= (options[:matcher].is_a?(String) ? options[:matcher].constantize.new : options[:matcher])

data/lib/data_miner/import.rb

@@ -49,6 +49,7 @@ module DataMiner
     def run(run)
       increment_counter = resource.column_names.include?('data_miner_touch_count')
       log_run = resource.column_names.include?('data_miner_last_run_id')
+      primary_key = resource.primary_key
       test_counter = 0
 
       table.each_row do |row|
@@ -67,7 +68,7 @@ OUT: #{attributes.inject(Hash.new) { |memo, v| attr_name, attr = v; memo[attr_na
           record.increment :data_miner_touch_count if increment_counter
           record.data_miner_last_run = run if log_run
         end
-        record.save!
+        record.save! if record.send(primary_key).present?
       end
       DataMiner.log_info "performed #{inspect}"
     end

data/test/data_miner_test.rb

@@ -482,6 +482,19 @@ class CensusDivision < ActiveRecord::Base
   end
 end
 
+class CensusDivisionDeux < ActiveRecord::Base
+  set_primary_key :number
+  
+  data_miner do
+    import :url => 'http://www.census.gov/popest/geographic/codes02.csv', :skip => 9, :select => lambda { |row| row['Division'].to_s.strip != 'X' and row['FIPS CODE STATE'].to_s.strip == 'X'} do
+      key 'number', :field_name => 'Division'
+      store 'name', :field_name => 'Name'
+      store 'census_region_number', :field_name => 'Region'
+      store 'census_region_name', :field_name => 'Region', :dictionary => DataMiner::Dictionary.new(:input => 'number', :output => 'name', :url => 'http://data.brighterplanet.com/census_regions.csv')
+    end
+  end
+end
+
 class CrosscallingCensusRegion < ActiveRecord::Base
   set_primary_key :number
   
@@ -1009,6 +1022,12 @@ class Aircraft < ActiveRecord::Base
         store 'manufacturer_name', :field_name => 'Manufacturer'
         store 'name', :field_name => 'Model'
       end
+      
+      import 'Brighter Planet aircraft class codes',
+             :url => 'http://static.brighterplanet.com/science/data/transport/air/bts_aircraft_type/bts_aircraft_types-brighter_planet_aircraft_classes.csv' do
+        key   'bts_aircraft_type_code', :field_name => 'bts_aircraft_type'
+        store 'brighter_planet_aircraft_class_code'
+      end
     end
   end
 end
@@ -1110,6 +1129,16 @@ class DataMinerTest < Test::Unit::TestCase
   end
     
   if ENV['ALL'] == 'true' or ENV['FAST'] == 'true'
+    should "allow specifying dictionaries explicitly" do
+      CensusDivisionDeux.run_data_miner!
+      assert_equal 'South Region', CensusDivisionDeux.find(5).census_region_name
+    end
+    
+    should "be able to key on things other than the primary key" do
+      Aircraft.run_data_miner!
+      assert_equal 'SP', Aircraft.find('DHC6').brighter_planet_aircraft_class_code
+    end
+    
     should "be able to synthesize rows without using a full parser class" do
       AutomobileMakeFleetYear.run_data_miner!
       assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')

data/test/test_helper.rb

@@ -15,7 +15,7 @@ ActiveRecord::Base.establish_connection(
 )
 
 ActiveSupport::Inflector.inflections do |inflect|
-  inflect.uncountable %w{ aircraft aircraft_deux }
+  inflect.uncountable %w{ aircraft aircraft_deux census_division_deux }
 end
 
 class Test::Unit::TestCase
@@ -138,6 +138,19 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
   end
   execute 'ALTER TABLE census_divisions ADD PRIMARY KEY (number);'
   
+  create_table 'census_division_deux', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+    t.integer  'number'
+    t.string   'name'
+    t.datetime 'updated_at'
+    t.datetime 'created_at'
+    t.string   'census_region_name'
+    t.integer  'census_region_number'
+    
+    t.integer 'data_miner_touch_count'
+    t.integer 'data_miner_last_run_id'
+  end
+  execute 'ALTER TABLE census_division_deux ADD PRIMARY KEY (number);'
+  
   create_table 'crosscalling_census_divisions', :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
     t.integer  'number'
     t.string   'name'
@@ -282,7 +295,7 @@ ActiveRecord::Schema.define(:version => 20090819143429) do
     t.string   "bts_name"
     t.string "bts_aircraft_type_code"
 
-    # t.string   'brighter_planet_aircraft_class_code'
+    t.string   'brighter_planet_aircraft_class_code'
     # t.float    'm3'
     # t.float    'm2'
     # t.float    'm1'

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 4
-  - 29
-  version: 0.4.29
+  - 30
+  version: 0.4.30
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-17 00:00:00 -04:00
+date: 2010-05-18 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency