data_miner 0.4.27 → 0.4.28

data/VERSION

@@ -1 +1 @@
-0.4.27
+0.4.28

data/data_miner.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_miner}
-  s.version = "0.4.27"
+  s.version = "0.4.28"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
-  s.date = %q{2010-05-07}
+  s.date = %q{2010-05-14}
   s.description = %q{Mine remote data into your ActiveRecord models. You can also perform associations and convert units.}
   s.email = %q{seamus@abshere.net}
   s.extra_rdoc_files = [

data/lib/data_miner/attribute.rb

@@ -23,7 +23,8 @@ module DataMiner
       :units_field_name,
       :units_field_number,
       :field_number,
-      :chars
+      :chars,
+      :synthesize
     ]
 
     def initialize(step, name, options = {})
@@ -82,6 +83,7 @@ module DataMiner
       value = value_in_source row
       return value if value.is_a? ActiveRecord::Base # carry through trapdoor
       value = value_in_dictionary value if wants_dictionary?
+      value = synthesize.call(row) if wants_synthesize?
       value
     end
         
@@ -153,6 +155,9 @@ module DataMiner
     def wants_chars?
       chars.present?
     end
+    def wants_synthesize?
+      synthesize.is_a?(Proc)
+    end
     def wants_overwriting?
       overwrite != false
     end
@@ -212,6 +217,9 @@ module DataMiner
     def chars
       options[:chars]
     end
+    def synthesize
+      options[:synthesize]
+    end
     def static
       options[:static]
     end

data/test/data_miner_test.rb

@@ -1068,12 +1068,53 @@ class AircraftDeux < ActiveRecord::Base
   end
 end
 
+class AutomobileMakeFleetYear < ActiveRecord::Base
+  set_primary_key :name
+
+  data_miner do
+    process "create a table on the fly" do
+      create_table "automobile_make_fleet_years", :force => true, :options => 'ENGINE=InnoDB default charset=utf8', :id => false do |t|
+        t.string "name"
+        t.string   "make_name"
+        t.string   "fleet"
+        t.integer   "year"
+        t.float    "fuel_efficiency"
+        t.string "fuel_efficiency_units"
+        t.integer  "volume"
+        t.string  "make_year_name"
+        t.datetime "created_at"
+        t.datetime "updated_at"
+        t.integer  'data_miner_touch_count'
+        t.integer  'data_miner_last_run_id'
+      end
+      execute 'ALTER TABLE automobile_make_fleet_years ADD PRIMARY KEY (name)'
+    end
+
+    # CAFE data privately emailed to Andy from Terry Anderson at the DOT/NHTSA
+    import :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/make_fleet_years/make_fleet_years.csv',
+           :errata => 'http://static.brighterplanet.com/science/data/transport/automobiles/make_fleet_years/errata.csv',
+           :select => lambda { |row| row['volume'].to_i > 0 } do
+      key   'name', :synthesize => lambda { |row| [ row['manufacturer_name'], row['fleet'][2,2], row['year_content'] ].join ' ' }
+      store 'make_name', :field_name => 'manufacturer_name'
+      store 'year', :field_name => 'year_content'
+      store 'fleet', :chars => 2..3 # zero-based
+      store 'fuel_efficiency', :from_units => :miles_per_gallon, :to_units => :kilometres_per_litre
+      store 'volume'
+    end
+  end
+end
+
 # todo: have somebody properly organize these
 class DataMinerTest < Test::Unit::TestCase
   if ENV['ALL'] == 'true' or ENV['NEW'] == 'true'
   end
     
   if ENV['ALL'] == 'true' or ENV['FAST'] == 'true'
+    should "be able to synthesize rows without using a full parser class" do
+      AutomobileMakeFleetYear.run_data_miner!
+      assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
+    end
+    
     should "keep a call stack so that you can call run_data_miner! on a child" do
       CrosscallingCensusDivision.run_data_miner!
       assert CrosscallingCensusDivision.exists? :name => 'Mountain Division', :number => 8, :census_region_number => 4, :census_region_name => 'West Region'

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 4
-  - 27
-  version: 0.4.27
+  - 28
+  version: 0.4.28
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-07 00:00:00 -04:00
+date: 2010-05-14 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency