data_miner 0.5.6 → 0.5.7

data/lib/data_miner/base.rb

@@ -42,6 +42,14 @@ module DataMiner
       self.step_counter += 1
     end
 
+    def verify(*args, &block)
+      description = args.shift or '(no description)'
+
+      step = DataMiner::Verify.new self, step_counter, description, block
+      steps << step
+      self.step_counter += 1
+    end
+
     # Mine data for this class.
     def run(options = {})
       options.symbolize_keys!
@@ -68,6 +76,8 @@ module DataMiner
         finished = true
       rescue DataMiner::Skip
         skipped = true
+      rescue DataMiner::Verify::VerificationFailed
+        skipped = true
       ensure
         if DataMiner::Run.table_exists?
           run.update_attributes! :terminated_at => Time.now, :finished => finished, :skipped => skipped, :killed => false

data/lib/data_miner/import.rb

@@ -3,29 +3,37 @@ module DataMiner
     include Blockenspiel::DSL
     
     attr_reader :attributes
-    attr_accessor :base, :position_in_run, :table
+    attr_accessor :base
+    attr_accessor :position_in_run
+    attr_accessor :table_options
     attr_accessor :description
     delegate :resource, :to => :base
     
     def initialize(base, position_in_run, description, table_options = {})
-      table_options.symbolize_keys!
+      @table_options = table_options
+      @table_options.symbolize_keys!
 
       @attributes = ActiveSupport::OrderedHash.new
       @base = base
       @position_in_run = position_in_run
       @description = description
       
-      if table_options[:errata].is_a?(String)
-        table_options[:errata] = Errata.new :url => table_options[:errata], :responder => resource
+      if @table_options[:errata].is_a?(String)
+        @table_options[:errata] = Errata.new :url => @table_options[:errata], :responder => resource
       end
         
-      if table_options[:table].present?
-        DataMiner.log_or_raise "You should specify :table or :url, but not both" if table_options[:url].present?
-        @table = table_options[:table]
-      else
-        @table = RemoteTable.new table_options
+      if @table_options[:table] and @table_options[:url].present?
+        DataMiner.log_or_raise "You should specify :table or :url, but not both"
       end
     end
+    
+    def table
+      @table ||= (table_options[:table] || RemoteTable.new(table_options))
+    end
+    
+    def clear_table
+      @table = nil
+    end
 
     def inspect
       "Import(#{resource}) position #{position_in_run} (#{description})"
@@ -65,6 +73,8 @@ OUT: #{attributes.inject(Hash.new) { |memo, v| attr_name, attr = v; memo[attr_na
         record.save! if record.send(primary_key).present?
       end
       DataMiner.log_info "performed #{inspect}"
+      clear_table
+      nil
     end
   end
 end

data/lib/data_miner/verify.rb

@@ -0,0 +1,35 @@
+module DataMiner
+  class Verify
+    class VerificationFailed < RuntimeError; end
+
+    attr_accessor :base, :position_in_run, :check, :description
+    delegate :resource, :to => :base
+    
+    def initialize(base, position_in_run, description, check)
+      self.base = base
+      self.position_in_run = position_in_run
+      self.description = description
+      self.check = check
+    end
+
+    def inspect
+      "Verify(#{resource}) position #{position_in_run} (#{description})"
+    end
+
+    def run(run)
+      begin
+        verification = check.call
+      rescue Exception => e  # need this to catch Test::Unit assertions
+        raise VerificationFailed,
+          "#{e.inspect}: #{e.backtrace.join("\n")}"
+      rescue => e
+        raise VerificationFailed,
+          "#{e.inspect}: #{e.backtrace.join("\n")}"
+      end
+      unless verification
+        raise VerificationFailed, "Result of check was false" 
+      end
+      DataMiner.log_info "performed #{inspect}"
+    end
+  end
+end

data/lib/data_miner.rb

@@ -28,6 +28,7 @@ require 'data_miner/tap'
 require 'data_miner/process'
 require 'data_miner/run'
 require 'data_miner/schema'
+require 'data_miner/verify'
 
 module DataMiner
   class MissingHashColumn < StandardError; end

data/test/data_miner/verify_test.rb

@@ -0,0 +1,29 @@
+require 'test_helper'
+
+class DataMiner::VerifyTest < Test::Unit::TestCase
+  context '#run' do
+    setup do
+      @run = Object.new
+      check = lambda do
+        assert true
+      end
+      @verify = DataMiner::Verify.new Aircraft.new.data_miner_base, 1,
+        'verification of engine type', check
+    end
+    should 'raise an exception if the verification block fails through exception' do
+      @verify.check = lambda do
+        assert false
+      end
+      assert_raise(DataMiner::Verify::VerificationFailed) { @verify.run @run }
+    end
+    should 'raise an exception if the result of the verification block is false' do
+      @verify.check = lambda do
+        false
+      end
+      assert_raise(DataMiner::Verify::VerificationFailed) { @verify.run @run }
+    end
+    should 'return true if the verification block succeeds' do
+      assert @verify.run(@run)
+    end
+  end
+end

data/test/data_miner_test.rb

@@ -870,4 +870,15 @@ class DataMinerTest < Test::Unit::TestCase
       assert ResidentialEnergyConsumptionSurveyResponse.find(6).residence_class.start_with?('Single-family detached house')
     end
   end
+  should "mark the run as skipped if verification fails" do
+    AutomobileFuelType.data_miner_base.instance_eval do
+      verify "failure" do
+        false
+      end
+    end
+
+    DataMiner::Run.delete_all
+    AutomobileFuelType.run_data_miner! :from_scratch => true
+    assert DataMiner::Run.first.skipped
+  end
 end

data/test/support/aircraft.rb

@@ -93,6 +93,10 @@ class Aircraft < ActiveRecord::Base
         key   'bts_aircraft_type_code', :field_name => 'bts_aircraft_type'
         store 'brighter_planet_aircraft_class_code', :nullify => true
       end
+
+      verify 'all aircraft have a manufcaturer name specified' do
+        assert Aircraft.scoped.where(:manufacturer_name => nil).empty?
+      end
     end
   end
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 5
-  - 6
-  version: 0.5.6
+  - 7
+  version: 0.5.7
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-01 00:00:00 -04:00
+date: 2010-11-16 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -270,7 +270,9 @@ files:
 - lib/data_miner/run.rb
 - lib/data_miner/schema.rb
 - lib/data_miner/tap.rb
+- lib/data_miner/verify.rb
 - test/data_miner/attribute_test.rb
+- test/data_miner/verify_test.rb
 - test/support/airport.rb
 - test/support/country.rb
 - test/support/automobile_fuel_type.rb
@@ -314,6 +316,7 @@ specification_version: 3
 summary: Mine remote data into your ActiveRecord models.
 test_files: 
 - test/data_miner/attribute_test.rb
+- test/data_miner/verify_test.rb
 - test/support/airport.rb
 - test/support/country.rb
 - test/support/automobile_fuel_type.rb