data_miner 0.5.6 → 0.5.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -42,6 +42,14 @@ module DataMiner
42
42
  self.step_counter += 1
43
43
  end
44
44
 
45
+ def verify(*args, &block)
46
+ description = args.shift or '(no description)'
47
+
48
+ step = DataMiner::Verify.new self, step_counter, description, block
49
+ steps << step
50
+ self.step_counter += 1
51
+ end
52
+
45
53
  # Mine data for this class.
46
54
  def run(options = {})
47
55
  options.symbolize_keys!
@@ -68,6 +76,8 @@ module DataMiner
68
76
  finished = true
69
77
  rescue DataMiner::Skip
70
78
  skipped = true
79
+ rescue DataMiner::Verify::VerificationFailed
80
+ skipped = true
71
81
  ensure
72
82
  if DataMiner::Run.table_exists?
73
83
  run.update_attributes! :terminated_at => Time.now, :finished => finished, :skipped => skipped, :killed => false
@@ -3,29 +3,37 @@ module DataMiner
3
3
  include Blockenspiel::DSL
4
4
 
5
5
  attr_reader :attributes
6
- attr_accessor :base, :position_in_run, :table
6
+ attr_accessor :base
7
+ attr_accessor :position_in_run
8
+ attr_accessor :table_options
7
9
  attr_accessor :description
8
10
  delegate :resource, :to => :base
9
11
 
10
12
  def initialize(base, position_in_run, description, table_options = {})
11
- table_options.symbolize_keys!
13
+ @table_options = table_options
14
+ @table_options.symbolize_keys!
12
15
 
13
16
  @attributes = ActiveSupport::OrderedHash.new
14
17
  @base = base
15
18
  @position_in_run = position_in_run
16
19
  @description = description
17
20
 
18
- if table_options[:errata].is_a?(String)
19
- table_options[:errata] = Errata.new :url => table_options[:errata], :responder => resource
21
+ if @table_options[:errata].is_a?(String)
22
+ @table_options[:errata] = Errata.new :url => @table_options[:errata], :responder => resource
20
23
  end
21
24
 
22
- if table_options[:table].present?
23
- DataMiner.log_or_raise "You should specify :table or :url, but not both" if table_options[:url].present?
24
- @table = table_options[:table]
25
- else
26
- @table = RemoteTable.new table_options
25
+ if @table_options[:table] and @table_options[:url].present?
26
+ DataMiner.log_or_raise "You should specify :table or :url, but not both"
27
27
  end
28
28
  end
29
+
30
+ def table
31
+ @table ||= (table_options[:table] || RemoteTable.new(table_options))
32
+ end
33
+
34
+ def clear_table
35
+ @table = nil
36
+ end
29
37
 
30
38
  def inspect
31
39
  "Import(#{resource}) position #{position_in_run} (#{description})"
@@ -65,6 +73,8 @@ OUT: #{attributes.inject(Hash.new) { |memo, v| attr_name, attr = v; memo[attr_na
65
73
  record.save! if record.send(primary_key).present?
66
74
  end
67
75
  DataMiner.log_info "performed #{inspect}"
76
+ clear_table
77
+ nil
68
78
  end
69
79
  end
70
80
  end
@@ -0,0 +1,35 @@
1
+ module DataMiner
2
+ class Verify
3
+ class VerificationFailed < RuntimeError; end
4
+
5
+ attr_accessor :base, :position_in_run, :check, :description
6
+ delegate :resource, :to => :base
7
+
8
+ def initialize(base, position_in_run, description, check)
9
+ self.base = base
10
+ self.position_in_run = position_in_run
11
+ self.description = description
12
+ self.check = check
13
+ end
14
+
15
+ def inspect
16
+ "Verify(#{resource}) position #{position_in_run} (#{description})"
17
+ end
18
+
19
+ def run(run)
20
+ begin
21
+ verification = check.call
22
+ rescue Exception => e # need this to catch Test::Unit assertions
23
+ raise VerificationFailed,
24
+ "#{e.inspect}: #{e.backtrace.join("\n")}"
25
+ rescue => e
26
+ raise VerificationFailed,
27
+ "#{e.inspect}: #{e.backtrace.join("\n")}"
28
+ end
29
+ unless verification
30
+ raise VerificationFailed, "Result of check was false"
31
+ end
32
+ DataMiner.log_info "performed #{inspect}"
33
+ end
34
+ end
35
+ end
data/lib/data_miner.rb CHANGED
@@ -28,6 +28,7 @@ require 'data_miner/tap'
28
28
  require 'data_miner/process'
29
29
  require 'data_miner/run'
30
30
  require 'data_miner/schema'
31
+ require 'data_miner/verify'
31
32
 
32
33
  module DataMiner
33
34
  class MissingHashColumn < StandardError; end
@@ -0,0 +1,29 @@
1
+ require 'test_helper'
2
+
3
+ class DataMiner::VerifyTest < Test::Unit::TestCase
4
+ context '#run' do
5
+ setup do
6
+ @run = Object.new
7
+ check = lambda do
8
+ assert true
9
+ end
10
+ @verify = DataMiner::Verify.new Aircraft.new.data_miner_base, 1,
11
+ 'verification of engine type', check
12
+ end
13
+ should 'raise an exception if the verification block fails through exception' do
14
+ @verify.check = lambda do
15
+ assert false
16
+ end
17
+ assert_raise(DataMiner::Verify::VerificationFailed) { @verify.run @run }
18
+ end
19
+ should 'raise an exception if the result of the verification block is false' do
20
+ @verify.check = lambda do
21
+ false
22
+ end
23
+ assert_raise(DataMiner::Verify::VerificationFailed) { @verify.run @run }
24
+ end
25
+ should 'return true if the verification block succeeds' do
26
+ assert @verify.run(@run)
27
+ end
28
+ end
29
+ end
@@ -870,4 +870,15 @@ class DataMinerTest < Test::Unit::TestCase
870
870
  assert ResidentialEnergyConsumptionSurveyResponse.find(6).residence_class.start_with?('Single-family detached house')
871
871
  end
872
872
  end
873
+ should "mark the run as skipped if verification fails" do
874
+ AutomobileFuelType.data_miner_base.instance_eval do
875
+ verify "failure" do
876
+ false
877
+ end
878
+ end
879
+
880
+ DataMiner::Run.delete_all
881
+ AutomobileFuelType.run_data_miner! :from_scratch => true
882
+ assert DataMiner::Run.first.skipped
883
+ end
873
884
  end
@@ -93,6 +93,10 @@ class Aircraft < ActiveRecord::Base
93
93
  key 'bts_aircraft_type_code', :field_name => 'bts_aircraft_type'
94
94
  store 'brighter_planet_aircraft_class_code', :nullify => true
95
95
  end
96
+
97
+ verify 'all aircraft have a manufcaturer name specified' do
98
+ assert Aircraft.scoped.where(:manufacturer_name => nil).empty?
99
+ end
96
100
  end
97
101
  end
98
102
  end
metadata CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
5
5
  segments:
6
6
  - 0
7
7
  - 5
8
- - 6
9
- version: 0.5.6
8
+ - 7
9
+ version: 0.5.7
10
10
  platform: ruby
11
11
  authors:
12
12
  - Seamus Abshere
@@ -16,7 +16,7 @@ autorequire:
16
16
  bindir: bin
17
17
  cert_chain: []
18
18
 
19
- date: 2010-11-01 00:00:00 -04:00
19
+ date: 2010-11-16 00:00:00 -05:00
20
20
  default_executable:
21
21
  dependencies:
22
22
  - !ruby/object:Gem::Dependency
@@ -270,7 +270,9 @@ files:
270
270
  - lib/data_miner/run.rb
271
271
  - lib/data_miner/schema.rb
272
272
  - lib/data_miner/tap.rb
273
+ - lib/data_miner/verify.rb
273
274
  - test/data_miner/attribute_test.rb
275
+ - test/data_miner/verify_test.rb
274
276
  - test/support/airport.rb
275
277
  - test/support/country.rb
276
278
  - test/support/automobile_fuel_type.rb
@@ -314,6 +316,7 @@ specification_version: 3
314
316
  summary: Mine remote data into your ActiveRecord models.
315
317
  test_files:
316
318
  - test/data_miner/attribute_test.rb
319
+ - test/data_miner/verify_test.rb
317
320
  - test/support/airport.rb
318
321
  - test/support/country.rb
319
322
  - test/support/automobile_fuel_type.rb