data_miner 0.5.6 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/data_miner/base.rb +10 -0
- data/lib/data_miner/import.rb +19 -9
- data/lib/data_miner/verify.rb +35 -0
- data/lib/data_miner.rb +1 -0
- data/test/data_miner/verify_test.rb +29 -0
- data/test/data_miner_test.rb +11 -0
- data/test/support/aircraft.rb +4 -0
- metadata +6 -3
data/lib/data_miner/base.rb
CHANGED
@@ -42,6 +42,14 @@ module DataMiner
|
|
42
42
|
self.step_counter += 1
|
43
43
|
end
|
44
44
|
|
45
|
+
def verify(*args, &block)
|
46
|
+
description = args.shift or '(no description)'
|
47
|
+
|
48
|
+
step = DataMiner::Verify.new self, step_counter, description, block
|
49
|
+
steps << step
|
50
|
+
self.step_counter += 1
|
51
|
+
end
|
52
|
+
|
45
53
|
# Mine data for this class.
|
46
54
|
def run(options = {})
|
47
55
|
options.symbolize_keys!
|
@@ -68,6 +76,8 @@ module DataMiner
|
|
68
76
|
finished = true
|
69
77
|
rescue DataMiner::Skip
|
70
78
|
skipped = true
|
79
|
+
rescue DataMiner::Verify::VerificationFailed
|
80
|
+
skipped = true
|
71
81
|
ensure
|
72
82
|
if DataMiner::Run.table_exists?
|
73
83
|
run.update_attributes! :terminated_at => Time.now, :finished => finished, :skipped => skipped, :killed => false
|
data/lib/data_miner/import.rb
CHANGED
@@ -3,29 +3,37 @@ module DataMiner
|
|
3
3
|
include Blockenspiel::DSL
|
4
4
|
|
5
5
|
attr_reader :attributes
|
6
|
-
attr_accessor :base
|
6
|
+
attr_accessor :base
|
7
|
+
attr_accessor :position_in_run
|
8
|
+
attr_accessor :table_options
|
7
9
|
attr_accessor :description
|
8
10
|
delegate :resource, :to => :base
|
9
11
|
|
10
12
|
def initialize(base, position_in_run, description, table_options = {})
|
11
|
-
table_options
|
13
|
+
@table_options = table_options
|
14
|
+
@table_options.symbolize_keys!
|
12
15
|
|
13
16
|
@attributes = ActiveSupport::OrderedHash.new
|
14
17
|
@base = base
|
15
18
|
@position_in_run = position_in_run
|
16
19
|
@description = description
|
17
20
|
|
18
|
-
if table_options[:errata].is_a?(String)
|
19
|
-
table_options[:errata] = Errata.new :url => table_options[:errata], :responder => resource
|
21
|
+
if @table_options[:errata].is_a?(String)
|
22
|
+
@table_options[:errata] = Errata.new :url => @table_options[:errata], :responder => resource
|
20
23
|
end
|
21
24
|
|
22
|
-
if table_options[:table].present?
|
23
|
-
DataMiner.log_or_raise "You should specify :table or :url, but not both"
|
24
|
-
@table = table_options[:table]
|
25
|
-
else
|
26
|
-
@table = RemoteTable.new table_options
|
25
|
+
if @table_options[:table] and @table_options[:url].present?
|
26
|
+
DataMiner.log_or_raise "You should specify :table or :url, but not both"
|
27
27
|
end
|
28
28
|
end
|
29
|
+
|
30
|
+
def table
|
31
|
+
@table ||= (table_options[:table] || RemoteTable.new(table_options))
|
32
|
+
end
|
33
|
+
|
34
|
+
def clear_table
|
35
|
+
@table = nil
|
36
|
+
end
|
29
37
|
|
30
38
|
def inspect
|
31
39
|
"Import(#{resource}) position #{position_in_run} (#{description})"
|
@@ -65,6 +73,8 @@ OUT: #{attributes.inject(Hash.new) { |memo, v| attr_name, attr = v; memo[attr_na
|
|
65
73
|
record.save! if record.send(primary_key).present?
|
66
74
|
end
|
67
75
|
DataMiner.log_info "performed #{inspect}"
|
76
|
+
clear_table
|
77
|
+
nil
|
68
78
|
end
|
69
79
|
end
|
70
80
|
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module DataMiner
|
2
|
+
class Verify
|
3
|
+
class VerificationFailed < RuntimeError; end
|
4
|
+
|
5
|
+
attr_accessor :base, :position_in_run, :check, :description
|
6
|
+
delegate :resource, :to => :base
|
7
|
+
|
8
|
+
def initialize(base, position_in_run, description, check)
|
9
|
+
self.base = base
|
10
|
+
self.position_in_run = position_in_run
|
11
|
+
self.description = description
|
12
|
+
self.check = check
|
13
|
+
end
|
14
|
+
|
15
|
+
def inspect
|
16
|
+
"Verify(#{resource}) position #{position_in_run} (#{description})"
|
17
|
+
end
|
18
|
+
|
19
|
+
def run(run)
|
20
|
+
begin
|
21
|
+
verification = check.call
|
22
|
+
rescue Exception => e # need this to catch Test::Unit assertions
|
23
|
+
raise VerificationFailed,
|
24
|
+
"#{e.inspect}: #{e.backtrace.join("\n")}"
|
25
|
+
rescue => e
|
26
|
+
raise VerificationFailed,
|
27
|
+
"#{e.inspect}: #{e.backtrace.join("\n")}"
|
28
|
+
end
|
29
|
+
unless verification
|
30
|
+
raise VerificationFailed, "Result of check was false"
|
31
|
+
end
|
32
|
+
DataMiner.log_info "performed #{inspect}"
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
data/lib/data_miner.rb
CHANGED
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class DataMiner::VerifyTest < Test::Unit::TestCase
|
4
|
+
context '#run' do
|
5
|
+
setup do
|
6
|
+
@run = Object.new
|
7
|
+
check = lambda do
|
8
|
+
assert true
|
9
|
+
end
|
10
|
+
@verify = DataMiner::Verify.new Aircraft.new.data_miner_base, 1,
|
11
|
+
'verification of engine type', check
|
12
|
+
end
|
13
|
+
should 'raise an exception if the verification block fails through exception' do
|
14
|
+
@verify.check = lambda do
|
15
|
+
assert false
|
16
|
+
end
|
17
|
+
assert_raise(DataMiner::Verify::VerificationFailed) { @verify.run @run }
|
18
|
+
end
|
19
|
+
should 'raise an exception if the result of the verification block is false' do
|
20
|
+
@verify.check = lambda do
|
21
|
+
false
|
22
|
+
end
|
23
|
+
assert_raise(DataMiner::Verify::VerificationFailed) { @verify.run @run }
|
24
|
+
end
|
25
|
+
should 'return true if the verification block succeeds' do
|
26
|
+
assert @verify.run(@run)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/test/data_miner_test.rb
CHANGED
@@ -870,4 +870,15 @@ class DataMinerTest < Test::Unit::TestCase
|
|
870
870
|
assert ResidentialEnergyConsumptionSurveyResponse.find(6).residence_class.start_with?('Single-family detached house')
|
871
871
|
end
|
872
872
|
end
|
873
|
+
should "mark the run as skipped if verification fails" do
|
874
|
+
AutomobileFuelType.data_miner_base.instance_eval do
|
875
|
+
verify "failure" do
|
876
|
+
false
|
877
|
+
end
|
878
|
+
end
|
879
|
+
|
880
|
+
DataMiner::Run.delete_all
|
881
|
+
AutomobileFuelType.run_data_miner! :from_scratch => true
|
882
|
+
assert DataMiner::Run.first.skipped
|
883
|
+
end
|
873
884
|
end
|
data/test/support/aircraft.rb
CHANGED
@@ -93,6 +93,10 @@ class Aircraft < ActiveRecord::Base
|
|
93
93
|
key 'bts_aircraft_type_code', :field_name => 'bts_aircraft_type'
|
94
94
|
store 'brighter_planet_aircraft_class_code', :nullify => true
|
95
95
|
end
|
96
|
+
|
97
|
+
verify 'all aircraft have a manufcaturer name specified' do
|
98
|
+
assert Aircraft.scoped.where(:manufacturer_name => nil).empty?
|
99
|
+
end
|
96
100
|
end
|
97
101
|
end
|
98
102
|
end
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 5
|
8
|
-
-
|
9
|
-
version: 0.5.
|
8
|
+
- 7
|
9
|
+
version: 0.5.7
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Seamus Abshere
|
@@ -16,7 +16,7 @@ autorequire:
|
|
16
16
|
bindir: bin
|
17
17
|
cert_chain: []
|
18
18
|
|
19
|
-
date: 2010-11-
|
19
|
+
date: 2010-11-16 00:00:00 -05:00
|
20
20
|
default_executable:
|
21
21
|
dependencies:
|
22
22
|
- !ruby/object:Gem::Dependency
|
@@ -270,7 +270,9 @@ files:
|
|
270
270
|
- lib/data_miner/run.rb
|
271
271
|
- lib/data_miner/schema.rb
|
272
272
|
- lib/data_miner/tap.rb
|
273
|
+
- lib/data_miner/verify.rb
|
273
274
|
- test/data_miner/attribute_test.rb
|
275
|
+
- test/data_miner/verify_test.rb
|
274
276
|
- test/support/airport.rb
|
275
277
|
- test/support/country.rb
|
276
278
|
- test/support/automobile_fuel_type.rb
|
@@ -314,6 +316,7 @@ specification_version: 3
|
|
314
316
|
summary: Mine remote data into your ActiveRecord models.
|
315
317
|
test_files:
|
316
318
|
- test/data_miner/attribute_test.rb
|
319
|
+
- test/data_miner/verify_test.rb
|
317
320
|
- test/support/airport.rb
|
318
321
|
- test/support/country.rb
|
319
322
|
- test/support/automobile_fuel_type.rb
|