resque_jobs_tree 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/resque_jobs_tree.rb +25 -3
- data/lib/resque_jobs_tree/definitions.rb +15 -0
- data/lib/resque_jobs_tree/definitions/node.rb +77 -0
- data/lib/resque_jobs_tree/definitions/tree.rb +39 -0
- data/lib/resque_jobs_tree/factory.rb +7 -7
- data/lib/resque_jobs_tree/job.rb +13 -31
- data/lib/resque_jobs_tree/node.rb +64 -60
- data/lib/resque_jobs_tree/resources_serializer.rb +2 -3
- data/lib/resque_jobs_tree/storage.rb +5 -112
- data/lib/resque_jobs_tree/storage/node.rb +80 -0
- data/lib/resque_jobs_tree/storage/tree.rb +26 -0
- data/lib/resque_jobs_tree/tree.rb +32 -27
- data/lib/resque_jobs_tree/version.rb +1 -1
- data/test/definitions_test.rb +99 -0
- data/test/factory_test.rb +3 -4
- data/test/job_test.rb +6 -8
- data/test/process_test.rb +214 -0
- data/test/resources_serializer_test.rb +6 -6
- data/test/storage_node_test.rb +117 -0
- data/test/storage_tree_test.rb +27 -0
- data/test/test_helper.rb +12 -8
- data/test/tree_test.rb +14 -109
- metadata +15 -6
- data/test/node_test.rb +0 -188
- data/test/storage_test.rb +0 -74
@@ -0,0 +1,80 @@
|
|
1
|
+
module ResqueJobsTree::Storage::Node
|
2
|
+
include ResqueJobsTree::Storage
|
3
|
+
|
4
|
+
def store
|
5
|
+
raise 'Can\'t store a root node' if root?
|
6
|
+
redis.hset PARENTS_KEY, key, parent.key
|
7
|
+
unless redis.sadd parent.childs_key, key
|
8
|
+
raise ResqueJobsTree::JobNotUniq,
|
9
|
+
"Job #{parent.name} already has the child #{name} with resources: #{resources}"
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def unstore
|
14
|
+
redis.srem parent.childs_key, key
|
15
|
+
redis.hdel PARENTS_KEY, key
|
16
|
+
end
|
17
|
+
|
18
|
+
def cleanup
|
19
|
+
unless definition.leaf?
|
20
|
+
stored_childs.each &:cleanup
|
21
|
+
redis.del childs_key
|
22
|
+
end
|
23
|
+
redis.hdel PARENTS_KEY, key
|
24
|
+
tree.unstore if root?
|
25
|
+
end
|
26
|
+
|
27
|
+
def childs_key
|
28
|
+
"#{key}:childs"
|
29
|
+
end
|
30
|
+
|
31
|
+
def key
|
32
|
+
"ResqueJobsTree:Node:#{serialize}"
|
33
|
+
end
|
34
|
+
|
35
|
+
def only_stored_child?
|
36
|
+
(redis.smembers(parent.childs_key) - [key]).empty?
|
37
|
+
end
|
38
|
+
|
39
|
+
def stored_childs
|
40
|
+
redis.smembers(childs_key).map do |_key|
|
41
|
+
node_name, _resources = node_info_from_key _key
|
42
|
+
definition.find(node_name).spawn _resources
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def parent
|
47
|
+
@parent ||= definition.parent.spawn node_info_from_key(parent_key).last
|
48
|
+
end
|
49
|
+
|
50
|
+
def lock_key
|
51
|
+
"#{key}:lock"
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def lock
|
57
|
+
_key = parent.lock_key
|
58
|
+
while !redis.setnx(_key, 'locked')
|
59
|
+
sleep 0.05 # 50 ms
|
60
|
+
end
|
61
|
+
yield
|
62
|
+
ensure
|
63
|
+
redis.del _key
|
64
|
+
end
|
65
|
+
|
66
|
+
def parent_key
|
67
|
+
redis.hget PARENTS_KEY, key
|
68
|
+
end
|
69
|
+
|
70
|
+
def node_info_from_key _key
|
71
|
+
tree_name, node_name, *resources_arguments = JSON.load(_key.gsub /ResqueJobsTree:Node:/, '')
|
72
|
+
_resources = ResqueJobsTree::ResourcesSerializer.instancize(resources_arguments)
|
73
|
+
[node_name, _resources]
|
74
|
+
end
|
75
|
+
|
76
|
+
def main_arguments
|
77
|
+
[definition.tree.name, name]
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module ResqueJobsTree::Storage::Tree
|
2
|
+
include ResqueJobsTree::Storage
|
3
|
+
|
4
|
+
def store
|
5
|
+
redis.sadd LAUNCHED_TREES, key
|
6
|
+
end
|
7
|
+
|
8
|
+
def unstore
|
9
|
+
redis.srem LAUNCHED_TREES, key
|
10
|
+
end
|
11
|
+
|
12
|
+
def key
|
13
|
+
"ResqueJobsTree:Tree:#{serialize}"
|
14
|
+
end
|
15
|
+
|
16
|
+
def uniq?
|
17
|
+
!redis.sismember LAUNCHED_TREES, key
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def main_arguments
|
23
|
+
[name]
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
@@ -1,55 +1,60 @@
|
|
1
1
|
class ResqueJobsTree::Tree
|
2
2
|
|
3
|
-
|
4
|
-
attr_reader :jobs
|
3
|
+
include ResqueJobsTree::Storage::Tree
|
5
4
|
|
6
|
-
|
7
|
-
@name = name.to_s
|
8
|
-
@jobs = []
|
9
|
-
end
|
5
|
+
attr_reader :definition, :resources, :leaves
|
10
6
|
|
11
|
-
def
|
12
|
-
@
|
13
|
-
|
14
|
-
|
7
|
+
def initialize definition, resources
|
8
|
+
@definition = definition
|
9
|
+
@resources = resources
|
10
|
+
@leaves = []
|
15
11
|
end
|
16
12
|
|
17
|
-
def
|
18
|
-
@
|
13
|
+
def name
|
14
|
+
@definition.name
|
19
15
|
end
|
20
16
|
|
21
|
-
def launch
|
22
|
-
|
23
|
-
|
17
|
+
def launch
|
18
|
+
if uniq?
|
19
|
+
before_perform
|
20
|
+
store
|
21
|
+
root.launch
|
24
22
|
enqueue_leaves_jobs
|
25
23
|
end
|
26
24
|
end
|
27
25
|
|
28
|
-
|
29
|
-
|
26
|
+
%w(before_perform after_perform on_failure).each do |callback|
|
27
|
+
class_eval %Q{def #{callback} ; run_callback :#{callback} ; end}
|
30
28
|
end
|
31
29
|
|
32
|
-
def
|
33
|
-
root.
|
30
|
+
def root
|
31
|
+
@root ||= ResqueJobsTree::Node.new(definition.root, resources, nil, self)
|
34
32
|
end
|
35
33
|
|
36
|
-
def
|
37
|
-
|
38
|
-
root.validate!
|
34
|
+
def register_a_leaf node
|
35
|
+
@leaves << node
|
39
36
|
end
|
40
37
|
|
41
|
-
def
|
42
|
-
|
38
|
+
def inspect
|
39
|
+
"<ResqueJobsTree::Tree @name=#{name} @resources=#{resources} >"
|
43
40
|
end
|
44
41
|
|
45
|
-
def
|
46
|
-
|
42
|
+
def finish
|
43
|
+
after_perform
|
44
|
+
unstore
|
47
45
|
end
|
48
46
|
|
49
47
|
private
|
50
48
|
|
51
49
|
def enqueue_leaves_jobs
|
52
|
-
@
|
50
|
+
@leaves.each do |leaf|
|
51
|
+
leaf.enqueue unless leaf.definition.options[:async]
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def run_callback callback
|
56
|
+
callback = definition.send(callback)
|
57
|
+
callback.call(*resources) if callback.kind_of? Proc
|
53
58
|
end
|
54
59
|
|
55
60
|
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class DefinitionsTest < MiniTest::Unit::TestCase
|
4
|
+
|
5
|
+
def setup
|
6
|
+
@tree = ResqueJobsTree::Definitions::Tree.new :tree1
|
7
|
+
@root = ResqueJobsTree::Definitions::Node.new :node1, @tree
|
8
|
+
@tree.root = @root
|
9
|
+
@leaf = ResqueJobsTree::Definitions::Node.new :node2, @tree, @root
|
10
|
+
@root.node_childs = [@leaf]
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_perform
|
14
|
+
variable = 1
|
15
|
+
@root.perform do |n|
|
16
|
+
variable = n
|
17
|
+
end
|
18
|
+
@root.perform.call 2
|
19
|
+
assert_equal variable, 2
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_childs
|
23
|
+
variable = 1
|
24
|
+
@leaf.childs do |n|
|
25
|
+
variable = n
|
26
|
+
end
|
27
|
+
@leaf.childs.call 2
|
28
|
+
assert_equal variable, 2
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_node
|
32
|
+
node3 = @root.node :node3
|
33
|
+
assert_equal @root.find('node3').object_id, node3.object_id
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_leaf
|
37
|
+
assert @leaf.leaf?
|
38
|
+
assert !@root.leaf?
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_root
|
42
|
+
assert @root.root?
|
43
|
+
assert !@leaf.root?
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_siblings
|
47
|
+
node3 = ResqueJobsTree::Node.new :node3, @tree, @root
|
48
|
+
assert @leaf.siblings, [node3]
|
49
|
+
end
|
50
|
+
|
51
|
+
def test_childs_validation
|
52
|
+
assert_raises ResqueJobsTree::NodeDefinitionInvalid do
|
53
|
+
ResqueJobsTree::Factory.create :tree1 do
|
54
|
+
root :job1 do
|
55
|
+
perform {}
|
56
|
+
childs {}
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
ResqueJobsTree::Factory.create :tree1 do
|
61
|
+
root :job1 do
|
62
|
+
perform {}
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_perform_validation
|
68
|
+
assert_raises ResqueJobsTree::NodeDefinitionInvalid do
|
69
|
+
ResqueJobsTree::Factory.create :tree1 do
|
70
|
+
root :job1 do
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def test_perform_validation
|
77
|
+
assert_raises ResqueJobsTree::NodeDefinitionInvalid do
|
78
|
+
ResqueJobsTree::Factory.create :tree1 do
|
79
|
+
root :job1 do
|
80
|
+
perform {}
|
81
|
+
childs {}
|
82
|
+
node :job2 do
|
83
|
+
perform {}
|
84
|
+
end
|
85
|
+
node :job2 do
|
86
|
+
perform {}
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def test_options
|
94
|
+
options = { async: true }
|
95
|
+
@leaf.options = options
|
96
|
+
assert_equal options, @tree.find(:node2).options
|
97
|
+
end
|
98
|
+
|
99
|
+
end
|
data/test/factory_test.rb
CHANGED
@@ -7,12 +7,11 @@ class FactoryTest < MiniTest::Unit::TestCase
|
|
7
7
|
end
|
8
8
|
|
9
9
|
def test_tree_creation
|
10
|
-
assert_equal ResqueJobsTree::Factory.trees.first.
|
10
|
+
assert_equal ResqueJobsTree::Factory.trees.values.first.name, @tree_definition.name
|
11
11
|
end
|
12
12
|
|
13
|
-
def
|
14
|
-
assert_equal ResqueJobsTree::Factory.
|
15
|
-
@tree.object_id
|
13
|
+
def test_find
|
14
|
+
assert_equal ResqueJobsTree::Factory.find(@tree_definition.name).name, @tree_definition.name
|
16
15
|
end
|
17
16
|
|
18
17
|
end
|
data/test/job_test.rb
CHANGED
@@ -1,15 +1,13 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
|
3
3
|
class JobTest < MiniTest::Unit::TestCase
|
4
|
-
|
5
|
-
def setup
|
6
|
-
create_tree
|
7
|
-
@args = [@tree.name, @tree.find_node_by_name('job1').name, 1, 2, 3]
|
8
|
-
end
|
9
4
|
|
10
|
-
def
|
11
|
-
|
12
|
-
|
5
|
+
def test_node
|
6
|
+
create_tree
|
7
|
+
args = [@tree_definition.name, @tree_definition.find('job1').name, 1, 2, 3]
|
8
|
+
assert_equal @tree_definition.find(:job1), ResqueJobsTree::Job.send(:node, *args).definition
|
9
|
+
assert_equal ResqueJobsTree.find(:tree1), ResqueJobsTree::Job.send(:node, *args).definition.tree
|
10
|
+
assert_equal [1, 2, 3], ResqueJobsTree::Job.send(:node, *args).resources
|
13
11
|
end
|
14
12
|
|
15
13
|
end
|
@@ -0,0 +1,214 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class ProcessTest < MiniTest::Unit::TestCase
|
4
|
+
|
5
|
+
def test_launch
|
6
|
+
create_tree
|
7
|
+
resources = [1, 2, 3]
|
8
|
+
@tree_definition.spawn(resources).launch
|
9
|
+
history = ['tree1 job2']*3+['tree1 job1']
|
10
|
+
assert_equal history, redis.lrange('history', 0, -1)
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_launch_with_no_resources
|
14
|
+
create_tree
|
15
|
+
@tree_definition.spawn([]).launch
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_leaf_failure
|
19
|
+
tree_definition = ResqueJobsTree::Factory.create :tree1 do
|
20
|
+
root :job1 do
|
21
|
+
perform {}
|
22
|
+
childs { [:job2] }
|
23
|
+
node :job2 do
|
24
|
+
perform { raise ExpectedException, 'an expected exception'}
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
assert_raises ExpectedException do
|
29
|
+
tree_definition.spawn([1, 2, 3]).launch
|
30
|
+
end
|
31
|
+
assert_redis_empty
|
32
|
+
end
|
33
|
+
|
34
|
+
def test_launch_async
|
35
|
+
tree_definition = ResqueJobsTree::Factory.create :tree1 do
|
36
|
+
root :job1 do
|
37
|
+
perform { raise 'should not arrive here' }
|
38
|
+
childs { [:job2] }
|
39
|
+
node :job2, async: true do
|
40
|
+
perform {}
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
tree = tree_definition.spawn [1, 2, 3]
|
45
|
+
tree.launch
|
46
|
+
end
|
47
|
+
|
48
|
+
def test_launch_continue_on_failure
|
49
|
+
tree_definition = ResqueJobsTree::Factory.create :tree1 do
|
50
|
+
root :job1 do
|
51
|
+
perform do
|
52
|
+
Resque.redis.rpush 'history', 'tree1 job1'
|
53
|
+
raise ExpectedException, 'an expected failure'
|
54
|
+
end
|
55
|
+
childs { [:job2] }
|
56
|
+
node :job2, continue_on_failure: true do
|
57
|
+
perform do
|
58
|
+
Resque.redis.rpush 'history', 'tree1 job2'
|
59
|
+
raise 'an expected failure'
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
assert_raises ExpectedException do
|
65
|
+
tree = tree_definition.spawn [1, 2, 3]
|
66
|
+
tree.launch
|
67
|
+
end
|
68
|
+
assert_equal ['tree1 job2','tree1 job1'], redis.lrange('history', 0, -1)
|
69
|
+
redis.del 'history'
|
70
|
+
assert_redis_empty
|
71
|
+
end
|
72
|
+
|
73
|
+
def test_root_failure
|
74
|
+
Resque.inline = false
|
75
|
+
tree_definition = ResqueJobsTree::Factory.create :tree1 do
|
76
|
+
root :job1 do
|
77
|
+
perform do
|
78
|
+
Resque.redis.rpush 'history', 'tree1 job1'
|
79
|
+
raise ExpectedException, 'an expected exception'
|
80
|
+
end
|
81
|
+
childs { [:job2] }
|
82
|
+
node :job2 do
|
83
|
+
perform do
|
84
|
+
Resque.redis.rpush 'history', 'tree1 job2'
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
tree = tree_definition.spawn [1,2,3]
|
90
|
+
tree.launch
|
91
|
+
run_resque_workers tree_definition.name
|
92
|
+
assert_raises ExpectedException do
|
93
|
+
run_resque_workers tree_definition.name
|
94
|
+
end
|
95
|
+
assert_equal ['tree1 job2','tree1 job1'], redis.lrange('history', 0, -1)
|
96
|
+
redis.del 'history'
|
97
|
+
assert_redis_empty
|
98
|
+
end
|
99
|
+
|
100
|
+
def test_store_already_stored
|
101
|
+
wrong_tree_definition = ResqueJobsTree::Factory.create :tree1 do
|
102
|
+
root :job1 do
|
103
|
+
perform {}
|
104
|
+
childs do |resources|
|
105
|
+
[ [:job2], [:job2] ]
|
106
|
+
end
|
107
|
+
node :job2 do
|
108
|
+
perform {}
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
assert_raises ResqueJobsTree::JobNotUniq do
|
113
|
+
wrong_tree_definition.spawn([1]).launch
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def test_on_failure
|
118
|
+
tree_definition = ResqueJobsTree::Factory.create :tree1 do
|
119
|
+
root :job1 do
|
120
|
+
perform { raise }
|
121
|
+
end
|
122
|
+
on_failure do
|
123
|
+
raise ExpectedException, 'called from on_failure block'
|
124
|
+
end
|
125
|
+
end
|
126
|
+
assert_raises ExpectedException do
|
127
|
+
tree_definition.spawn([1]).launch
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
def test_tree_with_resource
|
132
|
+
tree_definition = ResqueJobsTree::Factory.create :tree1 do
|
133
|
+
root :job1 do
|
134
|
+
perform do |resource, number|
|
135
|
+
raise 'unknown resource' unless resource.kind_of?(Model)
|
136
|
+
raise 'unknown resource' unless number.kind_of?(Integer)
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
ResqueJobsTree.launch tree_definition.name, Model.new(1), 1
|
141
|
+
end
|
142
|
+
|
143
|
+
def test_nested_tree
|
144
|
+
Resque.inline = false
|
145
|
+
create_nested_tree
|
146
|
+
@tree_definition.spawn([1,2,3]).launch
|
147
|
+
assert_raises RuntimeError do # job4 error
|
148
|
+
run_resque_workers @tree_definition.name
|
149
|
+
end
|
150
|
+
Resque.enqueue_to 'tree1', ResqueJobsTree::Job, 'tree1', 'job3'
|
151
|
+
run_resque_workers @tree_definition.name # job3 error
|
152
|
+
assert_raises RuntimeError do # job2 error
|
153
|
+
run_resque_workers @tree_definition.name
|
154
|
+
end
|
155
|
+
assert_raises ExpectedException do # job1 error
|
156
|
+
run_resque_workers @tree_definition.name
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def test_async_tree
|
161
|
+
tree_definition = ResqueJobsTree::Factory.create :tree1 do
|
162
|
+
root :job1 do
|
163
|
+
perform { raise 'should not arrive here' }
|
164
|
+
childs { [ [:job2], [:job3] ] }
|
165
|
+
node :job2, async: true do
|
166
|
+
perform {}
|
167
|
+
end
|
168
|
+
node :job3 do
|
169
|
+
perform {}
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
ResqueJobsTree.launch tree_definition.name
|
174
|
+
assert_equal ["ResqueJobsTree:Node:[\"tree1\",\"job2\"]"],
|
175
|
+
Resque.redis.smembers("ResqueJobsTree:Node:[\"tree1\",\"job1\"]:childs")
|
176
|
+
parents_hash = { 'ResqueJobsTree:Node:["tree1","job2"]'=>'ResqueJobsTree:Node:["tree1","job1"]' }
|
177
|
+
assert_equal parents_hash, Resque.redis.hgetall(ResqueJobsTree::Storage::PARENTS_KEY)
|
178
|
+
end
|
179
|
+
|
180
|
+
def test_async_tree_with_fail
|
181
|
+
Resque.inline = false
|
182
|
+
tree_definition = ResqueJobsTree::Factory.create :tree1 do
|
183
|
+
root :job1 do
|
184
|
+
perform { raise 'should not arrive here' }
|
185
|
+
childs { [ [:job2], [:job3] ] }
|
186
|
+
node :job2, async: true do
|
187
|
+
perform {}
|
188
|
+
end
|
189
|
+
node :job3, continue_on_failure: true do
|
190
|
+
perform { raise ExpectedException, 'an expected failure' }
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
tree_definition.spawn([]).launch
|
195
|
+
assert_raises ExpectedException do # job3 exception
|
196
|
+
run_resque_workers tree_definition.name
|
197
|
+
end
|
198
|
+
assert_equal ["ResqueJobsTree:Node:[\"tree1\",\"job2\"]"],
|
199
|
+
Resque.redis.smembers("ResqueJobsTree:Node:[\"tree1\",\"job1\"]:childs")
|
200
|
+
end
|
201
|
+
|
202
|
+
private
|
203
|
+
|
204
|
+
def assert_redis_empty
|
205
|
+
assert_equal [], Resque.keys
|
206
|
+
end
|
207
|
+
|
208
|
+
# Inline mode is messy when dealing with on failure callbacks.
|
209
|
+
def run_resque_workers queue_name
|
210
|
+
Resque::Job.reserve(queue_name).perform
|
211
|
+
redis.srem 'queues', queue_name
|
212
|
+
end
|
213
|
+
|
214
|
+
end
|