resque_jobs_tree 0.3.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,80 @@
1
+ module ResqueJobsTree::Storage::Node
2
+ include ResqueJobsTree::Storage
3
+
4
+ def store
5
+ raise 'Can\'t store a root node' if root?
6
+ redis.hset PARENTS_KEY, key, parent.key
7
+ unless redis.sadd parent.childs_key, key
8
+ raise ResqueJobsTree::JobNotUniq,
9
+ "Job #{parent.name} already has the child #{name} with resources: #{resources}"
10
+ end
11
+ end
12
+
13
+ def unstore
14
+ redis.srem parent.childs_key, key
15
+ redis.hdel PARENTS_KEY, key
16
+ end
17
+
18
+ def cleanup
19
+ unless definition.leaf?
20
+ stored_childs.each &:cleanup
21
+ redis.del childs_key
22
+ end
23
+ redis.hdel PARENTS_KEY, key
24
+ tree.unstore if root?
25
+ end
26
+
27
+ def childs_key
28
+ "#{key}:childs"
29
+ end
30
+
31
+ def key
32
+ "ResqueJobsTree:Node:#{serialize}"
33
+ end
34
+
35
+ def only_stored_child?
36
+ (redis.smembers(parent.childs_key) - [key]).empty?
37
+ end
38
+
39
+ def stored_childs
40
+ redis.smembers(childs_key).map do |_key|
41
+ node_name, _resources = node_info_from_key _key
42
+ definition.find(node_name).spawn _resources
43
+ end
44
+ end
45
+
46
+ def parent
47
+ @parent ||= definition.parent.spawn node_info_from_key(parent_key).last
48
+ end
49
+
50
+ def lock_key
51
+ "#{key}:lock"
52
+ end
53
+
54
+ private
55
+
56
+ def lock
57
+ _key = parent.lock_key
58
+ while !redis.setnx(_key, 'locked')
59
+ sleep 0.05 # 50 ms
60
+ end
61
+ yield
62
+ ensure
63
+ redis.del _key
64
+ end
65
+
66
+ def parent_key
67
+ redis.hget PARENTS_KEY, key
68
+ end
69
+
70
+ def node_info_from_key _key
71
+ tree_name, node_name, *resources_arguments = JSON.load(_key.gsub /ResqueJobsTree:Node:/, '')
72
+ _resources = ResqueJobsTree::ResourcesSerializer.instancize(resources_arguments)
73
+ [node_name, _resources]
74
+ end
75
+
76
+ def main_arguments
77
+ [definition.tree.name, name]
78
+ end
79
+
80
+ end
@@ -0,0 +1,26 @@
1
+ module ResqueJobsTree::Storage::Tree
2
+ include ResqueJobsTree::Storage
3
+
4
+ def store
5
+ redis.sadd LAUNCHED_TREES, key
6
+ end
7
+
8
+ def unstore
9
+ redis.srem LAUNCHED_TREES, key
10
+ end
11
+
12
+ def key
13
+ "ResqueJobsTree:Tree:#{serialize}"
14
+ end
15
+
16
+ def uniq?
17
+ !redis.sismember LAUNCHED_TREES, key
18
+ end
19
+
20
+ private
21
+
22
+ def main_arguments
23
+ [name]
24
+ end
25
+
26
+ end
@@ -1,55 +1,60 @@
1
1
  class ResqueJobsTree::Tree
2
2
 
3
- attr_accessor :name
4
- attr_reader :jobs
3
+ include ResqueJobsTree::Storage::Tree
5
4
 
6
- def initialize name
7
- @name = name.to_s
8
- @jobs = []
9
- end
5
+ attr_reader :definition, :resources, :leaves
10
6
 
11
- def root name=nil, &block
12
- @root ||= ResqueJobsTree::Node.new(name, self).tap do |root|
13
- root.instance_eval &block
14
- end
7
+ def initialize definition, resources
8
+ @definition = definition
9
+ @resources = resources
10
+ @leaves = []
15
11
  end
16
12
 
17
- def on_failure &block
18
- @on_failure ||= block
13
+ def name
14
+ @definition.name
19
15
  end
20
16
 
21
- def launch *resources
22
- ResqueJobsTree::Storage.track_launch self, resources do
23
- @root.launch resources
17
+ def launch
18
+ if uniq?
19
+ before_perform
20
+ store
21
+ root.launch
24
22
  enqueue_leaves_jobs
25
23
  end
26
24
  end
27
25
 
28
- def enqueue job_name, *resources
29
- @jobs << ([name, job_name] + ResqueJobsTree::ResourcesSerializer.to_args(resources))
26
+ %w(before_perform after_perform on_failure).each do |callback|
27
+ class_eval %Q{def #{callback} ; run_callback :#{callback} ; end}
30
28
  end
31
29
 
32
- def find_node_by_name name
33
- root.find_node_by_name name.to_s
30
+ def root
31
+ @root ||= ResqueJobsTree::Node.new(definition.root, resources, nil, self)
34
32
  end
35
33
 
36
- def validate!
37
- raise(ResqueJobsTree::TreeInvalid, "`#{name}` has no root node") unless @root
38
- root.validate!
34
+ def register_a_leaf node
35
+ @leaves << node
39
36
  end
40
37
 
41
- def nodes
42
- [root, root.nodes].flatten
38
+ def inspect
39
+ "<ResqueJobsTree::Tree @name=#{name} @resources=#{resources} >"
43
40
  end
44
41
 
45
- def inspect
46
- "<ResqueJobsTree::Tree @name=#{name}>"
42
+ def finish
43
+ after_perform
44
+ unstore
47
45
  end
48
46
 
49
47
  private
50
48
 
51
49
  def enqueue_leaves_jobs
52
- @jobs.each{ |job_args| Resque.enqueue_to name, ResqueJobsTree::Job, *job_args }
50
+ @leaves.each do |leaf|
51
+ leaf.enqueue unless leaf.definition.options[:async]
52
+ end
53
+ end
54
+
55
+ def run_callback callback
56
+ callback = definition.send(callback)
57
+ callback.call(*resources) if callback.kind_of? Proc
53
58
  end
54
59
 
55
60
  end
@@ -1,3 +1,3 @@
1
1
  module ResqueJobsTree
2
- VERSION = "0.3.4"
2
+ VERSION = "0.4.0"
3
3
  end
@@ -0,0 +1,99 @@
1
+ require 'test_helper'
2
+
3
+ class DefinitionsTest < MiniTest::Unit::TestCase
4
+
5
+ def setup
6
+ @tree = ResqueJobsTree::Definitions::Tree.new :tree1
7
+ @root = ResqueJobsTree::Definitions::Node.new :node1, @tree
8
+ @tree.root = @root
9
+ @leaf = ResqueJobsTree::Definitions::Node.new :node2, @tree, @root
10
+ @root.node_childs = [@leaf]
11
+ end
12
+
13
+ def test_perform
14
+ variable = 1
15
+ @root.perform do |n|
16
+ variable = n
17
+ end
18
+ @root.perform.call 2
19
+ assert_equal variable, 2
20
+ end
21
+
22
+ def test_childs
23
+ variable = 1
24
+ @leaf.childs do |n|
25
+ variable = n
26
+ end
27
+ @leaf.childs.call 2
28
+ assert_equal variable, 2
29
+ end
30
+
31
+ def test_node
32
+ node3 = @root.node :node3
33
+ assert_equal @root.find('node3').object_id, node3.object_id
34
+ end
35
+
36
+ def test_leaf
37
+ assert @leaf.leaf?
38
+ assert !@root.leaf?
39
+ end
40
+
41
+ def test_root
42
+ assert @root.root?
43
+ assert !@leaf.root?
44
+ end
45
+
46
+ def test_siblings
47
+ node3 = ResqueJobsTree::Node.new :node3, @tree, @root
48
+ assert @leaf.siblings, [node3]
49
+ end
50
+
51
+ def test_childs_validation
52
+ assert_raises ResqueJobsTree::NodeDefinitionInvalid do
53
+ ResqueJobsTree::Factory.create :tree1 do
54
+ root :job1 do
55
+ perform {}
56
+ childs {}
57
+ end
58
+ end
59
+ end
60
+ ResqueJobsTree::Factory.create :tree1 do
61
+ root :job1 do
62
+ perform {}
63
+ end
64
+ end
65
+ end
66
+
67
+ def test_perform_validation
68
+ assert_raises ResqueJobsTree::NodeDefinitionInvalid do
69
+ ResqueJobsTree::Factory.create :tree1 do
70
+ root :job1 do
71
+ end
72
+ end
73
+ end
74
+ end
75
+
76
+ def test_perform_validation
77
+ assert_raises ResqueJobsTree::NodeDefinitionInvalid do
78
+ ResqueJobsTree::Factory.create :tree1 do
79
+ root :job1 do
80
+ perform {}
81
+ childs {}
82
+ node :job2 do
83
+ perform {}
84
+ end
85
+ node :job2 do
86
+ perform {}
87
+ end
88
+ end
89
+ end
90
+ end
91
+ end
92
+
93
+ def test_options
94
+ options = { async: true }
95
+ @leaf.options = options
96
+ assert_equal options, @tree.find(:node2).options
97
+ end
98
+
99
+ end
data/test/factory_test.rb CHANGED
@@ -7,12 +7,11 @@ class FactoryTest < MiniTest::Unit::TestCase
7
7
  end
8
8
 
9
9
  def test_tree_creation
10
- assert_equal ResqueJobsTree::Factory.trees.first.object_id, @tree.object_id
10
+ assert_equal ResqueJobsTree::Factory.trees.values.first.name, @tree_definition.name
11
11
  end
12
12
 
13
- def test_find_by_name
14
- assert_equal ResqueJobsTree::Factory.find_tree_by_name(@tree.name).object_id,
15
- @tree.object_id
13
+ def test_find
14
+ assert_equal ResqueJobsTree::Factory.find(@tree_definition.name).name, @tree_definition.name
16
15
  end
17
16
 
18
17
  end
data/test/job_test.rb CHANGED
@@ -1,15 +1,13 @@
1
1
  require 'test_helper'
2
2
 
3
3
  class JobTest < MiniTest::Unit::TestCase
4
-
5
- def setup
6
- create_tree
7
- @args = [@tree.name, @tree.find_node_by_name('job1').name, 1, 2, 3]
8
- end
9
4
 
10
- def test_tree_node_and_resources
11
- result = [@tree.find_node_by_name('job1'), [1, 2, 3]]
12
- assert_equal result, ResqueJobsTree::Job.send(:node_and_resources, @args)
5
+ def test_node
6
+ create_tree
7
+ args = [@tree_definition.name, @tree_definition.find('job1').name, 1, 2, 3]
8
+ assert_equal @tree_definition.find(:job1), ResqueJobsTree::Job.send(:node, *args).definition
9
+ assert_equal ResqueJobsTree.find(:tree1), ResqueJobsTree::Job.send(:node, *args).definition.tree
10
+ assert_equal [1, 2, 3], ResqueJobsTree::Job.send(:node, *args).resources
13
11
  end
14
12
 
15
13
  end
@@ -0,0 +1,214 @@
1
+ require 'test_helper'
2
+
3
+ class ProcessTest < MiniTest::Unit::TestCase
4
+
5
+ def test_launch
6
+ create_tree
7
+ resources = [1, 2, 3]
8
+ @tree_definition.spawn(resources).launch
9
+ history = ['tree1 job2']*3+['tree1 job1']
10
+ assert_equal history, redis.lrange('history', 0, -1)
11
+ end
12
+
13
+ def test_launch_with_no_resources
14
+ create_tree
15
+ @tree_definition.spawn([]).launch
16
+ end
17
+
18
+ def test_leaf_failure
19
+ tree_definition = ResqueJobsTree::Factory.create :tree1 do
20
+ root :job1 do
21
+ perform {}
22
+ childs { [:job2] }
23
+ node :job2 do
24
+ perform { raise ExpectedException, 'an expected exception'}
25
+ end
26
+ end
27
+ end
28
+ assert_raises ExpectedException do
29
+ tree_definition.spawn([1, 2, 3]).launch
30
+ end
31
+ assert_redis_empty
32
+ end
33
+
34
+ def test_launch_async
35
+ tree_definition = ResqueJobsTree::Factory.create :tree1 do
36
+ root :job1 do
37
+ perform { raise 'should not arrive here' }
38
+ childs { [:job2] }
39
+ node :job2, async: true do
40
+ perform {}
41
+ end
42
+ end
43
+ end
44
+ tree = tree_definition.spawn [1, 2, 3]
45
+ tree.launch
46
+ end
47
+
48
+ def test_launch_continue_on_failure
49
+ tree_definition = ResqueJobsTree::Factory.create :tree1 do
50
+ root :job1 do
51
+ perform do
52
+ Resque.redis.rpush 'history', 'tree1 job1'
53
+ raise ExpectedException, 'an expected failure'
54
+ end
55
+ childs { [:job2] }
56
+ node :job2, continue_on_failure: true do
57
+ perform do
58
+ Resque.redis.rpush 'history', 'tree1 job2'
59
+ raise 'an expected failure'
60
+ end
61
+ end
62
+ end
63
+ end
64
+ assert_raises ExpectedException do
65
+ tree = tree_definition.spawn [1, 2, 3]
66
+ tree.launch
67
+ end
68
+ assert_equal ['tree1 job2','tree1 job1'], redis.lrange('history', 0, -1)
69
+ redis.del 'history'
70
+ assert_redis_empty
71
+ end
72
+
73
+ def test_root_failure
74
+ Resque.inline = false
75
+ tree_definition = ResqueJobsTree::Factory.create :tree1 do
76
+ root :job1 do
77
+ perform do
78
+ Resque.redis.rpush 'history', 'tree1 job1'
79
+ raise ExpectedException, 'an expected exception'
80
+ end
81
+ childs { [:job2] }
82
+ node :job2 do
83
+ perform do
84
+ Resque.redis.rpush 'history', 'tree1 job2'
85
+ end
86
+ end
87
+ end
88
+ end
89
+ tree = tree_definition.spawn [1,2,3]
90
+ tree.launch
91
+ run_resque_workers tree_definition.name
92
+ assert_raises ExpectedException do
93
+ run_resque_workers tree_definition.name
94
+ end
95
+ assert_equal ['tree1 job2','tree1 job1'], redis.lrange('history', 0, -1)
96
+ redis.del 'history'
97
+ assert_redis_empty
98
+ end
99
+
100
+ def test_store_already_stored
101
+ wrong_tree_definition = ResqueJobsTree::Factory.create :tree1 do
102
+ root :job1 do
103
+ perform {}
104
+ childs do |resources|
105
+ [ [:job2], [:job2] ]
106
+ end
107
+ node :job2 do
108
+ perform {}
109
+ end
110
+ end
111
+ end
112
+ assert_raises ResqueJobsTree::JobNotUniq do
113
+ wrong_tree_definition.spawn([1]).launch
114
+ end
115
+ end
116
+
117
+ def test_on_failure
118
+ tree_definition = ResqueJobsTree::Factory.create :tree1 do
119
+ root :job1 do
120
+ perform { raise }
121
+ end
122
+ on_failure do
123
+ raise ExpectedException, 'called from on_failure block'
124
+ end
125
+ end
126
+ assert_raises ExpectedException do
127
+ tree_definition.spawn([1]).launch
128
+ end
129
+ end
130
+
131
+ def test_tree_with_resource
132
+ tree_definition = ResqueJobsTree::Factory.create :tree1 do
133
+ root :job1 do
134
+ perform do |resource, number|
135
+ raise 'unknown resource' unless resource.kind_of?(Model)
136
+ raise 'unknown resource' unless number.kind_of?(Integer)
137
+ end
138
+ end
139
+ end
140
+ ResqueJobsTree.launch tree_definition.name, Model.new(1), 1
141
+ end
142
+
143
+ def test_nested_tree
144
+ Resque.inline = false
145
+ create_nested_tree
146
+ @tree_definition.spawn([1,2,3]).launch
147
+ assert_raises RuntimeError do # job4 error
148
+ run_resque_workers @tree_definition.name
149
+ end
150
+ Resque.enqueue_to 'tree1', ResqueJobsTree::Job, 'tree1', 'job3'
151
+ run_resque_workers @tree_definition.name # job3 error
152
+ assert_raises RuntimeError do # job2 error
153
+ run_resque_workers @tree_definition.name
154
+ end
155
+ assert_raises ExpectedException do # job1 error
156
+ run_resque_workers @tree_definition.name
157
+ end
158
+ end
159
+
160
+ def test_async_tree
161
+ tree_definition = ResqueJobsTree::Factory.create :tree1 do
162
+ root :job1 do
163
+ perform { raise 'should not arrive here' }
164
+ childs { [ [:job2], [:job3] ] }
165
+ node :job2, async: true do
166
+ perform {}
167
+ end
168
+ node :job3 do
169
+ perform {}
170
+ end
171
+ end
172
+ end
173
+ ResqueJobsTree.launch tree_definition.name
174
+ assert_equal ["ResqueJobsTree:Node:[\"tree1\",\"job2\"]"],
175
+ Resque.redis.smembers("ResqueJobsTree:Node:[\"tree1\",\"job1\"]:childs")
176
+ parents_hash = { 'ResqueJobsTree:Node:["tree1","job2"]'=>'ResqueJobsTree:Node:["tree1","job1"]' }
177
+ assert_equal parents_hash, Resque.redis.hgetall(ResqueJobsTree::Storage::PARENTS_KEY)
178
+ end
179
+
180
+ def test_async_tree_with_fail
181
+ Resque.inline = false
182
+ tree_definition = ResqueJobsTree::Factory.create :tree1 do
183
+ root :job1 do
184
+ perform { raise 'should not arrive here' }
185
+ childs { [ [:job2], [:job3] ] }
186
+ node :job2, async: true do
187
+ perform {}
188
+ end
189
+ node :job3, continue_on_failure: true do
190
+ perform { raise ExpectedException, 'an expected failure' }
191
+ end
192
+ end
193
+ end
194
+ tree_definition.spawn([]).launch
195
+ assert_raises ExpectedException do # job3 exception
196
+ run_resque_workers tree_definition.name
197
+ end
198
+ assert_equal ["ResqueJobsTree:Node:[\"tree1\",\"job2\"]"],
199
+ Resque.redis.smembers("ResqueJobsTree:Node:[\"tree1\",\"job1\"]:childs")
200
+ end
201
+
202
+ private
203
+
204
+ def assert_redis_empty
205
+ assert_equal [], Resque.keys
206
+ end
207
+
208
+ # Inline mode is messy when dealing with on failure callbacks.
209
+ def run_resque_workers queue_name
210
+ Resque::Job.reserve(queue_name).perform
211
+ redis.srem 'queues', queue_name
212
+ end
213
+
214
+ end