resque_jobs_tree 0.3.4 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/resque_jobs_tree.rb +25 -3
- data/lib/resque_jobs_tree/definitions.rb +15 -0
- data/lib/resque_jobs_tree/definitions/node.rb +77 -0
- data/lib/resque_jobs_tree/definitions/tree.rb +39 -0
- data/lib/resque_jobs_tree/factory.rb +7 -7
- data/lib/resque_jobs_tree/job.rb +13 -31
- data/lib/resque_jobs_tree/node.rb +64 -60
- data/lib/resque_jobs_tree/resources_serializer.rb +2 -3
- data/lib/resque_jobs_tree/storage.rb +5 -112
- data/lib/resque_jobs_tree/storage/node.rb +80 -0
- data/lib/resque_jobs_tree/storage/tree.rb +26 -0
- data/lib/resque_jobs_tree/tree.rb +32 -27
- data/lib/resque_jobs_tree/version.rb +1 -1
- data/test/definitions_test.rb +99 -0
- data/test/factory_test.rb +3 -4
- data/test/job_test.rb +6 -8
- data/test/process_test.rb +214 -0
- data/test/resources_serializer_test.rb +6 -6
- data/test/storage_node_test.rb +117 -0
- data/test/storage_tree_test.rb +27 -0
- data/test/test_helper.rb +12 -8
- data/test/tree_test.rb +14 -109
- metadata +15 -6
- data/test/node_test.rb +0 -188
- data/test/storage_test.rb +0 -74
data/lib/resque_jobs_tree.rb
CHANGED
@@ -1,6 +1,10 @@
|
|
1
1
|
require "resque_jobs_tree/version"
|
2
2
|
require 'resque'
|
3
3
|
|
4
|
+
require 'resque_jobs_tree/storage'
|
5
|
+
require 'resque_jobs_tree/storage/tree'
|
6
|
+
require 'resque_jobs_tree/storage/node'
|
7
|
+
|
4
8
|
require 'resque_jobs_tree/factory'
|
5
9
|
require 'resque_jobs_tree/tree'
|
6
10
|
require 'resque_jobs_tree/node'
|
@@ -8,8 +12,26 @@ require 'resque_jobs_tree/job'
|
|
8
12
|
require 'resque_jobs_tree/resources_serializer'
|
9
13
|
require 'resque_jobs_tree/storage'
|
10
14
|
|
15
|
+
require 'resque_jobs_tree/definitions'
|
16
|
+
require 'resque_jobs_tree/definitions/tree'
|
17
|
+
require 'resque_jobs_tree/definitions/node'
|
18
|
+
|
11
19
|
module ResqueJobsTree
|
12
|
-
|
13
|
-
class
|
14
|
-
class
|
20
|
+
extend self
|
21
|
+
class TreeDefinitionInvalid < Exception ; end
|
22
|
+
class NodeDefinitionInvalid < Exception ; end
|
23
|
+
class JobNotUniq < Exception ; end
|
24
|
+
|
25
|
+
def find name
|
26
|
+
Factory.find name.to_s
|
27
|
+
end
|
28
|
+
|
29
|
+
def launch name, *resources
|
30
|
+
tree_definition = find name
|
31
|
+
tree_definition ? tree_definition.spawn(resources).launch : raise("Can't find tree `#{name}`")
|
32
|
+
end
|
33
|
+
|
34
|
+
def create *resources
|
35
|
+
Factory.create *resources
|
36
|
+
end
|
15
37
|
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
class ResqueJobsTree::Definitions::Node < ResqueJobsTree::Definitions
|
2
|
+
|
3
|
+
attr_accessor :tree, :parent, :name, :node_childs, :options
|
4
|
+
|
5
|
+
def initialize name, tree, parent=nil
|
6
|
+
@tree = tree
|
7
|
+
@name = name.to_s
|
8
|
+
@parent = parent
|
9
|
+
@node_childs = []
|
10
|
+
@options = {}
|
11
|
+
end
|
12
|
+
|
13
|
+
def node name, options={}, &block
|
14
|
+
ResqueJobsTree::Definitions::Node.new(name, tree, self).tap do |node|
|
15
|
+
node.options = options
|
16
|
+
@node_childs << node
|
17
|
+
node.instance_eval(&block) if block_given?
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def spawn resources, parent=nil
|
22
|
+
ResqueJobsTree::Node.new self, resources, parent
|
23
|
+
end
|
24
|
+
|
25
|
+
def childs &block
|
26
|
+
@childs ||= block
|
27
|
+
end
|
28
|
+
|
29
|
+
def perform &block
|
30
|
+
@perform ||= block
|
31
|
+
end
|
32
|
+
|
33
|
+
def leaf?
|
34
|
+
@node_childs.empty?
|
35
|
+
end
|
36
|
+
|
37
|
+
def root?
|
38
|
+
parent.nil?
|
39
|
+
end
|
40
|
+
|
41
|
+
def siblings
|
42
|
+
root? ? [] : (parent.node_childs - [self])
|
43
|
+
end
|
44
|
+
|
45
|
+
def find _name
|
46
|
+
if name == _name.to_s
|
47
|
+
self
|
48
|
+
else
|
49
|
+
node_childs.inject(nil){|result,node| result ||= node.find _name }
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def validate!
|
54
|
+
if (childs.kind_of?(Proc) && node_childs.empty?) || (childs.nil? && !node_childs.empty?)
|
55
|
+
raise ResqueJobsTree::NodeDefinitionInvalid,
|
56
|
+
"node `#{name}` from tree `#{tree.name}` should defines childs and child nodes"
|
57
|
+
end
|
58
|
+
unless perform.kind_of? Proc
|
59
|
+
raise ResqueJobsTree::NodeDefinitionInvalid,
|
60
|
+
"node `#{name}` from tree `#{tree.name}` has no perform block"
|
61
|
+
end
|
62
|
+
if (tree.nodes - [self]).map(&:name).include? name
|
63
|
+
raise ResqueJobsTree::NodeDefinitionInvalid,
|
64
|
+
"node name `#{name}` is already taken in tree `#{tree.name}`"
|
65
|
+
end
|
66
|
+
node_childs.each &:validate!
|
67
|
+
end
|
68
|
+
|
69
|
+
def nodes
|
70
|
+
node_childs+node_childs.map(&:nodes)
|
71
|
+
end
|
72
|
+
|
73
|
+
def inspect
|
74
|
+
"<ResqueJobsTree::Node @name=#{name}>"
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
class ResqueJobsTree::Definitions::Tree < ResqueJobsTree::Definitions
|
2
|
+
|
3
|
+
attr_accessor :name, :root
|
4
|
+
|
5
|
+
def initialize name
|
6
|
+
@name = name
|
7
|
+
end
|
8
|
+
|
9
|
+
def spawn resources
|
10
|
+
ResqueJobsTree::Tree.new self, resources
|
11
|
+
end
|
12
|
+
|
13
|
+
def root name=nil, &block
|
14
|
+
@root ||= Node.new(name, self).tap do |root|
|
15
|
+
root.instance_eval &block
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def find name
|
20
|
+
root.find name.to_s
|
21
|
+
end
|
22
|
+
|
23
|
+
def validate!
|
24
|
+
if @root
|
25
|
+
root.validate!
|
26
|
+
else
|
27
|
+
raise ResqueJobsTree::TreeDefinitionInvalid, "`#{name}` has no root node"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def nodes
|
32
|
+
[root, root.nodes].flatten
|
33
|
+
end
|
34
|
+
|
35
|
+
def inspect
|
36
|
+
"<ResqueJobsTree::Definitions::Tree @name=#{name}>"
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
@@ -3,21 +3,21 @@ module ResqueJobsTree::Factory
|
|
3
3
|
extend self
|
4
4
|
|
5
5
|
def create name, &block
|
6
|
-
|
7
|
-
@trees
|
8
|
-
ResqueJobsTree::Tree.new(name).tap do |tree|
|
9
|
-
@trees << tree
|
6
|
+
name = name.to_s
|
7
|
+
@trees ||= {}
|
8
|
+
ResqueJobsTree::Definitions::Tree.new(name).tap do |tree|
|
10
9
|
tree.instance_eval &block
|
11
10
|
tree.validate!
|
11
|
+
@trees[name] = tree
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
15
15
|
def trees
|
16
|
-
@trees
|
16
|
+
@trees ||= {}
|
17
17
|
end
|
18
18
|
|
19
|
-
def
|
20
|
-
|
19
|
+
def find name
|
20
|
+
trees[name.to_s]
|
21
21
|
end
|
22
22
|
|
23
23
|
end
|
data/lib/resque_jobs_tree/job.rb
CHANGED
@@ -3,45 +3,27 @@ class ResqueJobsTree::Job
|
|
3
3
|
class << self
|
4
4
|
|
5
5
|
def perform *args
|
6
|
-
node
|
7
|
-
node.perform.call resources
|
6
|
+
node(*args).perform
|
8
7
|
end
|
9
8
|
|
10
9
|
protected
|
11
10
|
|
12
|
-
def
|
13
|
-
node
|
14
|
-
if node.root?
|
15
|
-
ResqueJobsTree::Storage.release_launch node.tree, resources
|
16
|
-
else
|
17
|
-
ResqueJobsTree::Storage.remove(node, resources) do
|
18
|
-
parent_job_args = ResqueJobsTree::Storage.parent_job_args node, resources
|
19
|
-
Resque.enqueue_to node.tree.name, ResqueJobsTree::Job, *parent_job_args
|
20
|
-
end
|
21
|
-
end
|
11
|
+
def before_perform_run_callback *args
|
12
|
+
node(*args).before_perform
|
22
13
|
end
|
23
14
|
|
24
|
-
def
|
25
|
-
node
|
26
|
-
if node.options[:continue_on_failure]
|
27
|
-
begin
|
28
|
-
after_perform_enqueue_parent *args
|
29
|
-
ensure
|
30
|
-
ResqueJobsTree::Storage.failure_cleanup node, resources
|
31
|
-
end
|
32
|
-
else
|
33
|
-
ResqueJobsTree::Storage.failure_cleanup node, resources, global: true
|
34
|
-
node.tree.on_failure.call(resources) if node.tree.on_failure.kind_of?(Proc)
|
35
|
-
raise exception
|
36
|
-
end
|
15
|
+
def after_perform_run_callback *args
|
16
|
+
node(*args).after_perform
|
37
17
|
end
|
38
18
|
|
39
|
-
def
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
19
|
+
def on_failure_run_callback exception, *args
|
20
|
+
node(*args).on_failure
|
21
|
+
end
|
22
|
+
|
23
|
+
def node tree_name, job_name, *resources_arguments
|
24
|
+
node_definition = ResqueJobsTree.find(tree_name).find job_name
|
25
|
+
resources = ResqueJobsTree::ResourcesSerializer.instancize resources_arguments
|
26
|
+
node_definition.spawn resources
|
45
27
|
end
|
46
28
|
|
47
29
|
end
|
@@ -1,92 +1,96 @@
|
|
1
1
|
class ResqueJobsTree::Node
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
@
|
9
|
-
@
|
10
|
-
@
|
3
|
+
include ResqueJobsTree::Storage::Node
|
4
|
+
|
5
|
+
attr_reader :resources, :definition, :tree
|
6
|
+
|
7
|
+
def initialize definition, resources, parent=nil, tree=nil
|
8
|
+
@childs = []
|
9
|
+
@definition = definition
|
10
|
+
@resources = resources
|
11
|
+
@parent = parent
|
12
|
+
@tree = tree
|
13
|
+
end
|
14
|
+
|
15
|
+
def enqueue
|
16
|
+
Resque.enqueue_to definition.tree.name, ResqueJobsTree::Job, *argumentize
|
11
17
|
end
|
12
18
|
|
13
|
-
def
|
14
|
-
|
19
|
+
def perform
|
20
|
+
definition.perform.call *resources
|
15
21
|
end
|
16
22
|
|
17
|
-
def
|
18
|
-
|
23
|
+
def before_perform
|
24
|
+
run_callback :before_perform
|
19
25
|
end
|
20
26
|
|
21
|
-
def
|
22
|
-
|
27
|
+
def after_perform
|
28
|
+
run_callback :after_perform
|
29
|
+
if root?
|
30
|
+
tree.finish
|
31
|
+
else
|
32
|
+
lock do
|
33
|
+
parent.enqueue if only_stored_child?
|
34
|
+
unstore
|
35
|
+
end
|
36
|
+
end
|
23
37
|
end
|
24
38
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
39
|
+
def on_failure
|
40
|
+
if definition.options[:continue_on_failure]
|
41
|
+
after_perform
|
42
|
+
else
|
43
|
+
root.tree.on_failure
|
44
|
+
root.cleanup
|
31
45
|
end
|
32
46
|
end
|
33
47
|
|
34
|
-
def
|
35
|
-
|
48
|
+
def tree
|
49
|
+
@tree ||= root? ? definition.tree.spawn(resources) : @parent.tree
|
50
|
+
end
|
51
|
+
|
52
|
+
def name
|
53
|
+
definition.name
|
54
|
+
end
|
55
|
+
|
56
|
+
def leaf?
|
57
|
+
childs.empty?
|
36
58
|
end
|
37
59
|
|
38
60
|
def root?
|
39
|
-
|
61
|
+
definition.root?
|
40
62
|
end
|
41
63
|
|
42
|
-
def
|
43
|
-
|
44
|
-
parent.node_childs - [self]
|
64
|
+
def root
|
65
|
+
@root ||= root? ? self : parent.root
|
45
66
|
end
|
46
67
|
|
47
|
-
def
|
48
|
-
unless
|
49
|
-
|
50
|
-
end
|
51
|
-
if node_childs.empty?
|
52
|
-
@tree.enqueue(name, *resources) unless options[:async]
|
53
|
-
else
|
54
|
-
childs.call(resources).each do |name, *child_resources|
|
55
|
-
find_node_by_name(name).launch child_resources, resources
|
56
|
-
end
|
57
|
-
end
|
68
|
+
def childs
|
69
|
+
return @childs unless @childs.empty?
|
70
|
+
@childs = definition.leaf? ? [] : definition.childs.call(*resources)
|
58
71
|
end
|
59
72
|
|
60
|
-
def
|
61
|
-
|
62
|
-
|
73
|
+
def launch
|
74
|
+
store unless root?
|
75
|
+
if leaf?
|
76
|
+
tree.register_a_leaf self
|
63
77
|
else
|
64
|
-
|
78
|
+
childs.each do |node_name, *resources|
|
79
|
+
node = definition.find(node_name).spawn resources, self
|
80
|
+
node.launch
|
81
|
+
end
|
65
82
|
end
|
66
83
|
end
|
67
84
|
|
68
|
-
def
|
69
|
-
|
70
|
-
raise ResqueJobsTree::NodeInvalid,
|
71
|
-
"node `#{name}` from tree `#{tree.name}` should defines childs and child nodes"
|
72
|
-
end
|
73
|
-
unless perform.kind_of? Proc
|
74
|
-
raise ResqueJobsTree::NodeInvalid,
|
75
|
-
"node `#{name}` from tree `#{tree.name}` has no perform block"
|
76
|
-
end
|
77
|
-
if (tree.nodes - [self]).map(&:name).include? name
|
78
|
-
raise ResqueJobsTree::NodeInvalid,
|
79
|
-
"node name `#{name}` is already taken in tree `#{tree.name}`"
|
80
|
-
end
|
81
|
-
node_childs.each &:validate!
|
85
|
+
def inspect
|
86
|
+
"<ResqueJobsTree::Node @name=#{name} @resources=#{resources}>"
|
82
87
|
end
|
83
88
|
|
84
|
-
|
85
|
-
node_childs+node_childs.map(&:nodes)
|
86
|
-
end
|
89
|
+
private
|
87
90
|
|
88
|
-
def
|
89
|
-
|
91
|
+
def run_callback callback
|
92
|
+
callback_block = definition.send callback
|
93
|
+
callback_block.call(*resources) if callback_block.kind_of? Proc
|
90
94
|
end
|
91
95
|
|
92
96
|
end
|
@@ -1,10 +1,9 @@
|
|
1
1
|
module ResqueJobsTree::ResourcesSerializer
|
2
|
-
|
3
2
|
extend self
|
4
3
|
|
5
4
|
# in: [<Localisation id=1>, :pdf]
|
6
5
|
# out: [[Localisation, 1], :pdf]
|
7
|
-
def
|
6
|
+
def argumentize resources
|
8
7
|
resources.to_a.map do |resource|
|
9
8
|
resource.respond_to?(:id) ? [resource.class.name, resource.id] : resource
|
10
9
|
end
|
@@ -12,7 +11,7 @@ module ResqueJobsTree::ResourcesSerializer
|
|
12
11
|
|
13
12
|
# in: [['Localisation', 1], :pdf]
|
14
13
|
# out: [<Localisation id=1>, :pdf]
|
15
|
-
def
|
14
|
+
def instancize args
|
16
15
|
args.to_a.map do |arg|
|
17
16
|
if arg.kind_of? Array
|
18
17
|
eval(arg[0]).find(arg[1]) rescue arg
|
@@ -1,127 +1,20 @@
|
|
1
|
-
# put expire on every key
|
2
1
|
module ResqueJobsTree::Storage
|
3
|
-
extend self
|
4
2
|
|
5
3
|
PARENTS_KEY = "ResqueJobsTree:Node:Parents"
|
6
4
|
LAUNCHED_TREES = "ResqueJobsTree:Tree:Launched"
|
7
5
|
|
8
|
-
def store node, resources, parent, parent_resources
|
9
|
-
node_key = key node, resources
|
10
|
-
parent_key = key parent, parent_resources
|
11
|
-
redis.hset PARENTS_KEY, node_key, parent_key
|
12
|
-
childs_key = childs_key parent, parent_resources
|
13
|
-
unless redis.sadd childs_key, node_key
|
14
|
-
raise ResqueJobsTree::JobNotUniq,
|
15
|
-
"Job #{parent.name} already has the child #{node.name} with resources: #{resources}"
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
def remove node, resources
|
20
|
-
lock node, resources do
|
21
|
-
siblings_key = siblings_key node, resources
|
22
|
-
node_key = key(node, resources)
|
23
|
-
redis.srem siblings_key, node_key
|
24
|
-
yield if redis.scard(siblings_key) == 0 && block_given?
|
25
|
-
redis.hdel PARENTS_KEY, node_key
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
def failure_cleanup node, resources, options={}
|
30
|
-
cleanup_childs node, resources
|
31
|
-
if node.root?
|
32
|
-
release_launch node.tree, resources
|
33
|
-
else
|
34
|
-
remove_from_siblings node, resources
|
35
|
-
options[:global] ? cleanup_parent(node, resources, options) : remove_parent_key(node, resources)
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
def parent_job_args node, resources
|
40
|
-
args_from_key parent_key(node, resources)
|
41
|
-
end
|
42
|
-
|
43
|
-
def track_launch *tree_info
|
44
|
-
yield if redis.sadd LAUNCHED_TREES, tree_reference(*tree_info)
|
45
|
-
end
|
46
|
-
|
47
|
-
def release_launch *tree_info
|
48
|
-
redis.srem LAUNCHED_TREES, tree_reference(*tree_info)
|
49
|
-
end
|
50
|
-
|
51
6
|
private
|
52
7
|
|
53
|
-
def
|
54
|
-
|
55
|
-
job_args.unshift node.name
|
56
|
-
job_args.unshift node.tree.name
|
57
|
-
"ResqueJobsTree:Node:#{job_args.to_json}"
|
58
|
-
end
|
59
|
-
|
60
|
-
def childs_key node, resources
|
61
|
-
"#{key node, resources}:childs"
|
62
|
-
end
|
63
|
-
|
64
|
-
def parent_key node, resources
|
65
|
-
node_key = key node, resources
|
66
|
-
redis.hget PARENTS_KEY, node_key
|
67
|
-
end
|
68
|
-
|
69
|
-
def siblings_key node, resources
|
70
|
-
"#{parent_key node, resources}:childs"
|
8
|
+
def serialize
|
9
|
+
argumentize.to_json
|
71
10
|
end
|
72
11
|
|
73
|
-
def
|
74
|
-
|
75
|
-
end
|
76
|
-
|
77
|
-
def lock *node_info
|
78
|
-
key = parent_lock_key *node_info
|
79
|
-
while !redis.setnx(key, 'locked')
|
80
|
-
sleep 0.05 # 50 ms
|
81
|
-
end
|
82
|
-
yield
|
83
|
-
ensure
|
84
|
-
redis.del key
|
85
|
-
end
|
86
|
-
|
87
|
-
def cleanup_childs *node_info
|
88
|
-
key = childs_key *node_info
|
89
|
-
redis.smembers(key).each do |child_key|
|
90
|
-
failure_cleanup *node_info_from_key(child_key)
|
91
|
-
end
|
92
|
-
redis.del key
|
93
|
-
end
|
94
|
-
|
95
|
-
def cleanup_parent node, resources, options
|
96
|
-
parent, parent_resources = node_info_from_key(parent_key(node, resources))
|
97
|
-
remove_parent_key node, resources
|
98
|
-
failure_cleanup parent, parent_resources, options
|
99
|
-
end
|
100
|
-
|
101
|
-
def remove_parent_key *node_info
|
102
|
-
redis.hdel PARENTS_KEY, key(*node_info)
|
103
|
-
end
|
104
|
-
|
105
|
-
def remove_from_siblings *node_info
|
106
|
-
redis.srem siblings_key(*node_info), key(*node_info)
|
107
|
-
end
|
108
|
-
|
109
|
-
def args_from_key key
|
110
|
-
JSON.load key.gsub(/ResqueJobsTree:Node:/, '')
|
111
|
-
end
|
112
|
-
|
113
|
-
def node_info_from_key key
|
114
|
-
tree_name, node_name, *resources = args_from_key(key)
|
115
|
-
node = ResqueJobsTree::Factory.find_tree_by_name(tree_name).find_node_by_name(node_name)
|
116
|
-
[node, resources]
|
12
|
+
def argumentize
|
13
|
+
main_arguments + ResqueJobsTree::ResourcesSerializer.argumentize(resources)
|
117
14
|
end
|
118
15
|
|
119
16
|
def redis
|
120
17
|
Resque.redis
|
121
18
|
end
|
122
|
-
|
123
|
-
def tree_reference tree, resources
|
124
|
-
[tree.name, ResqueJobsTree::ResourcesSerializer.to_args(resources)].to_json
|
125
|
-
end
|
126
|
-
|
19
|
+
|
127
20
|
end
|