sidekiq-superworker 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/app/models/sidekiq/superworker/subjob.rb +1 -1
- data/lib/sidekiq/superworker/dsl_hash.rb +27 -33
- data/lib/sidekiq/superworker/dsl_parser.rb +50 -27
- data/lib/sidekiq/superworker/version.rb +1 -1
- data/lib/sidekiq/superworker/worker.rb +10 -3
- data/lib/sidekiq/superworker/worker_class.rb +1 -1
- metadata +2 -2
@@ -3,7 +3,7 @@ module Sidekiq
|
|
3
3
|
class Subjob < ActiveRecord::Base
|
4
4
|
attr_accessible :jid, :subjob_id, :superjob_id, :parent_id, :children_ids, :next_id,
|
5
5
|
:subworker_class, :superworker_class, :arg_keys, :arg_values, :status, :descendants_are_complete,
|
6
|
-
:meta if ActiveRecord::VERSION::MAJOR < 4
|
6
|
+
:meta if ActiveRecord::VERSION::MAJOR < 4 || ActiveRecord.constants.include?(:MassAssignmentSecurity)
|
7
7
|
|
8
8
|
serialize :arg_keys
|
9
9
|
serialize :arg_values
|
@@ -1,33 +1,27 @@
|
|
1
1
|
module Sidekiq
|
2
2
|
module Superworker
|
3
3
|
class DSLHash
|
4
|
-
attr_accessor :record_id
|
4
|
+
attr_accessor :record_id
|
5
5
|
|
6
|
-
def initialize
|
7
|
-
|
8
|
-
end
|
9
|
-
|
10
|
-
def nested_hash_to_records(nested_hash, args)
|
11
|
-
reset
|
6
|
+
def initialize(hash, args={})
|
7
|
+
@hash = hash
|
12
8
|
@args = args
|
13
|
-
nested_hash_to_records_recursive(nested_hash)
|
14
9
|
end
|
15
10
|
|
16
|
-
def
|
17
|
-
|
18
|
-
@
|
19
|
-
rewrite_ids_of_nested_hash_recursive(nested_hash)
|
11
|
+
def rewrite_record_ids(first_record_id)
|
12
|
+
@record_id = first_record_id - 1
|
13
|
+
rewrite_ids_of_nested_hash(@hash)
|
20
14
|
end
|
21
15
|
|
22
|
-
|
23
|
-
|
24
|
-
def reset
|
25
|
-
@args = {}
|
26
|
-
@records = {}
|
16
|
+
def to_records
|
27
17
|
@record_id = 1
|
18
|
+
@records = {}
|
19
|
+
nested_hash_to_records(@hash)
|
28
20
|
end
|
29
21
|
|
30
|
-
|
22
|
+
private
|
23
|
+
|
24
|
+
def nested_hash_to_records(nested_hash, options={})
|
31
25
|
return @records if nested_hash.blank?
|
32
26
|
|
33
27
|
defaults = {
|
@@ -70,24 +64,11 @@ module Sidekiq
|
|
70
64
|
@records[parent_id][:children_ids] << id
|
71
65
|
end
|
72
66
|
|
73
|
-
|
67
|
+
nested_hash_to_records(value[:children], parent_id: id, scoped_args: options[:scoped_args]) if value[:children] && value[:subworker_class] != :batch
|
74
68
|
end
|
75
69
|
@records
|
76
70
|
end
|
77
71
|
|
78
|
-
def rewrite_ids_of_nested_hash_recursive(nested_hash)
|
79
|
-
new_hash = {}
|
80
|
-
nested_hash.each do |old_record_id, record|
|
81
|
-
@record_id += 1
|
82
|
-
parent_record_id = @record_id
|
83
|
-
new_hash[parent_record_id] = record
|
84
|
-
if record[:children]
|
85
|
-
new_hash[parent_record_id][:children] = rewrite_ids_of_nested_hash_recursive(record[:children])
|
86
|
-
end
|
87
|
-
end
|
88
|
-
new_hash
|
89
|
-
end
|
90
|
-
|
91
72
|
def children_ids_for_batch(subjobs, batch_keys_to_iteration_keys)
|
92
73
|
iteration_keys = batch_keys_to_iteration_keys.values
|
93
74
|
batch_iteration_arg_value_arrays = get_batch_iteration_arg_value_arrays(batch_keys_to_iteration_keys)
|
@@ -125,7 +106,7 @@ module Sidekiq
|
|
125
106
|
@records[subjob_id] = subjob
|
126
107
|
@records[last_subjob_id][:next_id] = subjob_id if last_subjob_id
|
127
108
|
last_subjob_id = subjob_id
|
128
|
-
|
109
|
+
nested_hash_to_records(children, parent_id: subjob_id, scoped_args: iteration_args)
|
129
110
|
end
|
130
111
|
|
131
112
|
children_ids << batch_child_id
|
@@ -145,6 +126,19 @@ module Sidekiq
|
|
145
126
|
batch_values = first_batch_value.zip(*batch_values)
|
146
127
|
batch_values
|
147
128
|
end
|
129
|
+
|
130
|
+
def rewrite_ids_of_nested_hash(nested_hash)
|
131
|
+
new_hash = {}
|
132
|
+
nested_hash.each do |old_record_id, record|
|
133
|
+
@record_id += 1
|
134
|
+
parent_record_id = @record_id
|
135
|
+
new_hash[parent_record_id] = record
|
136
|
+
if record[:children]
|
137
|
+
new_hash[parent_record_id][:children] = rewrite_ids_of_nested_hash(record[:children])
|
138
|
+
end
|
139
|
+
end
|
140
|
+
new_hash
|
141
|
+
end
|
148
142
|
end
|
149
143
|
end
|
150
144
|
end
|
@@ -1,49 +1,51 @@
|
|
1
1
|
module Sidekiq
|
2
2
|
module Superworker
|
3
3
|
class DSLParser
|
4
|
-
|
4
|
+
MODULE_SEPARATOR = '__'
|
5
|
+
|
6
|
+
def parse(block)
|
5
7
|
@dsl_evaluator = DSLEvaluator.new
|
6
|
-
@dsl_hash = DSLHash.new
|
7
8
|
@record_id = 0
|
8
9
|
nested_hash = block_to_nested_hash(block)
|
9
|
-
|
10
|
+
DSLHash.new(nested_hash).rewrite_record_ids(1)
|
10
11
|
end
|
11
12
|
|
12
|
-
def
|
13
|
+
def block_to_nested_hash(block)
|
13
14
|
fiber = Fiber.new do
|
14
15
|
@dsl_evaluator.instance_eval(&block)
|
15
16
|
end
|
16
17
|
|
17
18
|
nested_hash = {}
|
18
19
|
while (method_result = fiber.resume)
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
if method == :batch
|
24
|
-
nested_hash[@record_id] = { subworker_class: method, arg_keys: args, children: block_to_nested_hash(block) }
|
25
|
-
else
|
26
|
-
nested_hash[@record_id] = { subworker_class: method, arg_keys: args, children: block_to_nested_hash(block) }
|
27
|
-
end
|
28
|
-
else
|
29
|
-
nested_hash[@record_id] = { subworker_class: method, arg_keys: args }
|
30
|
-
end
|
20
|
+
add_method_result_to_nested_hash(nested_hash, method_result)
|
21
|
+
end
|
22
|
+
nested_hash
|
23
|
+
end
|
31
24
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
25
|
+
def add_method_result_to_nested_hash(nested_hash, method_result)
|
26
|
+
method, args, block = method_result
|
27
|
+
subworker_type = method_to_subworker_type(method)
|
28
|
+
@record_id += 1
|
29
|
+
method_record_id = @record_id
|
30
|
+
|
31
|
+
record = { subworker_class: subworker_type, arg_keys: args }
|
32
|
+
record[:children] = block_to_nested_hash(block) if block
|
33
|
+
nested_hash[method_record_id] = record
|
34
|
+
|
35
|
+
# For superworkers nested within other superworkers, we'll take the subworkers' nested_hash,
|
36
|
+
# adjust their ids to fit in with our current @record_id value, and add them into the tree.
|
37
|
+
unless [:parallel, :batch].include?(subworker_type)
|
38
|
+
subworker_class = subworker_type.to_s.constantize
|
39
|
+
if subworker_class.respond_to?(:is_a_superworker?) && subworker_class.is_a_superworker?
|
40
|
+
dsl_hash = DSLHash.new(subworker_class.nested_hash)
|
41
|
+
children = dsl_hash.rewrite_record_ids(@record_id + 1)
|
42
|
+
set_children_for_record_id(nested_hash, method_record_id, children)
|
43
|
+
@record_id = dsl_hash.record_id
|
41
44
|
end
|
42
45
|
end
|
43
|
-
nested_hash
|
44
46
|
end
|
45
47
|
|
46
|
-
def
|
48
|
+
def set_children_for_record_id(nested_hash, parent_record_id, children)
|
47
49
|
nested_hash.each do |record_id, value|
|
48
50
|
if record_id == parent_record_id
|
49
51
|
if nested_hash[record_id][:children].present?
|
@@ -57,6 +59,27 @@ module Sidekiq
|
|
57
59
|
end
|
58
60
|
end
|
59
61
|
end
|
62
|
+
|
63
|
+
def method_to_subworker_type(method)
|
64
|
+
method = method.to_s
|
65
|
+
if method.include?(MODULE_SEPARATOR)
|
66
|
+
method_pieces = method.split(MODULE_SEPARATOR)
|
67
|
+
class_name = method_pieces.pop
|
68
|
+
module_name = method_pieces.join('::')
|
69
|
+
else
|
70
|
+
module_name = nil
|
71
|
+
class_name = method
|
72
|
+
end
|
73
|
+
|
74
|
+
return class_name.to_sym if module_name.nil?
|
75
|
+
|
76
|
+
begin
|
77
|
+
namespaced_class = [module_name, class_name].compact.join('::').constantize
|
78
|
+
rescue NameError
|
79
|
+
namespaced_class = [module_name, class_name].compact.join(MODULE_SEPARATOR).constantize
|
80
|
+
end
|
81
|
+
namespaced_class.to_s
|
82
|
+
end
|
60
83
|
end
|
61
84
|
end
|
62
85
|
end
|
@@ -3,7 +3,7 @@ module Sidekiq
|
|
3
3
|
class Worker
|
4
4
|
def self.create(*args, &block)
|
5
5
|
class_name = args.shift.to_sym
|
6
|
-
nested_hash = DSLParser.parse(block)
|
6
|
+
nested_hash = DSLParser.new.parse(block)
|
7
7
|
create_class(class_name, args, nested_hash)
|
8
8
|
end
|
9
9
|
|
@@ -14,10 +14,17 @@ module Sidekiq
|
|
14
14
|
@class_name = class_name
|
15
15
|
@arg_keys = arg_keys
|
16
16
|
@nested_hash = nested_hash
|
17
|
-
@dsl_hash = DSLHash.new
|
18
17
|
end
|
19
18
|
|
20
|
-
|
19
|
+
class_components = class_name.to_s.split('::')
|
20
|
+
class_name = class_components.pop
|
21
|
+
module_name = class_components.join('::')
|
22
|
+
|
23
|
+
if module_name.empty?
|
24
|
+
Object.const_set(class_name, klass)
|
25
|
+
else
|
26
|
+
module_name.constantize.const_set(class_name, klass)
|
27
|
+
end
|
21
28
|
end
|
22
29
|
end
|
23
30
|
end
|
@@ -33,7 +33,7 @@ module Sidekiq
|
|
33
33
|
end
|
34
34
|
|
35
35
|
def create_subjobs(arg_values, options={})
|
36
|
-
records =
|
36
|
+
records = DSLHash.new(@nested_hash, @args).to_records
|
37
37
|
records = records.collect do |id, record|
|
38
38
|
record[:status] = 'initialized'
|
39
39
|
record[:superjob_id] = @superjob_id
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-superworker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-
|
12
|
+
date: 2014-03-26 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: sidekiq
|