sidekiq-batch 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/lib/sidekiq/batch.rb +95 -28
- data/lib/sidekiq/batch/callback.rb +64 -32
- data/lib/sidekiq/batch/extension/worker.rb +2 -2
- data/lib/sidekiq/batch/middleware.rb +4 -4
- data/lib/sidekiq/batch/status.rb +3 -1
- data/lib/sidekiq/batch/version.rb +1 -1
- metadata +3 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: '0811ae31d4850eee031831062fbe1aca229d9255f28ee85641fff02f22e4ff49'
|
4
|
+
data.tar.gz: d37febab0ff1087d3b3966f0beae1155246b3535f703e3bd62e9ae04fd83a553
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dafce26855e716909db76ed31c04f31381d2ca4ab9d9d22b4d85e95b4deda62cf5438d562cce5344596172bbc912dc5995a5b5c435cceadca78004bf8db0a7b3
|
7
|
+
data.tar.gz: e25c54027984ff63d2242037de20795c8e223c4b295504678aa5a4a5f339568f44e290458f1a732d25f4c692dd6b3961256525bed66c19e20b95e0de936bf3be
|
data/lib/sidekiq/batch.rb
CHANGED
@@ -33,6 +33,11 @@ module Sidekiq
|
|
33
33
|
persist_bid_attr('callback_queue', callback_queue)
|
34
34
|
end
|
35
35
|
|
36
|
+
def callback_batch=(callback_batch)
|
37
|
+
@callback_batch = callback_batch
|
38
|
+
persist_bid_attr('callback_batch', callback_batch)
|
39
|
+
end
|
40
|
+
|
36
41
|
def on(event, callback, options = {})
|
37
42
|
return unless %w(success complete).include?(event.to_s)
|
38
43
|
callback_key = "#{@bidkey}-callbacks-#{event}"
|
@@ -54,7 +59,7 @@ module Sidekiq
|
|
54
59
|
|
55
60
|
begin
|
56
61
|
if !@existing && !@initialized
|
57
|
-
parent_bid = Thread.current[:
|
62
|
+
parent_bid = Thread.current[:batch].bid if Thread.current[:batch]
|
58
63
|
|
59
64
|
Sidekiq.redis do |r|
|
60
65
|
r.multi do
|
@@ -70,11 +75,11 @@ module Sidekiq
|
|
70
75
|
@ready_to_queue = []
|
71
76
|
|
72
77
|
begin
|
73
|
-
parent = Thread.current[:
|
74
|
-
Thread.current[:
|
78
|
+
parent = Thread.current[:batch]
|
79
|
+
Thread.current[:batch] = self
|
75
80
|
yield
|
76
81
|
ensure
|
77
|
-
Thread.current[:
|
82
|
+
Thread.current[:batch] = parent
|
78
83
|
end
|
79
84
|
|
80
85
|
return [] if @ready_to_queue.size == 0
|
@@ -83,6 +88,7 @@ module Sidekiq
|
|
83
88
|
r.multi do
|
84
89
|
if parent_bid
|
85
90
|
r.hincrby("BID-#{parent_bid}", "children", 1)
|
91
|
+
r.hincrby("BID-#{parent_bid}", "total", @ready_to_queue.size)
|
86
92
|
r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
|
87
93
|
end
|
88
94
|
|
@@ -141,7 +147,7 @@ module Sidekiq
|
|
141
147
|
|
142
148
|
class << self
|
143
149
|
def process_failed_job(bid, jid)
|
144
|
-
_, pending, failed, children, complete = Sidekiq.redis do |r|
|
150
|
+
_, pending, failed, children, complete, parent_bid = Sidekiq.redis do |r|
|
145
151
|
r.multi do
|
146
152
|
r.sadd("BID-#{bid}-failed", jid)
|
147
153
|
|
@@ -149,12 +155,26 @@ module Sidekiq
|
|
149
155
|
r.scard("BID-#{bid}-failed")
|
150
156
|
r.hincrby("BID-#{bid}", "children", 0)
|
151
157
|
r.scard("BID-#{bid}-complete")
|
158
|
+
r.hget("BID-#{bid}", "parent_bid")
|
152
159
|
|
153
160
|
r.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
|
154
161
|
end
|
155
162
|
end
|
156
163
|
|
157
|
-
|
164
|
+
# if the batch failed, and has a parent, update the parent to show one pending and failed job
|
165
|
+
if parent_bid
|
166
|
+
Sidekiq.redis do |r|
|
167
|
+
r.multi do
|
168
|
+
r.hincrby("BID-#{parent_bid}", "pending", 1)
|
169
|
+
r.sadd("BID-#{parent_bid}-failed", jid)
|
170
|
+
r.expire("BID-#{parent_bid}-failed", BID_EXPIRE_TTL)
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
if pending.to_i == failed.to_i && children == complete
|
176
|
+
enqueue_callbacks(:complete, bid)
|
177
|
+
end
|
158
178
|
end
|
159
179
|
|
160
180
|
def process_successful_job(bid, jid)
|
@@ -174,49 +194,96 @@ module Sidekiq
|
|
174
194
|
end
|
175
195
|
end
|
176
196
|
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
197
|
+
# if complete or successfull call complete callback (the complete callback may then call successful)
|
198
|
+
if (pending.to_i == failed.to_i && children == complete) || (pending.to_i.zero? && children == success)
|
199
|
+
enqueue_callbacks(:complete, bid)
|
200
|
+
end
|
181
201
|
end
|
182
202
|
|
183
203
|
def enqueue_callbacks(event, bid)
|
184
204
|
batch_key = "BID-#{bid}"
|
185
205
|
callback_key = "#{batch_key}-callbacks-#{event}"
|
186
|
-
|
206
|
+
already_processed, _, callbacks, queue, parent_bid, callback_batch = Sidekiq.redis do |r|
|
187
207
|
r.multi do
|
188
208
|
r.hget(batch_key, event)
|
189
209
|
r.hset(batch_key, event, true)
|
190
210
|
r.smembers(callback_key)
|
191
211
|
r.hget(batch_key, "callback_queue")
|
192
212
|
r.hget(batch_key, "parent_bid")
|
213
|
+
r.hget(batch_key, "callback_batch")
|
193
214
|
end
|
194
215
|
end
|
195
|
-
return if needed == 'true'
|
196
216
|
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
217
|
+
return if already_processed == 'true'
|
218
|
+
|
219
|
+
queue ||= "default"
|
220
|
+
parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid # Basically parent_bid.blank?
|
221
|
+
callback_args = callbacks.reduce([]) do |memo, jcb|
|
222
|
+
cb = Sidekiq.load_json(jcb)
|
223
|
+
memo << [cb['callback'], event, cb['opts'], bid, parent_bid]
|
224
|
+
end
|
225
|
+
|
226
|
+
opts = {"bid" => bid, "event" => event}
|
227
|
+
|
228
|
+
# Run callback batch finalize synchronously
|
229
|
+
if callback_batch
|
230
|
+
# Extract opts from cb_args or use current
|
231
|
+
# Pass in stored event as callback finalize is processed on complete event
|
232
|
+
cb_opts = callback_args.first&.at(2) || opts
|
233
|
+
|
234
|
+
Sidekiq.logger.debug {"Run callback batch bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
|
235
|
+
# Finalize now
|
236
|
+
finalizer = Sidekiq::Batch::Callback::Finalize.new
|
237
|
+
status = Status.new bid
|
238
|
+
finalizer.dispatch(status, cb_opts)
|
239
|
+
|
240
|
+
return
|
241
|
+
end
|
242
|
+
|
243
|
+
Sidekiq.logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
|
244
|
+
|
245
|
+
if callback_args.empty?
|
246
|
+
# Finalize now
|
247
|
+
finalizer = Sidekiq::Batch::Callback::Finalize.new
|
248
|
+
status = Status.new bid
|
249
|
+
finalizer.dispatch(status, opts)
|
250
|
+
else
|
251
|
+
# Otherwise finalize in sub batch complete callback
|
252
|
+
cb_batch = self.new
|
253
|
+
cb_batch.callback_batch = true
|
254
|
+
Sidekiq.logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
|
255
|
+
cb_batch.on(:complete, "Sidekiq::Batch::Callback::Finalize#dispatch", opts)
|
256
|
+
cb_batch.jobs do
|
257
|
+
push_callbacks callback_args, queue
|
258
|
+
end
|
209
259
|
end
|
210
260
|
end
|
211
261
|
|
212
262
|
def cleanup_redis(bid)
|
263
|
+
Sidekiq.logger.debug {"Cleaning redis of batch #{bid}"}
|
213
264
|
Sidekiq.redis do |r|
|
214
|
-
r.del(
|
215
|
-
|
216
|
-
|
217
|
-
|
265
|
+
r.del(
|
266
|
+
"BID-#{bid}",
|
267
|
+
"BID-#{bid}-callbacks-complete",
|
268
|
+
"BID-#{bid}-callbacks-success",
|
269
|
+
"BID-#{bid}-failed",
|
270
|
+
|
271
|
+
"BID-#{bid}-success",
|
272
|
+
"BID-#{bid}-complete",
|
273
|
+
"BID-#{bid}-jids",
|
274
|
+
)
|
218
275
|
end
|
219
276
|
end
|
277
|
+
|
278
|
+
private
|
279
|
+
|
280
|
+
def push_callbacks args, queue
|
281
|
+
Sidekiq::Client.push_bulk(
|
282
|
+
'class' => Sidekiq::Batch::Callback::Worker,
|
283
|
+
'args' => args,
|
284
|
+
'queue' => queue
|
285
|
+
) unless args.empty?
|
286
|
+
end
|
220
287
|
end
|
221
288
|
end
|
222
289
|
end
|
@@ -6,66 +6,98 @@ module Sidekiq
|
|
6
6
|
|
7
7
|
def perform(clazz, event, opts, bid, parent_bid)
|
8
8
|
return unless %w(success complete).include?(event)
|
9
|
-
clazz, method = clazz.split("#") if (clazz.class == String && clazz.include?("#"))
|
9
|
+
clazz, method = clazz.split("#") if (clazz && clazz.class == String && clazz.include?("#"))
|
10
10
|
method = "on_#{event}" if method.nil?
|
11
11
|
status = Sidekiq::Batch::Status.new(bid)
|
12
12
|
|
13
|
-
if object = Object.const_get(clazz)
|
13
|
+
if clazz && object = Object.const_get(clazz)
|
14
14
|
instance = object.new
|
15
15
|
instance.send(method, status, opts) if instance.respond_to?(method)
|
16
16
|
end
|
17
|
-
|
18
|
-
send(event.to_sym, bid, status, parent_bid)
|
19
17
|
end
|
18
|
+
end
|
19
|
+
|
20
|
+
class Finalize
|
21
|
+
def dispatch status, opts
|
22
|
+
bid = opts["bid"]
|
23
|
+
callback_bid = status.bid
|
24
|
+
event = opts["event"].to_sym
|
25
|
+
callback_batch = bid != callback_bid
|
26
|
+
|
27
|
+
Sidekiq.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{callback_batch}"}
|
28
|
+
|
29
|
+
batch_status = Status.new bid
|
30
|
+
send(event, bid, batch_status, batch_status.parent_bid)
|
20
31
|
|
21
32
|
|
33
|
+
# Different events are run in different callback batches
|
34
|
+
Sidekiq::Batch.cleanup_redis callback_bid if callback_batch
|
35
|
+
Sidekiq::Batch.cleanup_redis bid if event == :success
|
36
|
+
end
|
37
|
+
|
22
38
|
def success(bid, status, parent_bid)
|
23
|
-
|
24
|
-
_, _, success, pending, children = Sidekiq.redis do |r|
|
25
|
-
r.multi do
|
26
|
-
r.sadd("BID-#{parent_bid}-success", bid)
|
27
|
-
r.expire("BID-#{parent_bid}-success", Sidekiq::Batch::BID_EXPIRE_TTL)
|
28
|
-
r.scard("BID-#{parent_bid}-success")
|
29
|
-
r.hincrby("BID-#{parent_bid}", "pending", 0)
|
30
|
-
r.hincrby("BID-#{parent_bid}", "children", 0)
|
31
|
-
end
|
32
|
-
end
|
39
|
+
return unless parent_bid
|
33
40
|
|
34
|
-
|
41
|
+
_, _, success, _, complete, pending, children, failure = Sidekiq.redis do |r|
|
42
|
+
r.multi do
|
43
|
+
r.sadd("BID-#{parent_bid}-success", bid)
|
44
|
+
r.expire("BID-#{parent_bid}-success", Sidekiq::Batch::BID_EXPIRE_TTL)
|
45
|
+
r.scard("BID-#{parent_bid}-success")
|
46
|
+
r.sadd("BID-#{parent_bid}-complete", bid)
|
47
|
+
r.scard("BID-#{parent_bid}-complete")
|
48
|
+
r.hincrby("BID-#{parent_bid}", "pending", 0)
|
49
|
+
r.hincrby("BID-#{parent_bid}", "children", 0)
|
50
|
+
r.scard("BID-#{parent_bid}-failed")
|
51
|
+
end
|
35
52
|
end
|
36
|
-
|
37
|
-
|
38
|
-
|
53
|
+
# if job finished successfully and parent batch completed call parent complete callback
|
54
|
+
# Success callback is called after complete callback
|
55
|
+
if complete == children && pending == failure
|
56
|
+
Sidekiq.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
|
57
|
+
Batch.enqueue_callbacks(:complete, parent_bid)
|
39
58
|
end
|
59
|
+
|
40
60
|
end
|
41
61
|
|
42
62
|
def complete(bid, status, parent_bid)
|
43
|
-
|
63
|
+
pending, children, success = Sidekiq.redis do |r|
|
64
|
+
r.multi do
|
65
|
+
r.hincrby("BID-#{bid}", "pending", 0)
|
66
|
+
r.hincrby("BID-#{bid}", "children", 0)
|
67
|
+
r.scard("BID-#{bid}-success")
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
# if we batch was successful run success callback
|
72
|
+
if pending.to_i.zero? && children == success
|
73
|
+
Batch.enqueue_callbacks(:success, bid)
|
74
|
+
|
75
|
+
elsif parent_bid
|
76
|
+
# if batch was not successfull check and see if its parent is complete
|
77
|
+
# if the parent is complete we trigger the complete callback
|
78
|
+
# We don't want to run this if the batch was successfull because the success
|
79
|
+
# callback may add more jobs to the parent batch
|
80
|
+
|
81
|
+
Sidekiq.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
|
44
82
|
_, complete, pending, children, failure = Sidekiq.redis do |r|
|
45
83
|
r.multi do
|
46
84
|
r.sadd("BID-#{parent_bid}-complete", bid)
|
47
85
|
r.scard("BID-#{parent_bid}-complete")
|
48
86
|
r.hincrby("BID-#{parent_bid}", "pending", 0)
|
49
87
|
r.hincrby("BID-#{parent_bid}", "children", 0)
|
50
|
-
r.
|
88
|
+
r.scard("BID-#{parent_bid}-failed")
|
51
89
|
end
|
52
90
|
end
|
53
|
-
|
54
|
-
|
55
|
-
end
|
56
|
-
|
57
|
-
pending, children, success = Sidekiq.redis do |r|
|
58
|
-
r.multi do
|
59
|
-
r.hincrby("BID-#{bid}", "pending", 0)
|
60
|
-
r.hincrby("BID-#{bid}", "children", 0)
|
61
|
-
r.scard("BID-#{bid}-success")
|
91
|
+
if complete == children && pending == failure
|
92
|
+
Batch.enqueue_callbacks(:complete, parent_bid)
|
62
93
|
end
|
63
94
|
end
|
64
|
-
|
65
|
-
Batch.enqueue_callbacks(:success, bid) if pending.to_i.zero? && children == success
|
66
|
-
|
67
95
|
end
|
68
96
|
|
97
|
+
def cleanup_redis bid, callback_bid=nil
|
98
|
+
Sidekiq::Batch.cleanup_redis bid
|
99
|
+
Sidekiq::Batch.cleanup_redis callback_bid if callback_bid
|
100
|
+
end
|
69
101
|
end
|
70
102
|
end
|
71
103
|
end
|
@@ -1,11 +1,11 @@
|
|
1
1
|
module Sidekiq::Batch::Extension
|
2
2
|
module Worker
|
3
3
|
def bid
|
4
|
-
Thread.current[:bid
|
4
|
+
Thread.current[:batch].bid
|
5
5
|
end
|
6
6
|
|
7
7
|
def batch
|
8
|
-
|
8
|
+
Thread.current[:batch]
|
9
9
|
end
|
10
10
|
|
11
11
|
def valid_within_batch?
|
@@ -5,7 +5,7 @@ module Sidekiq
|
|
5
5
|
module Middleware
|
6
6
|
class ClientMiddleware
|
7
7
|
def call(_worker, msg, _queue, _redis_pool = nil)
|
8
|
-
if (batch = Thread.current[:
|
8
|
+
if (batch = Thread.current[:batch])
|
9
9
|
batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
|
10
10
|
end
|
11
11
|
yield
|
@@ -16,15 +16,15 @@ module Sidekiq
|
|
16
16
|
def call(_worker, msg, _queue)
|
17
17
|
if (bid = msg['bid'])
|
18
18
|
begin
|
19
|
-
Thread.current[:
|
19
|
+
Thread.current[:batch] = Sidekiq::Batch.new(bid)
|
20
20
|
yield
|
21
|
-
Thread.current[:
|
21
|
+
Thread.current[:batch] = nil
|
22
22
|
Batch.process_successful_job(bid, msg['jid'])
|
23
23
|
rescue
|
24
24
|
Batch.process_failed_job(bid, msg['jid'])
|
25
25
|
raise
|
26
26
|
ensure
|
27
|
-
Thread.current[:
|
27
|
+
Thread.current[:batch] = nil
|
28
28
|
end
|
29
29
|
else
|
30
30
|
yield
|
data/lib/sidekiq/batch/status.rb
CHANGED
@@ -45,13 +45,15 @@ module Sidekiq
|
|
45
45
|
|
46
46
|
def data
|
47
47
|
{
|
48
|
+
bid: bid,
|
48
49
|
total: total,
|
49
50
|
failures: failures,
|
50
51
|
pending: pending,
|
51
52
|
created_at: created_at,
|
52
53
|
complete: complete?,
|
53
54
|
failure_info: failure_info,
|
54
|
-
parent_bid: parent_bid
|
55
|
+
parent_bid: parent_bid,
|
56
|
+
child_count: child_count
|
55
57
|
}
|
56
58
|
end
|
57
59
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-batch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Marcin Naglik
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-08-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -121,8 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
121
|
- !ruby/object:Gem::Version
|
122
122
|
version: '0'
|
123
123
|
requirements: []
|
124
|
-
|
125
|
-
rubygems_version: 2.5.2.3
|
124
|
+
rubygems_version: 3.0.3
|
126
125
|
signing_key:
|
127
126
|
specification_version: 4
|
128
127
|
summary: Sidekiq Batch Jobs
|