sidekiq-batch 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 2d61afa130ecc7d339ec1ff9c2b38ce8870ef0e5
4
- data.tar.gz: a28ac95c3bb93ebd274f99b253e8f7db7d01ffeb
2
+ SHA256:
3
+ metadata.gz: '0811ae31d4850eee031831062fbe1aca229d9255f28ee85641fff02f22e4ff49'
4
+ data.tar.gz: d37febab0ff1087d3b3966f0beae1155246b3535f703e3bd62e9ae04fd83a553
5
5
  SHA512:
6
- metadata.gz: 3ad9d8f6485798ebe314bd561f040b44742bf892d89d044793c77b3eadb9b548c81dfa5884ca35b89eab044159354d5a46fc9ff99864e0df1b9660c69b16d6dc
7
- data.tar.gz: dc66cae976602555b91ce985d6bdc7bf1c5fc3fb9491cc5f14a2debded4799951b1c7eafe7af9940b9986e25d76192969b7140ba58dd647b28a46482627e32f4
6
+ metadata.gz: dafce26855e716909db76ed31c04f31381d2ca4ab9d9d22b4d85e95b4deda62cf5438d562cce5344596172bbc912dc5995a5b5c435cceadca78004bf8db0a7b3
7
+ data.tar.gz: e25c54027984ff63d2242037de20795c8e223c4b295504678aa5a4a5f339568f44e290458f1a732d25f4c692dd6b3961256525bed66c19e20b95e0de936bf3be
@@ -33,6 +33,11 @@ module Sidekiq
33
33
  persist_bid_attr('callback_queue', callback_queue)
34
34
  end
35
35
 
36
+ def callback_batch=(callback_batch)
37
+ @callback_batch = callback_batch
38
+ persist_bid_attr('callback_batch', callback_batch)
39
+ end
40
+
36
41
  def on(event, callback, options = {})
37
42
  return unless %w(success complete).include?(event.to_s)
38
43
  callback_key = "#{@bidkey}-callbacks-#{event}"
@@ -54,7 +59,7 @@ module Sidekiq
54
59
 
55
60
  begin
56
61
  if !@existing && !@initialized
57
- parent_bid = Thread.current[:bid].bid if Thread.current[:bid]
62
+ parent_bid = Thread.current[:batch].bid if Thread.current[:batch]
58
63
 
59
64
  Sidekiq.redis do |r|
60
65
  r.multi do
@@ -70,11 +75,11 @@ module Sidekiq
70
75
  @ready_to_queue = []
71
76
 
72
77
  begin
73
- parent = Thread.current[:bid]
74
- Thread.current[:bid] = self
78
+ parent = Thread.current[:batch]
79
+ Thread.current[:batch] = self
75
80
  yield
76
81
  ensure
77
- Thread.current[:bid] = parent
82
+ Thread.current[:batch] = parent
78
83
  end
79
84
 
80
85
  return [] if @ready_to_queue.size == 0
@@ -83,6 +88,7 @@ module Sidekiq
83
88
  r.multi do
84
89
  if parent_bid
85
90
  r.hincrby("BID-#{parent_bid}", "children", 1)
91
+ r.hincrby("BID-#{parent_bid}", "total", @ready_to_queue.size)
86
92
  r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
87
93
  end
88
94
 
@@ -141,7 +147,7 @@ module Sidekiq
141
147
 
142
148
  class << self
143
149
  def process_failed_job(bid, jid)
144
- _, pending, failed, children, complete = Sidekiq.redis do |r|
150
+ _, pending, failed, children, complete, parent_bid = Sidekiq.redis do |r|
145
151
  r.multi do
146
152
  r.sadd("BID-#{bid}-failed", jid)
147
153
 
@@ -149,12 +155,26 @@ module Sidekiq
149
155
  r.scard("BID-#{bid}-failed")
150
156
  r.hincrby("BID-#{bid}", "children", 0)
151
157
  r.scard("BID-#{bid}-complete")
158
+ r.hget("BID-#{bid}", "parent_bid")
152
159
 
153
160
  r.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
154
161
  end
155
162
  end
156
163
 
157
- enqueue_callbacks(:complete, bid) if pending.to_i == failed.to_i && children == complete
164
+ # if the batch failed, and has a parent, update the parent to show one pending and failed job
165
+ if parent_bid
166
+ Sidekiq.redis do |r|
167
+ r.multi do
168
+ r.hincrby("BID-#{parent_bid}", "pending", 1)
169
+ r.sadd("BID-#{parent_bid}-failed", jid)
170
+ r.expire("BID-#{parent_bid}-failed", BID_EXPIRE_TTL)
171
+ end
172
+ end
173
+ end
174
+
175
+ if pending.to_i == failed.to_i && children == complete
176
+ enqueue_callbacks(:complete, bid)
177
+ end
158
178
  end
159
179
 
160
180
  def process_successful_job(bid, jid)
@@ -174,49 +194,96 @@ module Sidekiq
174
194
  end
175
195
  end
176
196
 
177
- Sidekiq.logger.info "done: #{jid} in batch #{bid}"
178
-
179
- enqueue_callbacks(:complete, bid) if pending.to_i == failed.to_i && children == complete
180
- enqueue_callbacks(:success, bid) if pending.to_i.zero? && children == success
197
+ # if complete or successfull call complete callback (the complete callback may then call successful)
198
+ if (pending.to_i == failed.to_i && children == complete) || (pending.to_i.zero? && children == success)
199
+ enqueue_callbacks(:complete, bid)
200
+ end
181
201
  end
182
202
 
183
203
  def enqueue_callbacks(event, bid)
184
204
  batch_key = "BID-#{bid}"
185
205
  callback_key = "#{batch_key}-callbacks-#{event}"
186
- needed, _, callbacks, queue, parent_bid = Sidekiq.redis do |r|
206
+ already_processed, _, callbacks, queue, parent_bid, callback_batch = Sidekiq.redis do |r|
187
207
  r.multi do
188
208
  r.hget(batch_key, event)
189
209
  r.hset(batch_key, event, true)
190
210
  r.smembers(callback_key)
191
211
  r.hget(batch_key, "callback_queue")
192
212
  r.hget(batch_key, "parent_bid")
213
+ r.hget(batch_key, "callback_batch")
193
214
  end
194
215
  end
195
- return if needed == 'true'
196
216
 
197
- begin
198
- parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid # Basically parent_bid.blank?
199
- Sidekiq::Client.push_bulk(
200
- 'class' => Sidekiq::Batch::Callback::Worker,
201
- 'args' => callbacks.reduce([]) do |memo, jcb|
202
- cb = Sidekiq.load_json(jcb)
203
- memo << [cb['callback'], event, cb['opts'], bid, parent_bid]
204
- end,
205
- 'queue' => queue ||= 'default'
206
- ) unless callbacks.empty?
207
- ensure
208
- cleanup_redis(bid) if event == :success
217
+ return if already_processed == 'true'
218
+
219
+ queue ||= "default"
220
+ parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid # Basically parent_bid.blank?
221
+ callback_args = callbacks.reduce([]) do |memo, jcb|
222
+ cb = Sidekiq.load_json(jcb)
223
+ memo << [cb['callback'], event, cb['opts'], bid, parent_bid]
224
+ end
225
+
226
+ opts = {"bid" => bid, "event" => event}
227
+
228
+ # Run callback batch finalize synchronously
229
+ if callback_batch
230
+ # Extract opts from cb_args or use current
231
+ # Pass in stored event as callback finalize is processed on complete event
232
+ cb_opts = callback_args.first&.at(2) || opts
233
+
234
+ Sidekiq.logger.debug {"Run callback batch bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
235
+ # Finalize now
236
+ finalizer = Sidekiq::Batch::Callback::Finalize.new
237
+ status = Status.new bid
238
+ finalizer.dispatch(status, cb_opts)
239
+
240
+ return
241
+ end
242
+
243
+ Sidekiq.logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
244
+
245
+ if callback_args.empty?
246
+ # Finalize now
247
+ finalizer = Sidekiq::Batch::Callback::Finalize.new
248
+ status = Status.new bid
249
+ finalizer.dispatch(status, opts)
250
+ else
251
+ # Otherwise finalize in sub batch complete callback
252
+ cb_batch = self.new
253
+ cb_batch.callback_batch = true
254
+ Sidekiq.logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
255
+ cb_batch.on(:complete, "Sidekiq::Batch::Callback::Finalize#dispatch", opts)
256
+ cb_batch.jobs do
257
+ push_callbacks callback_args, queue
258
+ end
209
259
  end
210
260
  end
211
261
 
212
262
  def cleanup_redis(bid)
263
+ Sidekiq.logger.debug {"Cleaning redis of batch #{bid}"}
213
264
  Sidekiq.redis do |r|
214
- r.del("BID-#{bid}",
215
- "BID-#{bid}-callbacks-complete",
216
- "BID-#{bid}-callbacks-success",
217
- "BID-#{bid}-failed")
265
+ r.del(
266
+ "BID-#{bid}",
267
+ "BID-#{bid}-callbacks-complete",
268
+ "BID-#{bid}-callbacks-success",
269
+ "BID-#{bid}-failed",
270
+
271
+ "BID-#{bid}-success",
272
+ "BID-#{bid}-complete",
273
+ "BID-#{bid}-jids",
274
+ )
218
275
  end
219
276
  end
277
+
278
+ private
279
+
280
+ def push_callbacks args, queue
281
+ Sidekiq::Client.push_bulk(
282
+ 'class' => Sidekiq::Batch::Callback::Worker,
283
+ 'args' => args,
284
+ 'queue' => queue
285
+ ) unless args.empty?
286
+ end
220
287
  end
221
288
  end
222
289
  end
@@ -6,66 +6,98 @@ module Sidekiq
6
6
 
7
7
  def perform(clazz, event, opts, bid, parent_bid)
8
8
  return unless %w(success complete).include?(event)
9
- clazz, method = clazz.split("#") if (clazz.class == String && clazz.include?("#"))
9
+ clazz, method = clazz.split("#") if (clazz && clazz.class == String && clazz.include?("#"))
10
10
  method = "on_#{event}" if method.nil?
11
11
  status = Sidekiq::Batch::Status.new(bid)
12
12
 
13
- if object = Object.const_get(clazz)
13
+ if clazz && object = Object.const_get(clazz)
14
14
  instance = object.new
15
15
  instance.send(method, status, opts) if instance.respond_to?(method)
16
16
  end
17
-
18
- send(event.to_sym, bid, status, parent_bid)
19
17
  end
18
+ end
19
+
20
+ class Finalize
21
+ def dispatch status, opts
22
+ bid = opts["bid"]
23
+ callback_bid = status.bid
24
+ event = opts["event"].to_sym
25
+ callback_batch = bid != callback_bid
26
+
27
+ Sidekiq.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{callback_batch}"}
28
+
29
+ batch_status = Status.new bid
30
+ send(event, bid, batch_status, batch_status.parent_bid)
20
31
 
21
32
 
33
+ # Different events are run in different callback batches
34
+ Sidekiq::Batch.cleanup_redis callback_bid if callback_batch
35
+ Sidekiq::Batch.cleanup_redis bid if event == :success
36
+ end
37
+
22
38
  def success(bid, status, parent_bid)
23
- if (parent_bid)
24
- _, _, success, pending, children = Sidekiq.redis do |r|
25
- r.multi do
26
- r.sadd("BID-#{parent_bid}-success", bid)
27
- r.expire("BID-#{parent_bid}-success", Sidekiq::Batch::BID_EXPIRE_TTL)
28
- r.scard("BID-#{parent_bid}-success")
29
- r.hincrby("BID-#{parent_bid}", "pending", 0)
30
- r.hincrby("BID-#{parent_bid}", "children", 0)
31
- end
32
- end
39
+ return unless parent_bid
33
40
 
34
- Batch.enqueue_callbacks(:success, parent_bid) if pending.to_i.zero? && children == success
41
+ _, _, success, _, complete, pending, children, failure = Sidekiq.redis do |r|
42
+ r.multi do
43
+ r.sadd("BID-#{parent_bid}-success", bid)
44
+ r.expire("BID-#{parent_bid}-success", Sidekiq::Batch::BID_EXPIRE_TTL)
45
+ r.scard("BID-#{parent_bid}-success")
46
+ r.sadd("BID-#{parent_bid}-complete", bid)
47
+ r.scard("BID-#{parent_bid}-complete")
48
+ r.hincrby("BID-#{parent_bid}", "pending", 0)
49
+ r.hincrby("BID-#{parent_bid}", "children", 0)
50
+ r.scard("BID-#{parent_bid}-failed")
51
+ end
35
52
  end
36
-
37
- Sidekiq.redis do |r|
38
- r.del "BID-#{bid}-success", "BID-#{bid}-complete", "BID-#{bid}-jids", "BID-#{bid}-failed"
53
+ # if job finished successfully and parent batch completed call parent complete callback
54
+ # Success callback is called after complete callback
55
+ if complete == children && pending == failure
56
+ Sidekiq.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
57
+ Batch.enqueue_callbacks(:complete, parent_bid)
39
58
  end
59
+
40
60
  end
41
61
 
42
62
  def complete(bid, status, parent_bid)
43
- if (parent_bid)
63
+ pending, children, success = Sidekiq.redis do |r|
64
+ r.multi do
65
+ r.hincrby("BID-#{bid}", "pending", 0)
66
+ r.hincrby("BID-#{bid}", "children", 0)
67
+ r.scard("BID-#{bid}-success")
68
+ end
69
+ end
70
+
71
+ # if we batch was successful run success callback
72
+ if pending.to_i.zero? && children == success
73
+ Batch.enqueue_callbacks(:success, bid)
74
+
75
+ elsif parent_bid
76
+ # if batch was not successfull check and see if its parent is complete
77
+ # if the parent is complete we trigger the complete callback
78
+ # We don't want to run this if the batch was successfull because the success
79
+ # callback may add more jobs to the parent batch
80
+
81
+ Sidekiq.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
44
82
  _, complete, pending, children, failure = Sidekiq.redis do |r|
45
83
  r.multi do
46
84
  r.sadd("BID-#{parent_bid}-complete", bid)
47
85
  r.scard("BID-#{parent_bid}-complete")
48
86
  r.hincrby("BID-#{parent_bid}", "pending", 0)
49
87
  r.hincrby("BID-#{parent_bid}", "children", 0)
50
- r.hlen("BID-#{parent_bid}-failed")
88
+ r.scard("BID-#{parent_bid}-failed")
51
89
  end
52
90
  end
53
-
54
- Batch.enqueue_callbacks(:complete, parent_bid) if complete == children && pending == failure
55
- end
56
-
57
- pending, children, success = Sidekiq.redis do |r|
58
- r.multi do
59
- r.hincrby("BID-#{bid}", "pending", 0)
60
- r.hincrby("BID-#{bid}", "children", 0)
61
- r.scard("BID-#{bid}-success")
91
+ if complete == children && pending == failure
92
+ Batch.enqueue_callbacks(:complete, parent_bid)
62
93
  end
63
94
  end
64
-
65
- Batch.enqueue_callbacks(:success, bid) if pending.to_i.zero? && children == success
66
-
67
95
  end
68
96
 
97
+ def cleanup_redis bid, callback_bid=nil
98
+ Sidekiq::Batch.cleanup_redis bid
99
+ Sidekiq::Batch.cleanup_redis callback_bid if callback_bid
100
+ end
69
101
  end
70
102
  end
71
103
  end
@@ -1,11 +1,11 @@
1
1
  module Sidekiq::Batch::Extension
2
2
  module Worker
3
3
  def bid
4
- Thread.current[:bid]
4
+ Thread.current[:batch].bid
5
5
  end
6
6
 
7
7
  def batch
8
- Sidekiq::Batch.new(Thread.current[:bid].bid) if Thread.current[:bid]
8
+ Thread.current[:batch]
9
9
  end
10
10
 
11
11
  def valid_within_batch?
@@ -5,7 +5,7 @@ module Sidekiq
5
5
  module Middleware
6
6
  class ClientMiddleware
7
7
  def call(_worker, msg, _queue, _redis_pool = nil)
8
- if (batch = Thread.current[:bid])
8
+ if (batch = Thread.current[:batch])
9
9
  batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
10
10
  end
11
11
  yield
@@ -16,15 +16,15 @@ module Sidekiq
16
16
  def call(_worker, msg, _queue)
17
17
  if (bid = msg['bid'])
18
18
  begin
19
- Thread.current[:bid] = Sidekiq::Batch.new(bid)
19
+ Thread.current[:batch] = Sidekiq::Batch.new(bid)
20
20
  yield
21
- Thread.current[:bid] = nil
21
+ Thread.current[:batch] = nil
22
22
  Batch.process_successful_job(bid, msg['jid'])
23
23
  rescue
24
24
  Batch.process_failed_job(bid, msg['jid'])
25
25
  raise
26
26
  ensure
27
- Thread.current[:bid] = nil
27
+ Thread.current[:batch] = nil
28
28
  end
29
29
  else
30
30
  yield
@@ -45,13 +45,15 @@ module Sidekiq
45
45
 
46
46
  def data
47
47
  {
48
+ bid: bid,
48
49
  total: total,
49
50
  failures: failures,
50
51
  pending: pending,
51
52
  created_at: created_at,
52
53
  complete: complete?,
53
54
  failure_info: failure_info,
54
- parent_bid: parent_bid
55
+ parent_bid: parent_bid,
56
+ child_count: child_count
55
57
  }
56
58
  end
57
59
  end
@@ -1,5 +1,5 @@
1
1
  module Sidekiq
2
2
  class Batch
3
- VERSION = '0.1.5'.freeze
3
+ VERSION = '0.1.6'.freeze
4
4
  end
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sidekiq-batch
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.5
4
+ version: 0.1.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Marcin Naglik
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-10-16 00:00:00.000000000 Z
11
+ date: 2019-08-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: sidekiq
@@ -121,8 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
121
121
  - !ruby/object:Gem::Version
122
122
  version: '0'
123
123
  requirements: []
124
- rubyforge_project:
125
- rubygems_version: 2.5.2.3
124
+ rubygems_version: 3.0.3
126
125
  signing_key:
127
126
  specification_version: 4
128
127
  summary: Sidekiq Batch Jobs