sidekiq-batch 0.1.4 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/dependabot.yml +8 -0
- data/.github/workflows/ci.yml +23 -0
- data/.github/workflows/stale.yml +19 -0
- data/Gemfile +0 -1
- data/lib/sidekiq/batch/callback.rb +72 -36
- data/lib/sidekiq/batch/extension/worker.rb +2 -2
- data/lib/sidekiq/batch/middleware.rb +4 -4
- data/lib/sidekiq/batch/status.rb +3 -1
- data/lib/sidekiq/batch/version.rb +1 -1
- data/lib/sidekiq/batch.rb +141 -72
- data/sidekiq-batch.gemspec +3 -3
- metadata +15 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 3d45a854229157eb70e9fb9b04390e6b4958fd535cb68d6dd2e0aab84c51e4b7
|
4
|
+
data.tar.gz: b8ec67ac9cce0cafde5b70883c0a77794f0d8506b694f74fb79765dc0ae5d7f2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e703eb7d37c4234269835ee009d73cafd4bb9afb11d8c818860e18e7504244db08142830eeac20d32bff96079febcb13ae130fd7e0a21eadeea6115cb5384989
|
7
|
+
data.tar.gz: e616deb4ed0f7599b9c09d1d0fd943f25131c89966ae0575b073c507053ac9012357ed07ac34456ba0f60d1a91ab7d75c337ffe6fb4e8aca6b21fe2aea2b1ea4
|
@@ -0,0 +1,23 @@
|
|
1
|
+
name: CI
|
2
|
+
|
3
|
+
on: [push, pull_request]
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
test:
|
7
|
+
|
8
|
+
runs-on: ubuntu-latest
|
9
|
+
|
10
|
+
strategy:
|
11
|
+
fail-fast: false
|
12
|
+
matrix:
|
13
|
+
ruby: ["2.5", "2.6", "2.7", "3.0", "3.1", ruby-head]
|
14
|
+
|
15
|
+
steps:
|
16
|
+
- uses: actions/checkout@v2
|
17
|
+
- name: Set up Ruby
|
18
|
+
uses: ruby/setup-ruby@v1
|
19
|
+
with:
|
20
|
+
bundler-cache: true # 'bundle install' and cache gems
|
21
|
+
ruby-version: ${{ matrix.ruby }}
|
22
|
+
- name: Run tests
|
23
|
+
run: bundle exec rake
|
@@ -0,0 +1,19 @@
|
|
1
|
+
name: Mark stale issues and pull requests
|
2
|
+
|
3
|
+
on:
|
4
|
+
schedule:
|
5
|
+
- cron: "0 0 * * *"
|
6
|
+
|
7
|
+
jobs:
|
8
|
+
stale:
|
9
|
+
|
10
|
+
runs-on: ubuntu-latest
|
11
|
+
|
12
|
+
steps:
|
13
|
+
- uses: actions/stale@v1
|
14
|
+
with:
|
15
|
+
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
16
|
+
stale-issue-message: 'Stale issue message'
|
17
|
+
stale-pr-message: 'Stale pull request message'
|
18
|
+
stale-issue-label: 'no-issue-activity'
|
19
|
+
stale-pr-label: 'no-pr-activity'
|
data/Gemfile
CHANGED
@@ -6,62 +6,98 @@ module Sidekiq
|
|
6
6
|
|
7
7
|
def perform(clazz, event, opts, bid, parent_bid)
|
8
8
|
return unless %w(success complete).include?(event)
|
9
|
-
clazz, method = clazz.split("#") if (clazz.class == String && clazz.include?("#"))
|
9
|
+
clazz, method = clazz.split("#") if (clazz && clazz.class == String && clazz.include?("#"))
|
10
10
|
method = "on_#{event}" if method.nil?
|
11
11
|
status = Sidekiq::Batch::Status.new(bid)
|
12
|
-
clazz.constantize.new.send(method, status, opts)
|
13
12
|
|
14
|
-
|
13
|
+
if clazz && object = Object.const_get(clazz)
|
14
|
+
instance = object.new
|
15
|
+
instance.send(method, status, opts) if instance.respond_to?(method)
|
16
|
+
end
|
15
17
|
end
|
18
|
+
end
|
16
19
|
|
20
|
+
class Finalize
|
21
|
+
def dispatch status, opts
|
22
|
+
bid = opts["bid"]
|
23
|
+
callback_bid = status.bid
|
24
|
+
event = opts["event"].to_sym
|
25
|
+
callback_batch = bid != callback_bid
|
17
26
|
|
18
|
-
|
19
|
-
if (parent_bid)
|
20
|
-
_, _, success, pending, children = Sidekiq.redis do |r|
|
21
|
-
r.multi do
|
22
|
-
r.sadd("BID-#{parent_bid}-success", bid)
|
23
|
-
r.expire("BID-#{parent_bid}-success", Sidekiq::Batch::BID_EXPIRE_TTL)
|
24
|
-
r.scard("BID-#{parent_bid}-success")
|
25
|
-
r.hincrby("BID-#{parent_bid}", "pending", 0)
|
26
|
-
r.hincrby("BID-#{parent_bid}", "children", 0)
|
27
|
-
end
|
28
|
-
end
|
27
|
+
Sidekiq.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{callback_batch}"}
|
29
28
|
|
30
|
-
|
31
|
-
|
29
|
+
batch_status = Status.new bid
|
30
|
+
send(event, bid, batch_status, batch_status.parent_bid)
|
32
31
|
|
33
|
-
|
34
|
-
|
35
|
-
|
32
|
+
|
33
|
+
# Different events are run in different callback batches
|
34
|
+
Sidekiq::Batch.cleanup_redis callback_bid if callback_batch
|
35
|
+
Sidekiq::Batch.cleanup_redis bid if event == :success
|
36
36
|
end
|
37
37
|
|
38
|
-
def
|
39
|
-
|
40
|
-
_, complete, pending, children, failure = Sidekiq.redis do |r|
|
41
|
-
r.multi do
|
42
|
-
r.sadd("BID-#{parent_bid}-complete", bid)
|
43
|
-
r.scard("BID-#{parent_bid}-complete")
|
44
|
-
r.hincrby("BID-#{parent_bid}", "pending", 0)
|
45
|
-
r.hincrby("BID-#{parent_bid}", "children", 0)
|
46
|
-
r.hlen("BID-#{parent_bid}-failed")
|
47
|
-
end
|
48
|
-
end
|
38
|
+
def success(bid, status, parent_bid)
|
39
|
+
return unless parent_bid
|
49
40
|
|
50
|
-
|
41
|
+
_, _, success, _, complete, pending, children, failure = Sidekiq.redis do |r|
|
42
|
+
r.multi do |pipeline|
|
43
|
+
pipeline.sadd("BID-#{parent_bid}-success", bid)
|
44
|
+
pipeline.expire("BID-#{parent_bid}-success", Sidekiq::Batch::BID_EXPIRE_TTL)
|
45
|
+
pipeline.scard("BID-#{parent_bid}-success")
|
46
|
+
pipeline.sadd("BID-#{parent_bid}-complete", bid)
|
47
|
+
pipeline.scard("BID-#{parent_bid}-complete")
|
48
|
+
pipeline.hincrby("BID-#{parent_bid}", "pending", 0)
|
49
|
+
pipeline.hincrby("BID-#{parent_bid}", "children", 0)
|
50
|
+
pipeline.scard("BID-#{parent_bid}-failed")
|
51
|
+
end
|
52
|
+
end
|
53
|
+
# if job finished successfully and parent batch completed call parent complete callback
|
54
|
+
# Success callback is called after complete callback
|
55
|
+
if complete == children && pending == failure
|
56
|
+
Sidekiq.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
|
57
|
+
Batch.enqueue_callbacks(:complete, parent_bid)
|
51
58
|
end
|
52
59
|
|
60
|
+
end
|
61
|
+
|
62
|
+
def complete(bid, status, parent_bid)
|
53
63
|
pending, children, success = Sidekiq.redis do |r|
|
54
|
-
r.multi do
|
55
|
-
|
56
|
-
|
57
|
-
|
64
|
+
r.multi do |pipeline|
|
65
|
+
pipeline.hincrby("BID-#{bid}", "pending", 0)
|
66
|
+
pipeline.hincrby("BID-#{bid}", "children", 0)
|
67
|
+
pipeline.scard("BID-#{bid}-success")
|
58
68
|
end
|
59
69
|
end
|
60
70
|
|
61
|
-
|
71
|
+
# if we batch was successful run success callback
|
72
|
+
if pending.to_i.zero? && children == success
|
73
|
+
Batch.enqueue_callbacks(:success, bid)
|
74
|
+
|
75
|
+
elsif parent_bid
|
76
|
+
# if batch was not successfull check and see if its parent is complete
|
77
|
+
# if the parent is complete we trigger the complete callback
|
78
|
+
# We don't want to run this if the batch was successfull because the success
|
79
|
+
# callback may add more jobs to the parent batch
|
62
80
|
|
81
|
+
Sidekiq.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
|
82
|
+
_, complete, pending, children, failure = Sidekiq.redis do |r|
|
83
|
+
r.multi do |pipeline|
|
84
|
+
pipeline.sadd("BID-#{parent_bid}-complete", bid)
|
85
|
+
pipeline.scard("BID-#{parent_bid}-complete")
|
86
|
+
pipeline.hincrby("BID-#{parent_bid}", "pending", 0)
|
87
|
+
pipeline.hincrby("BID-#{parent_bid}", "children", 0)
|
88
|
+
pipeline.scard("BID-#{parent_bid}-failed")
|
89
|
+
end
|
90
|
+
end
|
91
|
+
if complete == children && pending == failure
|
92
|
+
Batch.enqueue_callbacks(:complete, parent_bid)
|
93
|
+
end
|
94
|
+
end
|
63
95
|
end
|
64
96
|
|
97
|
+
def cleanup_redis bid, callback_bid=nil
|
98
|
+
Sidekiq::Batch.cleanup_redis bid
|
99
|
+
Sidekiq::Batch.cleanup_redis callback_bid if callback_bid
|
100
|
+
end
|
65
101
|
end
|
66
102
|
end
|
67
103
|
end
|
@@ -1,11 +1,11 @@
|
|
1
1
|
module Sidekiq::Batch::Extension
|
2
2
|
module Worker
|
3
3
|
def bid
|
4
|
-
Thread.current[:bid
|
4
|
+
Thread.current[:batch].bid
|
5
5
|
end
|
6
6
|
|
7
7
|
def batch
|
8
|
-
|
8
|
+
Thread.current[:batch]
|
9
9
|
end
|
10
10
|
|
11
11
|
def valid_within_batch?
|
@@ -5,7 +5,7 @@ module Sidekiq
|
|
5
5
|
module Middleware
|
6
6
|
class ClientMiddleware
|
7
7
|
def call(_worker, msg, _queue, _redis_pool = nil)
|
8
|
-
if (batch = Thread.current[:
|
8
|
+
if (batch = Thread.current[:batch])
|
9
9
|
batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
|
10
10
|
end
|
11
11
|
yield
|
@@ -16,15 +16,15 @@ module Sidekiq
|
|
16
16
|
def call(_worker, msg, _queue)
|
17
17
|
if (bid = msg['bid'])
|
18
18
|
begin
|
19
|
-
Thread.current[:
|
19
|
+
Thread.current[:batch] = Sidekiq::Batch.new(bid)
|
20
20
|
yield
|
21
|
-
Thread.current[:
|
21
|
+
Thread.current[:batch] = nil
|
22
22
|
Batch.process_successful_job(bid, msg['jid'])
|
23
23
|
rescue
|
24
24
|
Batch.process_failed_job(bid, msg['jid'])
|
25
25
|
raise
|
26
26
|
ensure
|
27
|
-
Thread.current[:
|
27
|
+
Thread.current[:batch] = nil
|
28
28
|
end
|
29
29
|
else
|
30
30
|
yield
|
data/lib/sidekiq/batch/status.rb
CHANGED
@@ -45,13 +45,15 @@ module Sidekiq
|
|
45
45
|
|
46
46
|
def data
|
47
47
|
{
|
48
|
+
bid: bid,
|
48
49
|
total: total,
|
49
50
|
failures: failures,
|
50
51
|
pending: pending,
|
51
52
|
created_at: created_at,
|
52
53
|
complete: complete?,
|
53
54
|
failure_info: failure_info,
|
54
|
-
parent_bid: parent_bid
|
55
|
+
parent_bid: parent_bid,
|
56
|
+
child_count: child_count
|
55
57
|
}
|
56
58
|
end
|
57
59
|
end
|
data/lib/sidekiq/batch.rb
CHANGED
@@ -10,7 +10,7 @@ module Sidekiq
|
|
10
10
|
class Batch
|
11
11
|
class NoBlockGivenError < StandardError; end
|
12
12
|
|
13
|
-
BID_EXPIRE_TTL =
|
13
|
+
BID_EXPIRE_TTL = 2_592_000
|
14
14
|
|
15
15
|
attr_reader :bid, :description, :callback_queue, :created_at
|
16
16
|
|
@@ -33,16 +33,21 @@ module Sidekiq
|
|
33
33
|
persist_bid_attr('callback_queue', callback_queue)
|
34
34
|
end
|
35
35
|
|
36
|
+
def callback_batch=(callback_batch)
|
37
|
+
@callback_batch = callback_batch
|
38
|
+
persist_bid_attr('callback_batch', callback_batch)
|
39
|
+
end
|
40
|
+
|
36
41
|
def on(event, callback, options = {})
|
37
42
|
return unless %w(success complete).include?(event.to_s)
|
38
43
|
callback_key = "#{@bidkey}-callbacks-#{event}"
|
39
44
|
Sidekiq.redis do |r|
|
40
|
-
r.multi do
|
41
|
-
|
45
|
+
r.multi do |pipeline|
|
46
|
+
pipeline.sadd(callback_key, JSON.unparse({
|
42
47
|
callback: callback,
|
43
48
|
opts: options
|
44
49
|
}))
|
45
|
-
|
50
|
+
pipeline.expire(callback_key, BID_EXPIRE_TTL)
|
46
51
|
end
|
47
52
|
end
|
48
53
|
end
|
@@ -54,13 +59,13 @@ module Sidekiq
|
|
54
59
|
|
55
60
|
begin
|
56
61
|
if !@existing && !@initialized
|
57
|
-
parent_bid = Thread.current[:
|
62
|
+
parent_bid = Thread.current[:batch].bid if Thread.current[:batch]
|
58
63
|
|
59
64
|
Sidekiq.redis do |r|
|
60
|
-
r.multi do
|
61
|
-
|
62
|
-
|
63
|
-
|
65
|
+
r.multi do |pipeline|
|
66
|
+
pipeline.hset(@bidkey, "created_at", @created_at)
|
67
|
+
pipeline.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
|
68
|
+
pipeline.expire(@bidkey, BID_EXPIRE_TTL)
|
64
69
|
end
|
65
70
|
end
|
66
71
|
|
@@ -70,28 +75,29 @@ module Sidekiq
|
|
70
75
|
@ready_to_queue = []
|
71
76
|
|
72
77
|
begin
|
73
|
-
parent = Thread.current[:
|
74
|
-
Thread.current[:
|
78
|
+
parent = Thread.current[:batch]
|
79
|
+
Thread.current[:batch] = self
|
75
80
|
yield
|
76
81
|
ensure
|
77
|
-
Thread.current[:
|
82
|
+
Thread.current[:batch] = parent
|
78
83
|
end
|
79
84
|
|
80
85
|
return [] if @ready_to_queue.size == 0
|
81
86
|
|
82
87
|
Sidekiq.redis do |r|
|
83
|
-
r.multi do
|
88
|
+
r.multi do |pipeline|
|
84
89
|
if parent_bid
|
85
|
-
|
86
|
-
|
90
|
+
pipeline.hincrby("BID-#{parent_bid}", "children", 1)
|
91
|
+
pipeline.hincrby("BID-#{parent_bid}", "total", @ready_to_queue.size)
|
92
|
+
pipeline.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
|
87
93
|
end
|
88
94
|
|
89
|
-
|
90
|
-
|
91
|
-
|
95
|
+
pipeline.hincrby(@bidkey, "pending", @ready_to_queue.size)
|
96
|
+
pipeline.hincrby(@bidkey, "total", @ready_to_queue.size)
|
97
|
+
pipeline.expire(@bidkey, BID_EXPIRE_TTL)
|
92
98
|
|
93
|
-
|
94
|
-
|
99
|
+
pipeline.sadd(@bidkey + "-jids", @ready_to_queue)
|
100
|
+
pipeline.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
|
95
101
|
end
|
96
102
|
end
|
97
103
|
|
@@ -132,91 +138,154 @@ module Sidekiq
|
|
132
138
|
|
133
139
|
def persist_bid_attr(attribute, value)
|
134
140
|
Sidekiq.redis do |r|
|
135
|
-
r.multi do
|
136
|
-
|
137
|
-
|
141
|
+
r.multi do |pipeline|
|
142
|
+
pipeline.hset(@bidkey, attribute, value)
|
143
|
+
pipeline.expire(@bidkey, BID_EXPIRE_TTL)
|
138
144
|
end
|
139
145
|
end
|
140
146
|
end
|
141
147
|
|
142
148
|
class << self
|
143
149
|
def process_failed_job(bid, jid)
|
144
|
-
_, pending, failed, children, complete = Sidekiq.redis do |r|
|
145
|
-
r.multi do
|
146
|
-
|
150
|
+
_, pending, failed, children, complete, parent_bid = Sidekiq.redis do |r|
|
151
|
+
r.multi do |pipeline|
|
152
|
+
pipeline.sadd("BID-#{bid}-failed", jid)
|
147
153
|
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
154
|
+
pipeline.hincrby("BID-#{bid}", "pending", 0)
|
155
|
+
pipeline.scard("BID-#{bid}-failed")
|
156
|
+
pipeline.hincrby("BID-#{bid}", "children", 0)
|
157
|
+
pipeline.scard("BID-#{bid}-complete")
|
158
|
+
pipeline.hget("BID-#{bid}", "parent_bid")
|
152
159
|
|
153
|
-
|
160
|
+
pipeline.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
# if the batch failed, and has a parent, update the parent to show one pending and failed job
|
165
|
+
if parent_bid
|
166
|
+
Sidekiq.redis do |r|
|
167
|
+
r.multi do |pipeline|
|
168
|
+
pipeline.hincrby("BID-#{parent_bid}", "pending", 1)
|
169
|
+
pipeline.sadd("BID-#{parent_bid}-failed", jid)
|
170
|
+
pipeline.expire("BID-#{parent_bid}-failed", BID_EXPIRE_TTL)
|
171
|
+
end
|
154
172
|
end
|
155
173
|
end
|
156
174
|
|
157
|
-
|
175
|
+
if pending.to_i == failed.to_i && children == complete
|
176
|
+
enqueue_callbacks(:complete, bid)
|
177
|
+
end
|
158
178
|
end
|
159
179
|
|
160
180
|
def process_successful_job(bid, jid)
|
161
181
|
failed, pending, children, complete, success, total, parent_bid = Sidekiq.redis do |r|
|
162
|
-
r.multi do
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
182
|
+
r.multi do |pipeline|
|
183
|
+
pipeline.scard("BID-#{bid}-failed")
|
184
|
+
pipeline.hincrby("BID-#{bid}", "pending", -1)
|
185
|
+
pipeline.hincrby("BID-#{bid}", "children", 0)
|
186
|
+
pipeline.scard("BID-#{bid}-complete")
|
187
|
+
pipeline.scard("BID-#{bid}-success")
|
188
|
+
pipeline.hget("BID-#{bid}", "total")
|
189
|
+
pipeline.hget("BID-#{bid}", "parent_bid")
|
190
|
+
|
191
|
+
pipeline.srem("BID-#{bid}-failed", jid)
|
192
|
+
pipeline.srem("BID-#{bid}-jids", jid)
|
193
|
+
pipeline.expire("BID-#{bid}", BID_EXPIRE_TTL)
|
174
194
|
end
|
175
195
|
end
|
176
196
|
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
197
|
+
all_success = pending.to_i.zero? && children == success
|
198
|
+
# if complete or successfull call complete callback (the complete callback may then call successful)
|
199
|
+
if (pending.to_i == failed.to_i && children == complete) || all_success
|
200
|
+
enqueue_callbacks(:complete, bid)
|
201
|
+
enqueue_callbacks(:success, bid) if all_success
|
202
|
+
end
|
181
203
|
end
|
182
204
|
|
183
205
|
def enqueue_callbacks(event, bid)
|
184
206
|
batch_key = "BID-#{bid}"
|
185
207
|
callback_key = "#{batch_key}-callbacks-#{event}"
|
186
|
-
|
187
|
-
r.multi do
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
208
|
+
already_processed, _, callbacks, queue, parent_bid, callback_batch = Sidekiq.redis do |r|
|
209
|
+
r.multi do |pipeline|
|
210
|
+
pipeline.hget(batch_key, event)
|
211
|
+
pipeline.hset(batch_key, event, true)
|
212
|
+
pipeline.smembers(callback_key)
|
213
|
+
pipeline.hget(batch_key, "callback_queue")
|
214
|
+
pipeline.hget(batch_key, "parent_bid")
|
215
|
+
pipeline.hget(batch_key, "callback_batch")
|
193
216
|
end
|
194
217
|
end
|
195
|
-
return if needed == 'true'
|
196
218
|
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
219
|
+
return if already_processed == 'true'
|
220
|
+
|
221
|
+
queue ||= "default"
|
222
|
+
parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid # Basically parent_bid.blank?
|
223
|
+
callback_args = callbacks.reduce([]) do |memo, jcb|
|
224
|
+
cb = Sidekiq.load_json(jcb)
|
225
|
+
memo << [cb['callback'], event, cb['opts'], bid, parent_bid]
|
226
|
+
end
|
227
|
+
|
228
|
+
opts = {"bid" => bid, "event" => event}
|
229
|
+
|
230
|
+
# Run callback batch finalize synchronously
|
231
|
+
if callback_batch
|
232
|
+
# Extract opts from cb_args or use current
|
233
|
+
# Pass in stored event as callback finalize is processed on complete event
|
234
|
+
cb_opts = callback_args.first&.at(2) || opts
|
235
|
+
|
236
|
+
Sidekiq.logger.debug {"Run callback batch bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
|
237
|
+
# Finalize now
|
238
|
+
finalizer = Sidekiq::Batch::Callback::Finalize.new
|
239
|
+
status = Status.new bid
|
240
|
+
finalizer.dispatch(status, cb_opts)
|
241
|
+
|
242
|
+
return
|
243
|
+
end
|
244
|
+
|
245
|
+
Sidekiq.logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
|
246
|
+
|
247
|
+
if callback_args.empty?
|
248
|
+
# Finalize now
|
249
|
+
finalizer = Sidekiq::Batch::Callback::Finalize.new
|
250
|
+
status = Status.new bid
|
251
|
+
finalizer.dispatch(status, opts)
|
252
|
+
else
|
253
|
+
# Otherwise finalize in sub batch complete callback
|
254
|
+
cb_batch = self.new
|
255
|
+
cb_batch.callback_batch = true
|
256
|
+
Sidekiq.logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
|
257
|
+
cb_batch.on(:complete, "Sidekiq::Batch::Callback::Finalize#dispatch", opts)
|
258
|
+
cb_batch.jobs do
|
259
|
+
push_callbacks callback_args, queue
|
260
|
+
end
|
209
261
|
end
|
210
262
|
end
|
211
263
|
|
212
264
|
def cleanup_redis(bid)
|
265
|
+
Sidekiq.logger.debug {"Cleaning redis of batch #{bid}"}
|
213
266
|
Sidekiq.redis do |r|
|
214
|
-
r.del(
|
215
|
-
|
216
|
-
|
217
|
-
|
267
|
+
r.del(
|
268
|
+
"BID-#{bid}",
|
269
|
+
"BID-#{bid}-callbacks-complete",
|
270
|
+
"BID-#{bid}-callbacks-success",
|
271
|
+
"BID-#{bid}-failed",
|
272
|
+
|
273
|
+
"BID-#{bid}-success",
|
274
|
+
"BID-#{bid}-complete",
|
275
|
+
"BID-#{bid}-jids",
|
276
|
+
)
|
218
277
|
end
|
219
278
|
end
|
279
|
+
|
280
|
+
private
|
281
|
+
|
282
|
+
def push_callbacks args, queue
|
283
|
+
Sidekiq::Client.push_bulk(
|
284
|
+
'class' => Sidekiq::Batch::Callback::Worker,
|
285
|
+
'args' => args,
|
286
|
+
'queue' => queue
|
287
|
+
) unless args.empty?
|
288
|
+
end
|
220
289
|
end
|
221
290
|
end
|
222
291
|
end
|
data/sidekiq-batch.gemspec
CHANGED
@@ -21,8 +21,8 @@ Gem::Specification.new do |spec|
|
|
21
21
|
|
22
22
|
spec.add_dependency "sidekiq", ">= 3"
|
23
23
|
|
24
|
-
spec.add_development_dependency "bundler", "~> 1
|
25
|
-
spec.add_development_dependency "rake", "~>
|
24
|
+
spec.add_development_dependency "bundler", "~> 2.1"
|
25
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
26
26
|
spec.add_development_dependency "rspec", "~> 3.0"
|
27
|
-
spec.add_development_dependency "fakeredis", "~> 0.
|
27
|
+
spec.add_development_dependency "fakeredis", "~> 0.8.0"
|
28
28
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-batch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Marcin Naglik
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-03-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -30,28 +30,28 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '1
|
33
|
+
version: '2.1'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '1
|
40
|
+
version: '2.1'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '13.0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '13.0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: rspec
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -72,14 +72,14 @@ dependencies:
|
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: 0.
|
75
|
+
version: 0.8.0
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: 0.
|
82
|
+
version: 0.8.0
|
83
83
|
description: Sidekiq Batch Jobs Implementation
|
84
84
|
email:
|
85
85
|
- marcin.naglik@gmail.com
|
@@ -87,6 +87,9 @@ executables: []
|
|
87
87
|
extensions: []
|
88
88
|
extra_rdoc_files: []
|
89
89
|
files:
|
90
|
+
- ".github/dependabot.yml"
|
91
|
+
- ".github/workflows/ci.yml"
|
92
|
+
- ".github/workflows/stale.yml"
|
90
93
|
- ".gitignore"
|
91
94
|
- ".rspec"
|
92
95
|
- ".travis.yml"
|
@@ -106,7 +109,7 @@ homepage: http://github.com/breamware/sidekiq-batch
|
|
106
109
|
licenses:
|
107
110
|
- MIT
|
108
111
|
metadata: {}
|
109
|
-
post_install_message:
|
112
|
+
post_install_message:
|
110
113
|
rdoc_options: []
|
111
114
|
require_paths:
|
112
115
|
- lib
|
@@ -121,9 +124,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
124
|
- !ruby/object:Gem::Version
|
122
125
|
version: '0'
|
123
126
|
requirements: []
|
124
|
-
|
125
|
-
|
126
|
-
signing_key:
|
127
|
+
rubygems_version: 3.3.7
|
128
|
+
signing_key:
|
127
129
|
specification_version: 4
|
128
130
|
summary: Sidekiq Batch Jobs
|
129
131
|
test_files: []
|