sidekiq-batch 0.1.9 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/FUNDING.yml +1 -1
- data/.github/workflows/ci.yml +16 -2
- data/README.md +35 -1
- data/lib/sidekiq/batch/version.rb +1 -1
- data/lib/sidekiq/batch.rb +53 -14
- data/sidekiq-batch.gemspec +1 -2
- metadata +11 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e0e119457317bdc0bbe60785e41d4fff908f0d818ec760802b96fd9905ad14fc
|
4
|
+
data.tar.gz: 46cd957ceb643e946a6bf5ebb49ae93e14e3514384753fa7d910da0476e720b3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c86dcadbf158fb9a533163ad042e77d18192241ab7985c4204b6526f78d3cb307a08a48fc173dcef7a7697dfb7316a847fe664ec098178b9e244d7d279978fbb
|
7
|
+
data.tar.gz: a070c7c91c502c06adc83269c3f28b97789250a80b8518ce80ba48511ddacb80537e7ffe9075c03f45062384a53b50fd03af46cb9f3b8dba39e3f03f0498e8a8
|
data/.github/FUNDING.yml
CHANGED
data/.github/workflows/ci.yml
CHANGED
@@ -1,16 +1,30 @@
|
|
1
1
|
name: CI
|
2
2
|
|
3
|
-
on: [
|
3
|
+
on: [pull_request, workflow_dispatch]
|
4
4
|
|
5
5
|
jobs:
|
6
6
|
test:
|
7
|
+
env:
|
8
|
+
REDIS_HOST: 'redis'
|
7
9
|
|
8
10
|
runs-on: ubuntu-latest
|
11
|
+
services:
|
12
|
+
redis:
|
13
|
+
image: redis
|
14
|
+
# Set health checks to wait until redis has started
|
15
|
+
options: >-
|
16
|
+
--health-cmd "redis-cli ping"
|
17
|
+
--health-interval 10s
|
18
|
+
--health-timeout 5s
|
19
|
+
--health-retries 5
|
20
|
+
ports:
|
21
|
+
# Maps port 6379 on service container to the host
|
22
|
+
- 6379:6379
|
9
23
|
|
10
24
|
strategy:
|
11
25
|
fail-fast: false
|
12
26
|
matrix:
|
13
|
-
ruby: ["
|
27
|
+
ruby: ["3.0", "3.1", "3.2", ruby-head]
|
14
28
|
|
15
29
|
steps:
|
16
30
|
- uses: actions/checkout@v2
|
data/README.md
CHANGED
@@ -32,7 +32,41 @@ Or install it yourself as:
|
|
32
32
|
|
33
33
|
## Usage
|
34
34
|
|
35
|
-
Sidekiq Batch is drop-in replacement for the API from Sidekiq PRO. See https://github.com/mperham/sidekiq/wiki/Batches for usage.
|
35
|
+
Sidekiq Batch is MOSTLY a drop-in replacement for the API from Sidekiq PRO. See https://github.com/mperham/sidekiq/wiki/Batches for usage.
|
36
|
+
|
37
|
+
## Caveats/Gotchas
|
38
|
+
|
39
|
+
Consider the following workflow:
|
40
|
+
|
41
|
+
* Batch Z created
|
42
|
+
* Worker A queued in batch Z
|
43
|
+
* Worker A starts Worker B in batch Z
|
44
|
+
* Worker B completes *before* worker A does
|
45
|
+
* Worker A completes
|
46
|
+
|
47
|
+
In the standard configuration, the `on(:success)` and `on(:complete)` callbacks will be triggered when Worker B completes.
|
48
|
+
This configuration is the default, simply for legacy reasons. This gem adds the following option to the sidekiq.yml options:
|
49
|
+
|
50
|
+
```yaml
|
51
|
+
:batch_push_interval: 0
|
52
|
+
```
|
53
|
+
|
54
|
+
When this value is *absent* (aka legacy), Worker A will only push the increment of batch jobs (aka Worker B) *when it completes*
|
55
|
+
|
56
|
+
When this value is set to `0`, Worker A will increment the count as soon as `WorkerB.perform_async` is called
|
57
|
+
|
58
|
+
When this value is a positive number, Worker A will wait a maximum of value-seconds before pushing the increment to redis, or until it's done, whichever comes first.
|
59
|
+
|
60
|
+
This comes into play if Worker A is queueing thousands of WorkerB jobs, or has some other reason for WorkerB to complete beforehand.
|
61
|
+
|
62
|
+
If you are queueing many WorkerB jobs, it is recommended to set this value to something like `3` to avoid thousands of calls to redis, and call WorkerB like so:
|
63
|
+
```ruby
|
64
|
+
WorkerB.perform_in(4.seconds, some, args)
|
65
|
+
```
|
66
|
+
this will ensure that the batch callback does not get triggered until WorkerA *and* the last WorkerB job complete.
|
67
|
+
|
68
|
+
If WorkerA is just slow for whatever reason, setting to `0` will update the batch status immediately so that the callbacks don't fire.
|
69
|
+
|
36
70
|
|
37
71
|
## Contributing
|
38
72
|
|
data/lib/sidekiq/batch.rb
CHANGED
@@ -20,7 +20,11 @@ module Sidekiq
|
|
20
20
|
@initialized = false
|
21
21
|
@created_at = Time.now.utc.to_f
|
22
22
|
@bidkey = "BID-" + @bid.to_s
|
23
|
-
@
|
23
|
+
@queued_jids = []
|
24
|
+
@pending_jids = []
|
25
|
+
|
26
|
+
@incremental_push = !Sidekiq.default_configuration[:batch_push_interval].nil?
|
27
|
+
@batch_push_interval = Sidekiq.default_configuration[:batch_push_interval]
|
24
28
|
end
|
25
29
|
|
26
30
|
def description=(description)
|
@@ -64,51 +68,86 @@ module Sidekiq
|
|
64
68
|
Sidekiq.redis do |r|
|
65
69
|
r.multi do |pipeline|
|
66
70
|
pipeline.hset(@bidkey, "created_at", @created_at)
|
67
|
-
pipeline.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
|
68
71
|
pipeline.expire(@bidkey, BID_EXPIRE_TTL)
|
72
|
+
if parent_bid
|
73
|
+
pipeline.hset(@bidkey, "parent_bid", parent_bid.to_s)
|
74
|
+
pipeline.hincrby("BID-#{parent_bid}", "children", 1)
|
75
|
+
end
|
69
76
|
end
|
70
77
|
end
|
71
78
|
|
72
79
|
@initialized = true
|
73
80
|
end
|
74
81
|
|
75
|
-
@
|
82
|
+
@queued_jids = []
|
83
|
+
@pending_jids = []
|
76
84
|
|
77
85
|
begin
|
78
86
|
parent = Thread.current[:batch]
|
79
87
|
Thread.current[:batch] = self
|
88
|
+
Thread.current[:parent_bid] = parent_bid
|
80
89
|
yield
|
81
90
|
ensure
|
82
91
|
Thread.current[:batch] = parent
|
92
|
+
Thread.current[:parent_bid] = nil
|
83
93
|
end
|
84
94
|
|
85
|
-
return [] if @
|
95
|
+
return [] if @queued_jids.size == 0
|
96
|
+
conditional_redis_increment!(true)
|
86
97
|
|
87
98
|
Sidekiq.redis do |r|
|
88
99
|
r.multi do |pipeline|
|
89
100
|
if parent_bid
|
90
|
-
pipeline.hincrby("BID-#{parent_bid}", "children", 1)
|
91
|
-
pipeline.hincrby("BID-#{parent_bid}", "total", @ready_to_queue.size)
|
92
101
|
pipeline.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
|
93
102
|
end
|
94
103
|
|
95
|
-
pipeline.hincrby(@bidkey, "pending", @ready_to_queue.size)
|
96
|
-
pipeline.hincrby(@bidkey, "total", @ready_to_queue.size)
|
97
104
|
pipeline.expire(@bidkey, BID_EXPIRE_TTL)
|
98
105
|
|
99
|
-
pipeline.sadd(@bidkey + "-jids",
|
106
|
+
pipeline.sadd(@bidkey + "-jids", @queued_jids)
|
100
107
|
pipeline.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
|
101
108
|
end
|
102
109
|
end
|
103
110
|
|
104
|
-
@
|
111
|
+
@queued_jids
|
105
112
|
ensure
|
106
113
|
Thread.current[:bid_data] = bid_data
|
107
114
|
end
|
108
115
|
end
|
109
116
|
|
110
117
|
def increment_job_queue(jid)
|
111
|
-
@
|
118
|
+
@queued_jids << jid
|
119
|
+
@pending_jids << jid
|
120
|
+
conditional_redis_increment!
|
121
|
+
end
|
122
|
+
|
123
|
+
def conditional_redis_increment!(force=false)
|
124
|
+
if should_increment? || force
|
125
|
+
parent_bid = Thread.current[:parent_bid]
|
126
|
+
Sidekiq.redis do |r|
|
127
|
+
r.multi do |pipeline|
|
128
|
+
if parent_bid
|
129
|
+
pipeline.hincrby("BID-#{parent_bid}", "total", @pending_jids.length)
|
130
|
+
pipeline.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
|
131
|
+
end
|
132
|
+
|
133
|
+
pipeline.hincrby(@bidkey, "pending", @pending_jids.length)
|
134
|
+
pipeline.hincrby(@bidkey, "total", @pending_jids.length)
|
135
|
+
pipeline.expire(@bidkey, BID_EXPIRE_TTL)
|
136
|
+
end
|
137
|
+
end
|
138
|
+
@pending_jids = []
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def should_increment?
|
143
|
+
return false unless @incremental_push
|
144
|
+
return true if @batch_push_interval == 0 || @queued_jids.length == 1
|
145
|
+
now = Time.now.to_f
|
146
|
+
@last_increment ||= now
|
147
|
+
if @last_increment + @batch_push_interval > now
|
148
|
+
@last_increment = now
|
149
|
+
return true
|
150
|
+
end
|
112
151
|
end
|
113
152
|
|
114
153
|
def invalidate_all
|
@@ -130,7 +169,7 @@ module Sidekiq
|
|
130
169
|
end
|
131
170
|
|
132
171
|
def valid?(batch = self)
|
133
|
-
valid =
|
172
|
+
valid = Sidekiq.redis { |r| r.exists("invalidated-bid-#{batch.bid}") }.zero?
|
134
173
|
batch.parent ? valid && valid?(batch.parent) : valid
|
135
174
|
end
|
136
175
|
|
@@ -209,7 +248,7 @@ module Sidekiq
|
|
209
248
|
already_processed, _, callbacks, queue, parent_bid, callback_batch = Sidekiq.redis do |r|
|
210
249
|
r.multi do |pipeline|
|
211
250
|
pipeline.hget(batch_key, event_name)
|
212
|
-
pipeline.hset(batch_key, event_name, true)
|
251
|
+
pipeline.hset(batch_key, event_name, 'true')
|
213
252
|
pipeline.smembers(callback_key)
|
214
253
|
pipeline.hget(batch_key, "callback_queue")
|
215
254
|
pipeline.hget(batch_key, "parent_bid")
|
@@ -253,7 +292,7 @@ module Sidekiq
|
|
253
292
|
else
|
254
293
|
# Otherwise finalize in sub batch complete callback
|
255
294
|
cb_batch = self.new
|
256
|
-
cb_batch.callback_batch = true
|
295
|
+
cb_batch.callback_batch = 'true'
|
257
296
|
Sidekiq.logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
|
258
297
|
cb_batch.on(:complete, "Sidekiq::Batch::Callback::Finalize#dispatch", opts)
|
259
298
|
cb_batch.jobs do
|
data/sidekiq-batch.gemspec
CHANGED
@@ -19,10 +19,9 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
20
|
spec.require_paths = ["lib"]
|
21
21
|
|
22
|
-
spec.add_dependency "sidekiq", ">=
|
22
|
+
spec.add_dependency "sidekiq", ">= 7", "<8"
|
23
23
|
|
24
24
|
spec.add_development_dependency "bundler", "~> 2.1"
|
25
25
|
spec.add_development_dependency "rake", "~> 13.0"
|
26
26
|
spec.add_development_dependency "rspec", "~> 3.0"
|
27
|
-
spec.add_development_dependency "fakeredis", "~> 0.8.0"
|
28
27
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-batch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Marcin Naglik
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-08-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -16,14 +16,20 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '7'
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '8'
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
27
|
- - ">="
|
25
28
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
29
|
+
version: '7'
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '8'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: bundler
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -66,20 +72,6 @@ dependencies:
|
|
66
72
|
- - "~>"
|
67
73
|
- !ruby/object:Gem::Version
|
68
74
|
version: '3.0'
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: fakeredis
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - "~>"
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: 0.8.0
|
76
|
-
type: :development
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - "~>"
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: 0.8.0
|
83
75
|
description: Sidekiq Batch Jobs Implementation
|
84
76
|
email:
|
85
77
|
- marcin.naglik@gmail.com
|
@@ -126,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
126
118
|
- !ruby/object:Gem::Version
|
127
119
|
version: '0'
|
128
120
|
requirements: []
|
129
|
-
rubygems_version: 3.
|
121
|
+
rubygems_version: 3.5.11
|
130
122
|
signing_key:
|
131
123
|
specification_version: 4
|
132
124
|
summary: Sidekiq Batch Jobs
|