sidekiq-unique-jobs 7.0.12 → 7.1.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq-unique-jobs might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/README.md +92 -25
- data/lib/sidekiq_unique_jobs.rb +4 -0
- data/lib/sidekiq_unique_jobs/config.rb +12 -0
- data/lib/sidekiq_unique_jobs/constants.rb +0 -1
- data/lib/sidekiq_unique_jobs/deprecation.rb +35 -0
- data/lib/sidekiq_unique_jobs/exceptions.rb +9 -0
- data/lib/sidekiq_unique_jobs/lock/base_lock.rb +56 -51
- data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +31 -9
- data/lib/sidekiq_unique_jobs/lock/until_executed.rb +17 -5
- data/lib/sidekiq_unique_jobs/lock/until_executing.rb +15 -1
- data/lib/sidekiq_unique_jobs/lock/until_expired.rb +21 -0
- data/lib/sidekiq_unique_jobs/lock/while_executing.rb +11 -6
- data/lib/sidekiq_unique_jobs/lock_config.rb +1 -1
- data/lib/sidekiq_unique_jobs/locksmith.rb +80 -81
- data/lib/sidekiq_unique_jobs/middleware/client.rb +8 -10
- data/lib/sidekiq_unique_jobs/middleware/server.rb +2 -0
- data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +7 -3
- data/lib/sidekiq_unique_jobs/options_with_fallback.rb +0 -9
- data/lib/sidekiq_unique_jobs/orphans/manager.rb +1 -0
- data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
- data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +1 -1
- data/lib/sidekiq_unique_jobs/reflectable.rb +17 -0
- data/lib/sidekiq_unique_jobs/reflections.rb +68 -0
- data/lib/sidekiq_unique_jobs/script/caller.rb +3 -1
- data/lib/sidekiq_unique_jobs/server.rb +1 -0
- data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +21 -0
- data/lib/sidekiq_unique_jobs/version.rb +1 -1
- metadata +7 -3
@@ -13,30 +13,52 @@ module SidekiqUniqueJobs
|
|
13
13
|
#
|
14
14
|
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
15
15
|
class UntilAndWhileExecuting < BaseLock
|
16
|
+
#
|
17
|
+
# Locks a sidekiq job
|
18
|
+
#
|
19
|
+
# @note Will call a conflict strategy if lock can't be achieved.
|
20
|
+
#
|
21
|
+
# @return [String, nil] the locked jid when properly locked, else nil.
|
22
|
+
#
|
23
|
+
# @yield to the caller when given a block
|
24
|
+
#
|
25
|
+
def lock(origin: :client)
|
26
|
+
return lock_failed(origin: origin) unless (token = locksmith.lock)
|
27
|
+
return yield token if block_given?
|
28
|
+
|
29
|
+
token
|
30
|
+
end
|
31
|
+
|
16
32
|
# Executes in the Sidekiq server process
|
17
33
|
# @yield to the worker class perform method
|
18
34
|
def execute
|
19
|
-
if unlock
|
20
|
-
|
21
|
-
|
22
|
-
end
|
35
|
+
if locksmith.unlock
|
36
|
+
# ensure_relocked do
|
37
|
+
runtime_lock.execute { return yield }
|
38
|
+
# end
|
23
39
|
else
|
24
|
-
|
40
|
+
reflect(:unlock_failed, item)
|
25
41
|
end
|
42
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
43
|
+
reflect(:execution_failed, item)
|
44
|
+
locksmith.lock(wait: 2)
|
45
|
+
|
46
|
+
raise
|
26
47
|
end
|
27
48
|
|
28
49
|
private
|
29
50
|
|
30
|
-
def
|
51
|
+
def ensure_relocked
|
31
52
|
yield
|
32
53
|
rescue Exception # rubocop:disable Lint/RescueException
|
33
|
-
|
34
|
-
lock
|
54
|
+
reflect(:execution_failed, item)
|
55
|
+
locksmith.lock
|
56
|
+
|
35
57
|
raise
|
36
58
|
end
|
37
59
|
|
38
60
|
def runtime_lock
|
39
|
-
@runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item, callback, redis_pool)
|
61
|
+
@runtime_lock ||= SidekiqUniqueJobs::Lock::WhileExecuting.new(item.dup, callback, redis_pool)
|
40
62
|
end
|
41
63
|
end
|
42
64
|
end
|
@@ -8,16 +8,28 @@ module SidekiqUniqueJobs
|
|
8
8
|
#
|
9
9
|
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
10
10
|
class UntilExecuted < BaseLock
|
11
|
-
|
11
|
+
#
|
12
|
+
# Locks a sidekiq job
|
13
|
+
#
|
14
|
+
# @note Will call a conflict strategy if lock can't be achieved.
|
15
|
+
#
|
16
|
+
# @return [String, nil] the locked jid when properly locked, else nil.
|
17
|
+
#
|
18
|
+
# @yield to the caller when given a block
|
19
|
+
#
|
20
|
+
def lock
|
21
|
+
return lock_failed(origin: :client) unless (token = locksmith.lock)
|
22
|
+
return yield token if block_given?
|
23
|
+
|
24
|
+
token
|
25
|
+
end
|
12
26
|
|
13
27
|
# Executes in the Sidekiq server process
|
14
28
|
# @yield to the worker class perform method
|
15
29
|
def execute
|
16
|
-
|
30
|
+
locksmith.execute do
|
17
31
|
yield
|
18
|
-
|
19
|
-
callback_safely
|
20
|
-
item[JID]
|
32
|
+
unlock_and_callback
|
21
33
|
end
|
22
34
|
end
|
23
35
|
end
|
@@ -8,10 +8,24 @@ module SidekiqUniqueJobs
|
|
8
8
|
#
|
9
9
|
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
10
10
|
class UntilExecuting < BaseLock
|
11
|
+
#
|
12
|
+
# Locks a sidekiq job
|
13
|
+
#
|
14
|
+
# @note Will call a conflict strategy if lock can't be achieved.
|
15
|
+
#
|
16
|
+
# @return [String, nil] the locked jid when properly locked, else nil.
|
17
|
+
#
|
18
|
+
def lock
|
19
|
+
return lock_failed unless (job_id = locksmith.lock)
|
20
|
+
return yield job_id if block_given?
|
21
|
+
|
22
|
+
job_id
|
23
|
+
end
|
24
|
+
|
11
25
|
# Executes in the Sidekiq server process
|
12
26
|
# @yield to the worker class perform method
|
13
27
|
def execute
|
14
|
-
|
28
|
+
callback_safely if locksmith.unlock
|
15
29
|
yield
|
16
30
|
end
|
17
31
|
end
|
@@ -8,6 +8,27 @@ module SidekiqUniqueJobs
|
|
8
8
|
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
9
9
|
#
|
10
10
|
class UntilExpired < UntilExecuted
|
11
|
+
#
|
12
|
+
# Locks a sidekiq job
|
13
|
+
#
|
14
|
+
# @note Will call a conflict strategy if lock can't be achieved.
|
15
|
+
#
|
16
|
+
# @return [String, nil] the locked jid when properly locked, else nil.
|
17
|
+
#
|
18
|
+
# @yield to the caller when given a block
|
19
|
+
#
|
20
|
+
def lock
|
21
|
+
return lock_failed unless (job_id = locksmith.lock)
|
22
|
+
return yield job_id if block_given?
|
23
|
+
|
24
|
+
job_id
|
25
|
+
end
|
26
|
+
|
27
|
+
# Executes in the Sidekiq server process
|
28
|
+
# @yield to the worker class perform method
|
29
|
+
def execute(&block)
|
30
|
+
locksmith.execute(&block)
|
31
|
+
end
|
11
32
|
end
|
12
33
|
end
|
13
34
|
end
|
@@ -29,7 +29,10 @@ module SidekiqUniqueJobs
|
|
29
29
|
# These locks should only ever be created in the server process.
|
30
30
|
# @return [true] always returns true
|
31
31
|
def lock
|
32
|
-
|
32
|
+
job_id = item[JID]
|
33
|
+
yield job_id if block_given?
|
34
|
+
|
35
|
+
job_id
|
33
36
|
end
|
34
37
|
|
35
38
|
# Executes in the Sidekiq server process.
|
@@ -37,13 +40,13 @@ module SidekiqUniqueJobs
|
|
37
40
|
# @yield to the worker class perform method
|
38
41
|
def execute
|
39
42
|
with_logging_context do
|
40
|
-
|
43
|
+
call_strategy(origin: :server) unless locksmith.execute do
|
41
44
|
yield
|
42
|
-
callback_safely
|
45
|
+
callback_safely if locksmith.unlock
|
46
|
+
ensure
|
47
|
+
locksmith.unlock
|
43
48
|
end
|
44
49
|
end
|
45
|
-
ensure
|
46
|
-
locksmith.unlock!
|
47
50
|
end
|
48
51
|
|
49
52
|
private
|
@@ -51,7 +54,9 @@ module SidekiqUniqueJobs
|
|
51
54
|
# This is safe as the base_lock always creates a new digest
|
52
55
|
# The append there for needs to be done every time
|
53
56
|
def append_unique_key_suffix
|
54
|
-
|
57
|
+
return if (lock_digest = item[LOCK_DIGEST]).end_with?(RUN_SUFFIX)
|
58
|
+
|
59
|
+
item[LOCK_DIGEST] = lock_digest + RUN_SUFFIX
|
55
60
|
end
|
56
61
|
end
|
57
62
|
end
|
@@ -13,6 +13,10 @@ module SidekiqUniqueJobs
|
|
13
13
|
# @!parse include SidekiqUniqueJobs::Logging
|
14
14
|
include SidekiqUniqueJobs::Logging
|
15
15
|
|
16
|
+
# includes "SidekiqUniqueJobs::Reflectable"
|
17
|
+
# @!parse include SidekiqUniqueJobs::Reflectable
|
18
|
+
include SidekiqUniqueJobs::Reflectable
|
19
|
+
|
16
20
|
# includes "SidekiqUniqueJobs::Timing"
|
17
21
|
# @!parse include SidekiqUniqueJobs::Timing
|
18
22
|
include SidekiqUniqueJobs::Timing
|
@@ -77,7 +81,7 @@ module SidekiqUniqueJobs
|
|
77
81
|
# Deletes the lock regardless of if it has a pttl set
|
78
82
|
#
|
79
83
|
def delete!
|
80
|
-
call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).positive?
|
84
|
+
call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).to_i.positive?
|
81
85
|
end
|
82
86
|
|
83
87
|
#
|
@@ -85,16 +89,23 @@ module SidekiqUniqueJobs
|
|
85
89
|
#
|
86
90
|
# @return [String] the Sidekiq job_id that was locked/queued
|
87
91
|
#
|
88
|
-
def lock(
|
92
|
+
def lock(wait: nil)
|
93
|
+
method_name = wait ? :primed_async : :primed_sync
|
89
94
|
redis(redis_pool) do |conn|
|
90
|
-
|
91
|
-
|
92
|
-
lock_sync(conn) do
|
95
|
+
lock!(conn, method(method_name), wait) do
|
93
96
|
return job_id
|
94
97
|
end
|
95
98
|
end
|
96
99
|
end
|
97
100
|
|
101
|
+
def execute(&block)
|
102
|
+
raise SidekiqUniqueJobs::InvalidArgument, "#execute needs a block" unless block
|
103
|
+
|
104
|
+
redis(redis_pool) do |conn|
|
105
|
+
lock!(conn, method(:primed_async), &block)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
98
109
|
#
|
99
110
|
# Removes the lock keys from Redis if locked by the provided jid/token
|
100
111
|
#
|
@@ -114,11 +125,17 @@ module SidekiqUniqueJobs
|
|
114
125
|
# @return [String] Sidekiq job_id (jid) if successful
|
115
126
|
#
|
116
127
|
def unlock!(conn = nil)
|
117
|
-
call_script(:unlock, key.to_a, argv, conn)
|
128
|
+
call_script(:unlock, key.to_a, argv, conn) do |unlocked_jid|
|
129
|
+
reflect(:debug, :unlocked, item, unlocked_jid) if unlocked_jid == job_id
|
130
|
+
|
131
|
+
unlocked_jid
|
132
|
+
end
|
118
133
|
end
|
119
134
|
|
120
135
|
# Checks if this instance is considered locked
|
121
136
|
#
|
137
|
+
# @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
|
138
|
+
#
|
122
139
|
# @return [true, false] true when the :LOCKED hash contains the job_id
|
123
140
|
#
|
124
141
|
def locked?(conn = nil)
|
@@ -134,7 +151,7 @@ module SidekiqUniqueJobs
|
|
134
151
|
# @return [String]
|
135
152
|
#
|
136
153
|
def to_s
|
137
|
-
"Locksmith##{object_id}(digest=#{key} job_id=#{job_id}
|
154
|
+
"Locksmith##{object_id}(digest=#{key} job_id=#{job_id} locked=#{locked?})"
|
138
155
|
end
|
139
156
|
|
140
157
|
#
|
@@ -159,70 +176,71 @@ module SidekiqUniqueJobs
|
|
159
176
|
|
160
177
|
attr_reader :redis_pool
|
161
178
|
|
162
|
-
def argv
|
163
|
-
[job_id, config.pttl, config.type, config.limit]
|
164
|
-
end
|
165
|
-
|
166
|
-
#
|
167
|
-
# Used for runtime locks that need automatic unlock after yielding
|
168
|
-
#
|
169
|
-
# @param [Redis] conn a redis connection
|
170
179
|
#
|
171
|
-
#
|
172
|
-
# @return [Object] whatever the block returns when lock was acquired
|
180
|
+
# Used to reduce some duplication from the two methods
|
173
181
|
#
|
174
|
-
# @
|
182
|
+
# @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
|
183
|
+
# @param [Method] primed_method reference to the method to use for getting a primed token
|
175
184
|
#
|
176
|
-
|
185
|
+
# @yieldparam [string] job_id the sidekiq JID
|
186
|
+
# @yieldreturn [void] whatever the calling block returns
|
187
|
+
def lock!(conn, primed_method, wait = nil)
|
177
188
|
return yield job_id if locked?(conn)
|
178
189
|
|
179
|
-
enqueue(conn) do
|
180
|
-
|
181
|
-
|
182
|
-
|
190
|
+
enqueue(conn) do |queued_jid|
|
191
|
+
reflect(:debug, item, queued_jid)
|
192
|
+
|
193
|
+
primed_method.call(conn, wait) do |primed_jid|
|
194
|
+
reflect(:debug, :primed, item, primed_jid)
|
195
|
+
|
196
|
+
locked_jid = call_script(:lock, key.to_a, argv, conn)
|
197
|
+
if locked_jid
|
198
|
+
reflect(:debug, :locked, item, locked_jid)
|
199
|
+
return yield job_id
|
200
|
+
end
|
183
201
|
end
|
184
202
|
end
|
185
|
-
ensure
|
186
|
-
unlock!(conn)
|
187
203
|
end
|
188
204
|
|
189
205
|
#
|
190
|
-
#
|
191
|
-
# @note Used for runtime locks to avoid problems with blocking commands
|
192
|
-
# in current thread
|
206
|
+
# Prepares all the various lock data
|
193
207
|
#
|
194
208
|
# @param [Redis] conn a redis connection
|
195
209
|
#
|
196
|
-
# @return [nil] when
|
197
|
-
# @return [
|
210
|
+
# @return [nil] when redis was already prepared for this lock
|
211
|
+
# @return [yield<String>] when successfully enqueued
|
198
212
|
#
|
199
|
-
def
|
200
|
-
|
201
|
-
|
202
|
-
|
213
|
+
def enqueue(conn)
|
214
|
+
queued_jid, elapsed = timed do
|
215
|
+
call_script(:queue, key.to_a, argv, conn)
|
216
|
+
end
|
217
|
+
|
218
|
+
return unless queued_jid
|
219
|
+
return unless [job_id, "1"].include?(queued_jid)
|
203
220
|
|
204
|
-
|
221
|
+
validity = config.pttl - elapsed - drift(config.pttl)
|
222
|
+
return unless validity >= 0 || config.pttl.zero?
|
223
|
+
|
224
|
+
write_lock_info(conn)
|
225
|
+
yield job_id
|
205
226
|
end
|
206
227
|
|
207
228
|
#
|
208
|
-
#
|
229
|
+
# Pops an enqueued token
|
230
|
+
# @note Used for runtime locks to avoid problems with blocking commands
|
231
|
+
# in current thread
|
209
232
|
#
|
210
233
|
# @param [Redis] conn a redis connection
|
211
234
|
#
|
212
235
|
# @return [nil] when lock was not possible
|
213
236
|
# @return [Object] whatever the block returns when lock was acquired
|
214
237
|
#
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
238
|
+
def primed_async(conn, wait = nil, &block)
|
239
|
+
primed_jid = Concurrent::Promises
|
240
|
+
.future(conn) { |red_con| pop_queued(red_con, wait) }
|
241
|
+
.value(add_drift(wait || config.ttl))
|
219
242
|
|
220
|
-
|
221
|
-
primed_sync(conn) do
|
222
|
-
locked_token = call_script(:lock, key.to_a, argv, conn)
|
223
|
-
return yield locked_token if locked_token
|
224
|
-
end
|
225
|
-
end
|
243
|
+
handle_primed(primed_jid, &block)
|
226
244
|
end
|
227
245
|
|
228
246
|
#
|
@@ -234,12 +252,15 @@ module SidekiqUniqueJobs
|
|
234
252
|
# @return [nil] when lock was not possible
|
235
253
|
# @return [Object] whatever the block returns when lock was acquired
|
236
254
|
#
|
237
|
-
def primed_sync(conn)
|
238
|
-
|
239
|
-
|
240
|
-
|
255
|
+
def primed_sync(conn, wait = nil, &block)
|
256
|
+
primed_jid = pop_queued(conn, wait)
|
257
|
+
handle_primed(primed_jid, &block)
|
258
|
+
end
|
241
259
|
|
242
|
-
|
260
|
+
def handle_primed(primed_jid)
|
261
|
+
return yield job_id if [job_id, "1"].include?(primed_jid)
|
262
|
+
|
263
|
+
reflect(:timeout, item) unless config.wait_for_lock?
|
243
264
|
end
|
244
265
|
|
245
266
|
#
|
@@ -249,9 +270,9 @@ module SidekiqUniqueJobs
|
|
249
270
|
#
|
250
271
|
# @return [String] a previously enqueued token (now taken off the queue)
|
251
272
|
#
|
252
|
-
def pop_queued(conn)
|
253
|
-
if config.wait_for_lock?
|
254
|
-
brpoplpush(conn)
|
273
|
+
def pop_queued(conn, wait = nil)
|
274
|
+
if wait || config.wait_for_lock?
|
275
|
+
brpoplpush(conn, wait)
|
255
276
|
else
|
256
277
|
rpoplpush(conn)
|
257
278
|
end
|
@@ -260,9 +281,10 @@ module SidekiqUniqueJobs
|
|
260
281
|
#
|
261
282
|
# @api private
|
262
283
|
#
|
263
|
-
def brpoplpush(conn)
|
284
|
+
def brpoplpush(conn, wait = nil)
|
285
|
+
wait ||= config.timeout
|
264
286
|
# passing timeout 0 to brpoplpush causes it to block indefinitely
|
265
|
-
conn.brpoplpush(key.queued, key.primed, timeout:
|
287
|
+
conn.brpoplpush(key.queued, key.primed, timeout: wait)
|
266
288
|
end
|
267
289
|
|
268
290
|
#
|
@@ -272,27 +294,6 @@ module SidekiqUniqueJobs
|
|
272
294
|
conn.rpoplpush(key.queued, key.primed)
|
273
295
|
end
|
274
296
|
|
275
|
-
#
|
276
|
-
# Prepares all the various lock data
|
277
|
-
#
|
278
|
-
# @param [Redis] conn a redis connection
|
279
|
-
#
|
280
|
-
# @return [nil] when redis was already prepared for this lock
|
281
|
-
# @return [yield<String>] when successfully enqueued
|
282
|
-
#
|
283
|
-
def enqueue(conn)
|
284
|
-
queued_token, elapsed = timed do
|
285
|
-
call_script(:queue, key.to_a, argv, conn)
|
286
|
-
end
|
287
|
-
|
288
|
-
validity = config.pttl - elapsed - drift(config.pttl)
|
289
|
-
|
290
|
-
return unless queued_token && (validity >= 0 || config.pttl.zero?)
|
291
|
-
|
292
|
-
write_lock_info(conn)
|
293
|
-
yield queued_token
|
294
|
-
end
|
295
|
-
|
296
297
|
#
|
297
298
|
# Writes lock information to redis.
|
298
299
|
# The lock information contains information about worker, queue, limit etc.
|
@@ -335,10 +336,8 @@ module SidekiqUniqueJobs
|
|
335
336
|
conn.hexists(key.locked, job_id)
|
336
337
|
end
|
337
338
|
|
338
|
-
def
|
339
|
-
|
340
|
-
|
341
|
-
log_debug("Timed out after #{config.timeout}s while waiting for primed token (digest: #{key}, job_id: #{job_id})")
|
339
|
+
def argv
|
340
|
+
[job_id, config.pttl, config.type, config.limit]
|
342
341
|
end
|
343
342
|
|
344
343
|
def lock_info
|