sidekiq-unique-jobs 7.0.13 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sidekiq-unique-jobs might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/README.md +92 -25
- data/lib/sidekiq_unique_jobs/config.rb +16 -8
- data/lib/sidekiq_unique_jobs/constants.rb +44 -45
- data/lib/sidekiq_unique_jobs/deprecation.rb +35 -0
- data/lib/sidekiq_unique_jobs/exceptions.rb +9 -0
- data/lib/sidekiq_unique_jobs/lock/base_lock.rb +56 -51
- data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +31 -9
- data/lib/sidekiq_unique_jobs/lock/until_executed.rb +17 -5
- data/lib/sidekiq_unique_jobs/lock/until_executing.rb +15 -1
- data/lib/sidekiq_unique_jobs/lock/until_expired.rb +21 -0
- data/lib/sidekiq_unique_jobs/lock/while_executing.rb +12 -7
- data/lib/sidekiq_unique_jobs/lock_config.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock_ttl.rb +1 -1
- data/lib/sidekiq_unique_jobs/locksmith.rb +80 -81
- data/lib/sidekiq_unique_jobs/middleware/client.rb +8 -10
- data/lib/sidekiq_unique_jobs/middleware/server.rb +2 -0
- data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +7 -3
- data/lib/sidekiq_unique_jobs/options_with_fallback.rb +4 -11
- data/lib/sidekiq_unique_jobs/orphans/manager.rb +1 -0
- data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
- data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +1 -1
- data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +1 -1
- data/lib/sidekiq_unique_jobs/reflectable.rb +17 -0
- data/lib/sidekiq_unique_jobs/reflections.rb +68 -0
- data/lib/sidekiq_unique_jobs/script/caller.rb +3 -1
- data/lib/sidekiq_unique_jobs/server.rb +2 -1
- data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +13 -35
- data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +21 -0
- data/lib/sidekiq_unique_jobs/version.rb +1 -1
- data/lib/sidekiq_unique_jobs.rb +4 -0
- data/lib/tasks/changelog.rake +14 -14
- metadata +12 -8
@@ -8,10 +8,24 @@ module SidekiqUniqueJobs
|
|
8
8
|
#
|
9
9
|
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
10
10
|
class UntilExecuting < BaseLock
|
11
|
+
#
|
12
|
+
# Locks a sidekiq job
|
13
|
+
#
|
14
|
+
# @note Will call a conflict strategy if lock can't be achieved.
|
15
|
+
#
|
16
|
+
# @return [String, nil] the locked jid when properly locked, else nil.
|
17
|
+
#
|
18
|
+
def lock
|
19
|
+
return lock_failed unless (job_id = locksmith.lock)
|
20
|
+
return yield job_id if block_given?
|
21
|
+
|
22
|
+
job_id
|
23
|
+
end
|
24
|
+
|
11
25
|
# Executes in the Sidekiq server process
|
12
26
|
# @yield to the worker class perform method
|
13
27
|
def execute
|
14
|
-
|
28
|
+
callback_safely if locksmith.unlock
|
15
29
|
yield
|
16
30
|
end
|
17
31
|
end
|
@@ -8,6 +8,27 @@ module SidekiqUniqueJobs
|
|
8
8
|
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
9
9
|
#
|
10
10
|
class UntilExpired < UntilExecuted
|
11
|
+
#
|
12
|
+
# Locks a sidekiq job
|
13
|
+
#
|
14
|
+
# @note Will call a conflict strategy if lock can't be achieved.
|
15
|
+
#
|
16
|
+
# @return [String, nil] the locked jid when properly locked, else nil.
|
17
|
+
#
|
18
|
+
# @yield to the caller when given a block
|
19
|
+
#
|
20
|
+
def lock
|
21
|
+
return lock_failed unless (job_id = locksmith.lock)
|
22
|
+
return yield job_id if block_given?
|
23
|
+
|
24
|
+
job_id
|
25
|
+
end
|
26
|
+
|
27
|
+
# Executes in the Sidekiq server process
|
28
|
+
# @yield to the worker class perform method
|
29
|
+
def execute(&block)
|
30
|
+
locksmith.execute(&block)
|
31
|
+
end
|
11
32
|
end
|
12
33
|
end
|
13
34
|
end
|
@@ -11,7 +11,7 @@ module SidekiqUniqueJobs
|
|
11
11
|
#
|
12
12
|
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
13
13
|
class WhileExecuting < BaseLock
|
14
|
-
RUN_SUFFIX
|
14
|
+
RUN_SUFFIX ||= ":RUN"
|
15
15
|
|
16
16
|
include SidekiqUniqueJobs::OptionsWithFallback
|
17
17
|
include SidekiqUniqueJobs::Logging::Middleware
|
@@ -29,7 +29,10 @@ module SidekiqUniqueJobs
|
|
29
29
|
# These locks should only ever be created in the server process.
|
30
30
|
# @return [true] always returns true
|
31
31
|
def lock
|
32
|
-
|
32
|
+
job_id = item[JID]
|
33
|
+
yield job_id if block_given?
|
34
|
+
|
35
|
+
job_id
|
33
36
|
end
|
34
37
|
|
35
38
|
# Executes in the Sidekiq server process.
|
@@ -37,13 +40,13 @@ module SidekiqUniqueJobs
|
|
37
40
|
# @yield to the worker class perform method
|
38
41
|
def execute
|
39
42
|
with_logging_context do
|
40
|
-
|
43
|
+
call_strategy(origin: :server) unless locksmith.execute do
|
41
44
|
yield
|
42
|
-
callback_safely
|
45
|
+
callback_safely if locksmith.unlock
|
46
|
+
ensure
|
47
|
+
locksmith.unlock
|
43
48
|
end
|
44
49
|
end
|
45
|
-
ensure
|
46
|
-
locksmith.unlock!
|
47
50
|
end
|
48
51
|
|
49
52
|
private
|
@@ -51,7 +54,9 @@ module SidekiqUniqueJobs
|
|
51
54
|
# This is safe as the base_lock always creates a new digest
|
52
55
|
# The append there for needs to be done every time
|
53
56
|
def append_unique_key_suffix
|
54
|
-
|
57
|
+
return if (lock_digest = item[LOCK_DIGEST]).end_with?(RUN_SUFFIX)
|
58
|
+
|
59
|
+
item[LOCK_DIGEST] = lock_digest + RUN_SUFFIX
|
55
60
|
end
|
56
61
|
end
|
57
62
|
end
|
@@ -71,7 +71,7 @@ module SidekiqUniqueJobs
|
|
71
71
|
ttl ||= item[LOCK_EXPIRATION] # TODO: Deprecate at some point
|
72
72
|
ttl ||= worker_options[LOCK_EXPIRATION] # TODO: Deprecate at some point
|
73
73
|
ttl ||= SidekiqUniqueJobs.config.lock_ttl
|
74
|
-
ttl &&
|
74
|
+
ttl && ttl.to_i + time_until_scheduled
|
75
75
|
end
|
76
76
|
end
|
77
77
|
end
|
@@ -13,6 +13,10 @@ module SidekiqUniqueJobs
|
|
13
13
|
# @!parse include SidekiqUniqueJobs::Logging
|
14
14
|
include SidekiqUniqueJobs::Logging
|
15
15
|
|
16
|
+
# includes "SidekiqUniqueJobs::Reflectable"
|
17
|
+
# @!parse include SidekiqUniqueJobs::Reflectable
|
18
|
+
include SidekiqUniqueJobs::Reflectable
|
19
|
+
|
16
20
|
# includes "SidekiqUniqueJobs::Timing"
|
17
21
|
# @!parse include SidekiqUniqueJobs::Timing
|
18
22
|
include SidekiqUniqueJobs::Timing
|
@@ -77,7 +81,7 @@ module SidekiqUniqueJobs
|
|
77
81
|
# Deletes the lock regardless of if it has a pttl set
|
78
82
|
#
|
79
83
|
def delete!
|
80
|
-
call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).positive?
|
84
|
+
call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).to_i.positive?
|
81
85
|
end
|
82
86
|
|
83
87
|
#
|
@@ -85,16 +89,23 @@ module SidekiqUniqueJobs
|
|
85
89
|
#
|
86
90
|
# @return [String] the Sidekiq job_id that was locked/queued
|
87
91
|
#
|
88
|
-
def lock(
|
92
|
+
def lock(wait: nil)
|
93
|
+
method_name = wait ? :primed_async : :primed_sync
|
89
94
|
redis(redis_pool) do |conn|
|
90
|
-
|
91
|
-
|
92
|
-
lock_sync(conn) do
|
95
|
+
lock!(conn, method(method_name), wait) do
|
93
96
|
return job_id
|
94
97
|
end
|
95
98
|
end
|
96
99
|
end
|
97
100
|
|
101
|
+
def execute(&block)
|
102
|
+
raise SidekiqUniqueJobs::InvalidArgument, "#execute needs a block" unless block
|
103
|
+
|
104
|
+
redis(redis_pool) do |conn|
|
105
|
+
lock!(conn, method(:primed_async), &block)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
98
109
|
#
|
99
110
|
# Removes the lock keys from Redis if locked by the provided jid/token
|
100
111
|
#
|
@@ -114,11 +125,17 @@ module SidekiqUniqueJobs
|
|
114
125
|
# @return [String] Sidekiq job_id (jid) if successful
|
115
126
|
#
|
116
127
|
def unlock!(conn = nil)
|
117
|
-
call_script(:unlock, key.to_a, argv, conn)
|
128
|
+
call_script(:unlock, key.to_a, argv, conn) do |unlocked_jid|
|
129
|
+
reflect(:debug, :unlocked, item, unlocked_jid) if unlocked_jid == job_id
|
130
|
+
|
131
|
+
unlocked_jid
|
132
|
+
end
|
118
133
|
end
|
119
134
|
|
120
135
|
# Checks if this instance is considered locked
|
121
136
|
#
|
137
|
+
# @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
|
138
|
+
#
|
122
139
|
# @return [true, false] true when the :LOCKED hash contains the job_id
|
123
140
|
#
|
124
141
|
def locked?(conn = nil)
|
@@ -134,7 +151,7 @@ module SidekiqUniqueJobs
|
|
134
151
|
# @return [String]
|
135
152
|
#
|
136
153
|
def to_s
|
137
|
-
"Locksmith##{object_id}(digest=#{key} job_id=#{job_id}
|
154
|
+
"Locksmith##{object_id}(digest=#{key} job_id=#{job_id} locked=#{locked?})"
|
138
155
|
end
|
139
156
|
|
140
157
|
#
|
@@ -159,70 +176,71 @@ module SidekiqUniqueJobs
|
|
159
176
|
|
160
177
|
attr_reader :redis_pool
|
161
178
|
|
162
|
-
def argv
|
163
|
-
[job_id, config.pttl, config.type, config.limit]
|
164
|
-
end
|
165
|
-
|
166
|
-
#
|
167
|
-
# Used for runtime locks that need automatic unlock after yielding
|
168
|
-
#
|
169
|
-
# @param [Redis] conn a redis connection
|
170
179
|
#
|
171
|
-
#
|
172
|
-
# @return [Object] whatever the block returns when lock was acquired
|
180
|
+
# Used to reduce some duplication from the two methods
|
173
181
|
#
|
174
|
-
# @
|
182
|
+
# @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
|
183
|
+
# @param [Method] primed_method reference to the method to use for getting a primed token
|
175
184
|
#
|
176
|
-
|
185
|
+
# @yieldparam [string] job_id the sidekiq JID
|
186
|
+
# @yieldreturn [void] whatever the calling block returns
|
187
|
+
def lock!(conn, primed_method, wait = nil)
|
177
188
|
return yield job_id if locked?(conn)
|
178
189
|
|
179
|
-
enqueue(conn) do
|
180
|
-
|
181
|
-
|
182
|
-
|
190
|
+
enqueue(conn) do |queued_jid|
|
191
|
+
reflect(:debug, item, queued_jid)
|
192
|
+
|
193
|
+
primed_method.call(conn, wait) do |primed_jid|
|
194
|
+
reflect(:debug, :primed, item, primed_jid)
|
195
|
+
|
196
|
+
locked_jid = call_script(:lock, key.to_a, argv, conn)
|
197
|
+
if locked_jid
|
198
|
+
reflect(:debug, :locked, item, locked_jid)
|
199
|
+
return yield job_id
|
200
|
+
end
|
183
201
|
end
|
184
202
|
end
|
185
|
-
ensure
|
186
|
-
unlock!(conn)
|
187
203
|
end
|
188
204
|
|
189
205
|
#
|
190
|
-
#
|
191
|
-
# @note Used for runtime locks to avoid problems with blocking commands
|
192
|
-
# in current thread
|
206
|
+
# Prepares all the various lock data
|
193
207
|
#
|
194
208
|
# @param [Redis] conn a redis connection
|
195
209
|
#
|
196
|
-
# @return [nil] when
|
197
|
-
# @return [
|
210
|
+
# @return [nil] when redis was already prepared for this lock
|
211
|
+
# @return [yield<String>] when successfully enqueued
|
198
212
|
#
|
199
|
-
def
|
200
|
-
|
201
|
-
|
202
|
-
|
213
|
+
def enqueue(conn)
|
214
|
+
queued_jid, elapsed = timed do
|
215
|
+
call_script(:queue, key.to_a, argv, conn)
|
216
|
+
end
|
217
|
+
|
218
|
+
return unless queued_jid
|
219
|
+
return unless [job_id, "1"].include?(queued_jid)
|
203
220
|
|
204
|
-
|
221
|
+
validity = config.pttl - elapsed - drift(config.pttl)
|
222
|
+
return unless validity >= 0 || config.pttl.zero?
|
223
|
+
|
224
|
+
write_lock_info(conn)
|
225
|
+
yield job_id
|
205
226
|
end
|
206
227
|
|
207
228
|
#
|
208
|
-
#
|
229
|
+
# Pops an enqueued token
|
230
|
+
# @note Used for runtime locks to avoid problems with blocking commands
|
231
|
+
# in current thread
|
209
232
|
#
|
210
233
|
# @param [Redis] conn a redis connection
|
211
234
|
#
|
212
235
|
# @return [nil] when lock was not possible
|
213
236
|
# @return [Object] whatever the block returns when lock was acquired
|
214
237
|
#
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
238
|
+
def primed_async(conn, wait = nil, &block)
|
239
|
+
primed_jid = Concurrent::Promises
|
240
|
+
.future(conn) { |red_con| pop_queued(red_con, wait) }
|
241
|
+
.value(add_drift(wait || config.ttl))
|
219
242
|
|
220
|
-
|
221
|
-
primed_sync(conn) do
|
222
|
-
locked_token = call_script(:lock, key.to_a, argv, conn)
|
223
|
-
return yield locked_token if locked_token
|
224
|
-
end
|
225
|
-
end
|
243
|
+
handle_primed(primed_jid, &block)
|
226
244
|
end
|
227
245
|
|
228
246
|
#
|
@@ -234,12 +252,15 @@ module SidekiqUniqueJobs
|
|
234
252
|
# @return [nil] when lock was not possible
|
235
253
|
# @return [Object] whatever the block returns when lock was acquired
|
236
254
|
#
|
237
|
-
def primed_sync(conn)
|
238
|
-
|
239
|
-
|
240
|
-
|
255
|
+
def primed_sync(conn, wait = nil, &block)
|
256
|
+
primed_jid = pop_queued(conn, wait)
|
257
|
+
handle_primed(primed_jid, &block)
|
258
|
+
end
|
241
259
|
|
242
|
-
|
260
|
+
def handle_primed(primed_jid)
|
261
|
+
return yield job_id if [job_id, "1"].include?(primed_jid)
|
262
|
+
|
263
|
+
reflect(:timeout, item) unless config.wait_for_lock?
|
243
264
|
end
|
244
265
|
|
245
266
|
#
|
@@ -249,9 +270,9 @@ module SidekiqUniqueJobs
|
|
249
270
|
#
|
250
271
|
# @return [String] a previously enqueued token (now taken off the queue)
|
251
272
|
#
|
252
|
-
def pop_queued(conn)
|
253
|
-
if config.wait_for_lock?
|
254
|
-
brpoplpush(conn)
|
273
|
+
def pop_queued(conn, wait = nil)
|
274
|
+
if wait || config.wait_for_lock?
|
275
|
+
brpoplpush(conn, wait)
|
255
276
|
else
|
256
277
|
rpoplpush(conn)
|
257
278
|
end
|
@@ -260,9 +281,10 @@ module SidekiqUniqueJobs
|
|
260
281
|
#
|
261
282
|
# @api private
|
262
283
|
#
|
263
|
-
def brpoplpush(conn)
|
284
|
+
def brpoplpush(conn, wait = nil)
|
285
|
+
wait ||= config.timeout
|
264
286
|
# passing timeout 0 to brpoplpush causes it to block indefinitely
|
265
|
-
conn.brpoplpush(key.queued, key.primed, timeout:
|
287
|
+
conn.brpoplpush(key.queued, key.primed, timeout: wait)
|
266
288
|
end
|
267
289
|
|
268
290
|
#
|
@@ -272,27 +294,6 @@ module SidekiqUniqueJobs
|
|
272
294
|
conn.rpoplpush(key.queued, key.primed)
|
273
295
|
end
|
274
296
|
|
275
|
-
#
|
276
|
-
# Prepares all the various lock data
|
277
|
-
#
|
278
|
-
# @param [Redis] conn a redis connection
|
279
|
-
#
|
280
|
-
# @return [nil] when redis was already prepared for this lock
|
281
|
-
# @return [yield<String>] when successfully enqueued
|
282
|
-
#
|
283
|
-
def enqueue(conn)
|
284
|
-
queued_token, elapsed = timed do
|
285
|
-
call_script(:queue, key.to_a, argv, conn)
|
286
|
-
end
|
287
|
-
|
288
|
-
validity = config.pttl - elapsed - drift(config.pttl)
|
289
|
-
|
290
|
-
return unless queued_token && (validity >= 0 || config.pttl.zero?)
|
291
|
-
|
292
|
-
write_lock_info(conn)
|
293
|
-
yield queued_token
|
294
|
-
end
|
295
|
-
|
296
297
|
#
|
297
298
|
# Writes lock information to redis.
|
298
299
|
# The lock information contains information about worker, queue, limit etc.
|
@@ -335,10 +336,8 @@ module SidekiqUniqueJobs
|
|
335
336
|
conn.hexists(key.locked, job_id)
|
336
337
|
end
|
337
338
|
|
338
|
-
def
|
339
|
-
|
340
|
-
|
341
|
-
log_debug("Timed out after #{config.timeout}s while waiting for primed token (digest: #{key}, job_id: #{job_id})")
|
339
|
+
def argv
|
340
|
+
[job_id, config.pttl, config.type, config.limit]
|
342
341
|
end
|
343
342
|
|
344
343
|
def lock_info
|
@@ -6,7 +6,12 @@ module SidekiqUniqueJobs
|
|
6
6
|
#
|
7
7
|
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
8
|
class Client
|
9
|
+
# prepend "SidekiqUniqueJobs::Middleware"
|
10
|
+
# @!parse prepends SidekiqUniqueJobs::Middleware
|
9
11
|
prepend SidekiqUniqueJobs::Middleware
|
12
|
+
# includes "SidekiqUniqueJobs::Reflectable"
|
13
|
+
# @!parse include SidekiqUniqueJobs::Reflectable
|
14
|
+
include SidekiqUniqueJobs::Reflectable
|
10
15
|
|
11
16
|
# Calls this client middleware
|
12
17
|
# Used from Sidekiq.process_single
|
@@ -25,18 +30,11 @@ module SidekiqUniqueJobs
|
|
25
30
|
private
|
26
31
|
|
27
32
|
def lock
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
warn_about_duplicate
|
33
|
+
lock_instance.lock do |_locked_jid|
|
34
|
+
reflect(:locked, item)
|
35
|
+
return yield
|
32
36
|
end
|
33
37
|
end
|
34
|
-
|
35
|
-
def warn_about_duplicate
|
36
|
-
return unless log_duplicate?
|
37
|
-
|
38
|
-
log_warn "Already locked with another job_id (#{dump_json(item)})"
|
39
|
-
end
|
40
38
|
end
|
41
39
|
end
|
42
40
|
end
|
@@ -9,6 +9,7 @@ module SidekiqUniqueJobs
|
|
9
9
|
include SidekiqUniqueJobs::SidekiqWorkerMethods
|
10
10
|
include SidekiqUniqueJobs::Logging
|
11
11
|
include SidekiqUniqueJobs::JSON
|
12
|
+
include SidekiqUniqueJobs::Reflectable
|
12
13
|
|
13
14
|
# @param [Hash] item sidekiq job hash
|
14
15
|
def initialize(item, redis_pool = nil)
|
@@ -20,10 +21,13 @@ module SidekiqUniqueJobs
|
|
20
21
|
# This will mess up sidekiq stats because a new job is created
|
21
22
|
def call
|
22
23
|
if sidekiq_worker_class?
|
23
|
-
|
24
|
-
|
24
|
+
if worker_class.perform_in(5, *item[ARGS])
|
25
|
+
reflect(:rescheduled, item)
|
26
|
+
else
|
27
|
+
reflect(:reschedule_failed, item)
|
28
|
+
end
|
25
29
|
else
|
26
|
-
|
30
|
+
reflect(:unknown_sidekiq_worker, item)
|
27
31
|
end
|
28
32
|
end
|
29
33
|
end
|
@@ -29,15 +29,6 @@ module SidekiqUniqueJobs
|
|
29
29
|
!unique_enabled?
|
30
30
|
end
|
31
31
|
|
32
|
-
# Should duplicate payloads be logged?
|
33
|
-
#
|
34
|
-
#
|
35
|
-
# @return [true, false, nil]
|
36
|
-
#
|
37
|
-
def log_duplicate?
|
38
|
-
options[LOG_DUPLICATE] || item[LOG_DUPLICATE]
|
39
|
-
end
|
40
|
-
|
41
32
|
#
|
42
33
|
# A new lock for this Sidekiq Job
|
43
34
|
#
|
@@ -55,8 +46,10 @@ module SidekiqUniqueJobs
|
|
55
46
|
# @return [Class]
|
56
47
|
#
|
57
48
|
def lock_class
|
58
|
-
@lock_class ||=
|
59
|
-
|
49
|
+
@lock_class ||= begin
|
50
|
+
locks.fetch(lock_type.to_sym) do
|
51
|
+
raise UnknownLock, "No implementation for `lock: :#{lock_type}`"
|
52
|
+
end
|
60
53
|
end
|
61
54
|
end
|
62
55
|
|
@@ -0,0 +1,170 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
module Orphans
|
5
|
+
# Restarts orphan manager if it is considered dead
|
6
|
+
module ReaperResurrector
|
7
|
+
module_function
|
8
|
+
|
9
|
+
include SidekiqUniqueJobs::Connection
|
10
|
+
include SidekiqUniqueJobs::Logging
|
11
|
+
|
12
|
+
DRIFT_FACTOR = 0.1
|
13
|
+
REAPERS = [:ruby, :lua].freeze
|
14
|
+
|
15
|
+
#
|
16
|
+
# Starts reaper resurrector that watches orphans reaper
|
17
|
+
#
|
18
|
+
# @return [SidekiqUniqueJobs::TimerTask] the task that was started
|
19
|
+
#
|
20
|
+
def start
|
21
|
+
return if resurrector_disabled?
|
22
|
+
return if reaper_disabled?
|
23
|
+
|
24
|
+
with_logging_context do
|
25
|
+
run_task
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
#
|
30
|
+
# Runs reaper resurrector task
|
31
|
+
#
|
32
|
+
# @return [SidekiqUniqueJobs::TimerTask]
|
33
|
+
def run_task
|
34
|
+
log_info("Starting Reaper Resurrector")
|
35
|
+
task.execute
|
36
|
+
task
|
37
|
+
end
|
38
|
+
|
39
|
+
#
|
40
|
+
# The task that runs the resurrector
|
41
|
+
#
|
42
|
+
# @return [SidekiqUniqueJobs::TimerTask]
|
43
|
+
def task
|
44
|
+
SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
|
45
|
+
with_logging_context do
|
46
|
+
restart_if_dead
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
#
|
52
|
+
# Starts new instance of orphan reaper if reaper is considered dead (reaper mutex has not been refreshed lately)
|
53
|
+
#
|
54
|
+
def restart_if_dead
|
55
|
+
return if reaper_registered?
|
56
|
+
|
57
|
+
log_info("Reaper is considered dead. Starting new reaper instance")
|
58
|
+
orphans_manager.start
|
59
|
+
end
|
60
|
+
|
61
|
+
#
|
62
|
+
# Returns orphan manager
|
63
|
+
#
|
64
|
+
# @return [SidekiqUniqueJobs::Orphans::Manager]
|
65
|
+
def orphans_manager
|
66
|
+
SidekiqUniqueJobs::Orphans::Manager
|
67
|
+
end
|
68
|
+
|
69
|
+
#
|
70
|
+
# Checks if resurrector is disabled
|
71
|
+
#
|
72
|
+
# @see resurrector_enabled?
|
73
|
+
#
|
74
|
+
# @return [true, false]
|
75
|
+
def resurrector_disabled?
|
76
|
+
!resurrector_enabled?
|
77
|
+
end
|
78
|
+
|
79
|
+
#
|
80
|
+
# Checks if resurrector is enabled
|
81
|
+
#
|
82
|
+
# @return [true, false]
|
83
|
+
def resurrector_enabled?
|
84
|
+
SidekiqUniqueJobs.config.reaper_resurrector_enabled
|
85
|
+
end
|
86
|
+
|
87
|
+
#
|
88
|
+
# Checks if reaping is disabled
|
89
|
+
#
|
90
|
+
# @see reaper_enabled?
|
91
|
+
#
|
92
|
+
# @return [true, false]
|
93
|
+
#
|
94
|
+
def reaper_disabled?
|
95
|
+
!reaper_enabled?
|
96
|
+
end
|
97
|
+
|
98
|
+
#
|
99
|
+
# Checks if reaping is enabled
|
100
|
+
#
|
101
|
+
# @return [true, false]
|
102
|
+
#
|
103
|
+
def reaper_enabled?
|
104
|
+
REAPERS.include?(reaper)
|
105
|
+
end
|
106
|
+
|
107
|
+
#
|
108
|
+
# Checks if reaper is registered
|
109
|
+
#
|
110
|
+
# @return [true, false]
|
111
|
+
def reaper_registered?
|
112
|
+
redis do |conn|
|
113
|
+
conn.get(UNIQUE_REAPER).to_i + drift_reaper_interval > current_timestamp
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
#
|
118
|
+
# @see SidekiqUniqueJobs::Config#reaper
|
119
|
+
#
|
120
|
+
def reaper
|
121
|
+
SidekiqUniqueJobs.config.reaper
|
122
|
+
end
|
123
|
+
|
124
|
+
#
|
125
|
+
# Arguments passed on to the timer task
|
126
|
+
#
|
127
|
+
#
|
128
|
+
# @return [Hash]
|
129
|
+
#
|
130
|
+
def timer_task_options
|
131
|
+
{ run_now: false,
|
132
|
+
execution_interval: reaper_resurrector_interval }
|
133
|
+
end
|
134
|
+
|
135
|
+
#
|
136
|
+
# A context to use for all log entries
|
137
|
+
#
|
138
|
+
#
|
139
|
+
# @return [Hash] when logger responds to `:with_context`
|
140
|
+
# @return [String] when logger does not responds to `:with_context`
|
141
|
+
#
|
142
|
+
def logging_context
|
143
|
+
if logger_context_hash?
|
144
|
+
{ "uniquejobs" => "reaper-resurrector" }
|
145
|
+
else
|
146
|
+
"uniquejobs=reaper-resurrector"
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
#
|
151
|
+
# @see SidekiqUniqueJobs::Config#reaper_resurrector_interval
|
152
|
+
#
|
153
|
+
def reaper_resurrector_interval
|
154
|
+
SidekiqUniqueJobs.config.reaper_resurrector_interval
|
155
|
+
end
|
156
|
+
|
157
|
+
def reaper_interval
|
158
|
+
SidekiqUniqueJobs.config.reaper_interval
|
159
|
+
end
|
160
|
+
|
161
|
+
def drift_reaper_interval
|
162
|
+
reaper_interval + (reaper_interval * DRIFT_FACTOR).to_i
|
163
|
+
end
|
164
|
+
|
165
|
+
def current_timestamp
|
166
|
+
Time.now.to_i
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
@@ -182,7 +182,7 @@ module SidekiqUniqueJobs
|
|
182
182
|
page_size = 50
|
183
183
|
|
184
184
|
loop do
|
185
|
-
range_start =
|
185
|
+
range_start = page * page_size - deleted_size
|
186
186
|
range_end = range_start + page_size - 1
|
187
187
|
entries = conn.lrange(queue_key, range_start, range_end)
|
188
188
|
page += 1
|