wurk 1.11.0 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/exe/wurk +4 -2
- data/lib/generators/wurk/install/templates/wurk.rb +5 -3
- data/lib/wurk/boot_checks.rb +130 -5
- data/lib/wurk/cli.rb +72 -10
- data/lib/wurk/configuration.rb +12 -3
- data/lib/wurk/cron.rb +66 -10
- data/lib/wurk/fetcher/capped.rb +8 -3
- data/lib/wurk/fetcher/oom_drain.rb +0 -5
- data/lib/wurk/fetcher/orphan_reclaim.rb +71 -0
- data/lib/wurk/fetcher/reaper.rb +18 -56
- data/lib/wurk/fetcher/self_reconcile.rb +8 -14
- data/lib/wurk/job_util.rb +18 -2
- data/lib/wurk/launcher.rb +7 -4
- data/lib/wurk/lua/reclaim_orphan.lua +31 -0
- data/lib/wurk/lua.rb +5 -20
- data/lib/wurk/middleware/poison_pill.rb +31 -47
- data/lib/wurk/rails_boot.rb +40 -11
- data/lib/wurk/unique.rb +12 -2
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/views/_job_info.erb +16 -0
- data/lib/wurk/web/views/_paging.erb +11 -0
- data/lib/wurk.rb +4 -0
- data/vendor/assets/dashboard/assets/{ArgsValue-CmaXArv8.js → ArgsValue-BJol9S0s.js} +1 -1
- data/vendor/assets/dashboard/assets/{BatchDetail-nc5yePZW.js → BatchDetail-D_zGQ9Sd.js} +1 -1
- data/vendor/assets/dashboard/assets/{Batches-DgEcyOZ_.js → Batches-Dt_xC660.js} +1 -1
- data/vendor/assets/dashboard/assets/{Busy-CUZncvCy.js → Busy-CWRthsmc.js} +1 -1
- data/vendor/assets/dashboard/assets/{Cron-CIZbaaZs.js → Cron-qeGr2YAm.js} +1 -1
- data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +1 -0
- data/vendor/assets/dashboard/assets/{Extension-Doy4bJoA.js → Extension-DONXVLdA.js} +1 -1
- data/vendor/assets/dashboard/assets/{FilterBox-C4bO26_d.js → FilterBox-BbB6gXHp.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowDetail-Dh3SJdFh.js → FlowDetail-CrmnVRp0.js} +1 -1
- data/vendor/assets/dashboard/assets/{Flows-C5SMRuhq.js → Flows-g4GC8KmE.js} +1 -1
- data/vendor/assets/dashboard/assets/{JobDetailModal-Cg6sAsJB.js → JobDetailModal-C86k_1wj.js} +1 -1
- data/vendor/assets/dashboard/assets/{Limiters-BIxwDOJM.js → Limiters-zyO-iMlK.js} +1 -1
- data/vendor/assets/dashboard/assets/{Metrics-Qp9q0nF8.js → Metrics-Ckq5fNBv.js} +1 -1
- data/vendor/assets/dashboard/assets/{NotFound-DwjPxFjU.js → NotFound-C8NnyjlY.js} +1 -1
- data/vendor/assets/dashboard/assets/{Profiles-7dabYVe3.js → Profiles-DRl-FXPA.js} +1 -1
- data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +1 -0
- data/vendor/assets/dashboard/assets/{Search-BAsVE2qw.js → Search-LaEpj2fM.js} +1 -1
- data/vendor/assets/dashboard/assets/{charts-DZs138BV.js → charts-D_PQLoyc.js} +1 -1
- data/vendor/assets/dashboard/assets/index-FBKGtysy.js +141 -0
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-DxkLZCIm.js → useResetPageOnEmpty-BiwhHqgh.js} +1 -1
- data/vendor/assets/dashboard/assets/{utils-B0kQFNE8.js → utils-CnWZzW2L.js} +1 -1
- data/vendor/assets/dashboard/index.html +1 -1
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +29 -25
- data/vendor/assets/dashboard/assets/Dashboard-BWmfAook.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-CGKsTZHq.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-CNX8ZtTH.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-ClVBj0nc.js +0 -1
- data/vendor/assets/dashboard/assets/index-CMzPFxNE.js +0 -141
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../keys'
|
|
4
|
+
require_relative '../lua'
|
|
5
|
+
require_relative '../middleware/poison_pill'
|
|
6
|
+
require_relative '../metrics/process_counters'
|
|
7
|
+
require_relative 'oom_drain'
|
|
8
|
+
|
|
9
|
+
module Wurk
|
|
10
|
+
class Fetcher
|
|
11
|
+
# Puts one payload from a private list back where it belongs, for both ways
|
|
12
|
+
# an orphan is found: the Reaper draining a dead owner's list, and
|
|
13
|
+
# SelfReconcile returning a payload its own live process lost track of.
|
|
14
|
+
#
|
|
15
|
+
# The poison-pill verdict is taken in the same script that moves the job
|
|
16
|
+
# (`lua/reclaim_orphan.lua`), so a job past the cap goes straight from the
|
|
17
|
+
# private list to the dead set and is never visible on its public queue. The
|
|
18
|
+
# notifications (death handlers, `super_fetch!`, `on_poison`) follow once
|
|
19
|
+
# the script has returned, for a job no worker can be running.
|
|
20
|
+
class OrphanReclaim
|
|
21
|
+
THRESHOLD = Middleware::PoisonPill::RECOVERY_THRESHOLD
|
|
22
|
+
TTL = Middleware::PoisonPill::RECOVERY_TTL
|
|
23
|
+
|
|
24
|
+
def initialize(config)
|
|
25
|
+
@config = config
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Returns true when this call took the payload off the private list, false
|
|
29
|
+
# when it had already left (another reaper, an ACK).
|
|
30
|
+
#
|
|
31
|
+
# Unclaimed apply-safety: the move is guarded, but a reply lost after a
|
|
32
|
+
# poison kill would leave the death handlers unfired, and a list holding
|
|
33
|
+
# two identical copies would move the second on a replay. A raise leaves
|
|
34
|
+
# the payload for the next pass instead.
|
|
35
|
+
def call(private_list, public_q, payload)
|
|
36
|
+
job = Middleware::PoisonPill.parse(payload)
|
|
37
|
+
count = @config.redis { |conn| move(conn, private_list, public_q, payload, job) }
|
|
38
|
+
return false unless count
|
|
39
|
+
|
|
40
|
+
Metrics::ProcessCounters.increment(:reaper_reclaimed)
|
|
41
|
+
unless count == :uncounted
|
|
42
|
+
Middleware::PoisonPill.recovered!(payload, job, count: count, queue: queue_name(public_q),
|
|
43
|
+
config: @config)
|
|
44
|
+
end
|
|
45
|
+
true
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
# Past maxmemory the counter cannot be written, nor the job killed: it goes
|
|
51
|
+
# back uncounted rather than waiting on the memory it would help free.
|
|
52
|
+
def move(conn, private_list, public_q, payload, job)
|
|
53
|
+
keys = [private_list, public_q, Keys::DEAD]
|
|
54
|
+
jid = job && job['jid']
|
|
55
|
+
keys << Middleware::PoisonPill.counter_key(jid) if jid.is_a?(String) && !jid.empty?
|
|
56
|
+
Wurk::Lua::Loader.eval_cached(conn, :reclaim_orphan, keys: keys, argv: [payload, THRESHOLD, TTL, now])
|
|
57
|
+
rescue RedisClient::OutOfMemoryError => e
|
|
58
|
+
:uncounted if OomDrain.requeue(conn, e, private_list, public_q, payload) == 1
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# DeadSet#kill's score format, so the entry sorts with every other kill.
|
|
62
|
+
def now
|
|
63
|
+
::Process.clock_gettime(::Process::CLOCK_REALTIME).to_s
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def queue_name(public_q)
|
|
67
|
+
public_q.delete_prefix(Keys::QUEUE_PREFIX)
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
data/lib/wurk/fetcher/reaper.rb
CHANGED
|
@@ -2,13 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative '../component'
|
|
4
4
|
require_relative '../keys'
|
|
5
|
-
require_relative '../middleware/poison_pill'
|
|
6
|
-
require_relative '../metrics/process_counters'
|
|
7
5
|
require_relative '../timer_loop'
|
|
8
6
|
require_relative 'private_list_key'
|
|
9
7
|
require_relative 'unparseable_keys'
|
|
10
8
|
require_relative 'orphan_grace'
|
|
11
9
|
require_relative 'oom_drain'
|
|
10
|
+
require_relative 'orphan_reclaim'
|
|
12
11
|
require_relative 'self_reconcile'
|
|
13
12
|
|
|
14
13
|
module Wurk
|
|
@@ -45,10 +44,10 @@ module Wurk
|
|
|
45
44
|
# process's own clock (OrphanGrace), and its identity must still have no
|
|
46
45
|
# `info` hash. Our own incarnation skips both: kill(0) is not a snapshot.
|
|
47
46
|
#
|
|
48
|
-
#
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
# its worker every time can't loop forever.
|
|
47
|
+
# Every move goes through OrphanReclaim, which caps a job at
|
|
48
|
+
# Middleware::PoisonPill::RECOVERY_THRESHOLD recoveries within 72h: past the
|
|
49
|
+
# cap the job goes from the private list straight to the dead set instead of
|
|
50
|
+
# its queue, so a job that crashes its worker every time can't loop forever.
|
|
52
51
|
#
|
|
53
52
|
# One sweep an interval: one SCAN of `queue:*|*` acting on every list, gated
|
|
54
53
|
# by a cluster lock so about one process sweeps per interval. Every list, and
|
|
@@ -62,7 +61,7 @@ module Wurk
|
|
|
62
61
|
# of a job.
|
|
63
62
|
#
|
|
64
63
|
# Spec: docs/target/sidekiq-pro.md §3.2.
|
|
65
|
-
class Reaper
|
|
64
|
+
class Reaper
|
|
66
65
|
include Component
|
|
67
66
|
|
|
68
67
|
# Sweep cadence in seconds. 60s matches the heartbeat TTL — the floor below
|
|
@@ -103,8 +102,8 @@ module Wurk
|
|
|
103
102
|
@sleeper = ::ConditionVariable.new
|
|
104
103
|
@unparseable = UnparseableKeys.new(config)
|
|
105
104
|
@next_sweep_at = 0.0
|
|
106
|
-
@
|
|
107
|
-
|
|
105
|
+
@orphan_reclaim = OrphanReclaim.new(config)
|
|
106
|
+
@self_reconcile = SelfReconcile.new(config, confirm: reconcile_confirm, stop: -> { done? })
|
|
108
107
|
end
|
|
109
108
|
|
|
110
109
|
# Spawns the sweep loop. Idempotent. The loop waits one interval before
|
|
@@ -313,27 +312,19 @@ module Wurk
|
|
|
313
312
|
identity.split(':')[0..1].join(':')
|
|
314
313
|
end
|
|
315
314
|
|
|
316
|
-
# Drain one orphaned private list back to its public queue
|
|
317
|
-
#
|
|
318
|
-
#
|
|
319
|
-
#
|
|
320
|
-
#
|
|
315
|
+
# Drain one orphaned private list back to its public queue, a job at a
|
|
316
|
+
# time from the tail: peek it, then OrphanReclaim moves it, to the public
|
|
317
|
+
# tail or, past the poison cap, to the dead set. The decision is taken
|
|
318
|
+
# before the job is visible, so a worker parked in BLMOVE can never take a
|
|
319
|
+
# job that is being killed. A peeked job that has left by the time it is
|
|
320
|
+
# moved (another reaper drained it) costs one more peek.
|
|
321
321
|
#
|
|
322
|
-
#
|
|
323
|
-
#
|
|
324
|
-
# one already on the public tail, so a job that kills its worker every time
|
|
325
|
-
# would get a free recovery past the cap. A raise instead lands in the
|
|
326
|
-
# rescue below, and the next sweep re-drains what's left.
|
|
322
|
+
# A raise lands in the rescue below; the job stays in the private list
|
|
323
|
+
# and the next sweep re-drains what's left.
|
|
327
324
|
def drain(private_list, public_q)
|
|
328
|
-
queue_name = public_q.delete_prefix(Keys::QUEUE_PREFIX)
|
|
329
325
|
count = 0
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
break unless job
|
|
333
|
-
|
|
334
|
-
count += 1
|
|
335
|
-
Metrics::ProcessCounters.increment(:reaper_reclaimed)
|
|
336
|
-
poison_off(public_q, job, queue_name)
|
|
326
|
+
while (job = redis(idempotent: true) { |c| c.call('LINDEX', private_list, -1) })
|
|
327
|
+
count += 1 if @orphan_reclaim.call(private_list, public_q, job)
|
|
337
328
|
end
|
|
338
329
|
count
|
|
339
330
|
rescue StandardError => e
|
|
@@ -341,35 +332,6 @@ module Wurk
|
|
|
341
332
|
count
|
|
342
333
|
end
|
|
343
334
|
|
|
344
|
-
# A full Redis must not hold a dead worker's jobs back from the workers
|
|
345
|
-
# still able to drain the queue.
|
|
346
|
-
def reclaim_one(conn, private_list, public_q)
|
|
347
|
-
Wurk::Lua::Loader.eval_cached(conn, :reliable_reclaim, keys: [private_list, public_q], argv: [])
|
|
348
|
-
rescue RedisClient::OutOfMemoryError => e
|
|
349
|
-
OomDrain.reclaim(conn, e, private_list, public_q)
|
|
350
|
-
end
|
|
351
|
-
|
|
352
|
-
# SelfReconcile put a payload back: the same accounting as a reclaim, so a
|
|
353
|
-
# job that kills its processor thread every time still hits the poison cap.
|
|
354
|
-
def recovered(public_q, job)
|
|
355
|
-
Metrics::ProcessCounters.increment(:reaper_reclaimed)
|
|
356
|
-
poison_off(public_q, job, public_q.delete_prefix(Keys::QUEUE_PREFIX))
|
|
357
|
-
end
|
|
358
|
-
|
|
359
|
-
# The job is back on its queue whatever happens here. Past maxmemory the
|
|
360
|
-
# count can't be taken (nor the job killed, leaving it where it is): raising
|
|
361
|
-
# would end the drain at one job per pass, with the rest of the list waiting
|
|
362
|
-
# on memory they would help free.
|
|
363
|
-
def poison_off(public_q, job, queue_name)
|
|
364
|
-
return unless Middleware::PoisonPill.track!(job, queue: queue_name, config: @config) == :poison
|
|
365
|
-
|
|
366
|
-
# track! already ZADDed the payload to the dead set; pull the copy we
|
|
367
|
-
# just moved onto the public tail so it isn't also re-run.
|
|
368
|
-
redis { |c| c.call('LREM', public_q, -1, job) }
|
|
369
|
-
rescue RedisClient::OutOfMemoryError => e
|
|
370
|
-
report(e)
|
|
371
|
-
end
|
|
372
|
-
|
|
373
335
|
def acquire_lock?
|
|
374
336
|
ttl = (@interval * LOCK_TTL_RATIO * 1000).ceil
|
|
375
337
|
redis do |c|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative '../keys'
|
|
4
|
-
require_relative '
|
|
4
|
+
require_relative 'orphan_reclaim'
|
|
5
5
|
|
|
6
6
|
module Wurk
|
|
7
7
|
class Fetcher
|
|
@@ -34,13 +34,14 @@ module Wurk
|
|
|
34
34
|
|
|
35
35
|
Suspect = Struct.new(:copies, :since)
|
|
36
36
|
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
|
|
37
|
+
# A payload put back goes through the same poison accounting as a reclaim
|
|
38
|
+
# (OrphanReclaim), so a job that kills its processor thread every time
|
|
39
|
+
# still hits the poison cap.
|
|
40
|
+
def initialize(config, confirm: DEFAULT_CONFIRM, stop: -> { false })
|
|
40
41
|
@config = config
|
|
41
42
|
@confirm = confirm
|
|
42
43
|
@stop = stop
|
|
43
|
-
@
|
|
44
|
+
@orphan_reclaim = OrphanReclaim.new(config)
|
|
44
45
|
@suspects = {}
|
|
45
46
|
end
|
|
46
47
|
|
|
@@ -154,16 +155,9 @@ module Wurk
|
|
|
154
155
|
moved
|
|
155
156
|
end
|
|
156
157
|
|
|
157
|
-
# LPOS-guarded, so a
|
|
158
|
-
# time, and a payload settled since the last read stays settled.
|
|
158
|
+
# LPOS-guarded, so a payload settled since the last read stays settled.
|
|
159
159
|
def requeue(list, public_q, payload)
|
|
160
|
-
|
|
161
|
-
Wurk::Lua::Loader.eval_cached(conn, :reliable_requeue, keys: [list, public_q], argv: [payload])
|
|
162
|
-
rescue RedisClient::OutOfMemoryError => e
|
|
163
|
-
OomDrain.requeue(conn, e, list, public_q, payload)
|
|
164
|
-
end.to_i
|
|
165
|
-
@on_moved&.call(public_q, payload) if applied == 1
|
|
166
|
-
applied
|
|
160
|
+
@orphan_reclaim.call(list, public_q, payload) ? 1 : 0
|
|
167
161
|
end
|
|
168
162
|
|
|
169
163
|
def forget_all
|
data/lib/wurk/job_util.rb
CHANGED
|
@@ -117,6 +117,17 @@ module Wurk
|
|
|
117
117
|
finalize(normalized)
|
|
118
118
|
end
|
|
119
119
|
|
|
120
|
+
# An ActiveSupport::Duration (`48.hours`) unwrapped to the plain number it
|
|
121
|
+
# stands for; anything else comes back as given. The Duration claims
|
|
122
|
+
# `is_a?(Numeric)` through its own override, but `case`/`when` (Module#===)
|
|
123
|
+
# does not ask it, and JSON.generate writes it as a String ("172800"), which
|
|
124
|
+
# no reader after the push can use as seconds.
|
|
125
|
+
def self.plain_seconds(value)
|
|
126
|
+
return value if !value.is_a?(::Numeric) || value.class <= ::Numeric
|
|
127
|
+
|
|
128
|
+
value.is_a?(::Integer) ? value.to_i : value.to_f
|
|
129
|
+
end
|
|
130
|
+
|
|
120
131
|
def now_in_millis
|
|
121
132
|
::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
|
|
122
133
|
end
|
|
@@ -166,7 +177,7 @@ module Wurk
|
|
|
166
177
|
end
|
|
167
178
|
|
|
168
179
|
def numeric_retry_for(value)
|
|
169
|
-
case value
|
|
180
|
+
case (value = JobUtil.plain_seconds(value))
|
|
170
181
|
when Integer then value
|
|
171
182
|
when Numeric then (value.to_i if value.finite? && value.real?)
|
|
172
183
|
when String then (Integer(value, 10) if value.match?(/\A-?\d+\z/))
|
|
@@ -206,13 +217,18 @@ module Wurk
|
|
|
206
217
|
def finalize(normalized)
|
|
207
218
|
TRANSIENT_ATTRIBUTES.each { |k| normalized.delete(k) }
|
|
208
219
|
normalized['jid'] ||= SecureRandom.hex(12)
|
|
209
|
-
|
|
220
|
+
coerce_seconds!(normalized)
|
|
210
221
|
normalized['created_at'] ||= now_in_millis
|
|
211
222
|
stamp_expiry(normalized)
|
|
212
223
|
stamp_deadline(normalized)
|
|
213
224
|
normalized
|
|
214
225
|
end
|
|
215
226
|
|
|
227
|
+
def coerce_seconds!(item)
|
|
228
|
+
item['retry_for'] = numeric_retry_for(item['retry_for']) if item.key?('retry_for')
|
|
229
|
+
BOUND_OPTIONS.each { |k| item[k] = JobUtil.plain_seconds(item[k]) if item.key?(k) }
|
|
230
|
+
end
|
|
231
|
+
|
|
216
232
|
# Pro `expires_in:` → absolute epoch-float `expiry` resolved once at push,
|
|
217
233
|
# so the server middleware doesn't redo the math. Spec: sidekiq-pro.md §7.
|
|
218
234
|
# nil.respond_to?(:to_f) is true on modern Ruby (returns 0.0), so we must
|
data/lib/wurk/launcher.rb
CHANGED
|
@@ -99,10 +99,12 @@ module Wurk
|
|
|
99
99
|
# 4. start the reaper and the boot-time reclaim sweep — kill-9 recovery,
|
|
100
100
|
# not decoration: a SIGKILLed sibling's in-flight jobs wait on this,
|
|
101
101
|
# so it stays ahead of the periodic loops.
|
|
102
|
-
# 5.
|
|
103
|
-
#
|
|
104
|
-
#
|
|
105
|
-
#
|
|
102
|
+
# 5. write the `config.periodic` registrations (Ent registers loops on
|
|
103
|
+
# startup), then start the periodic loops. None of them ticks at
|
|
104
|
+
# zero — TimerLoop#run waits an interval before its first yield,
|
|
105
|
+
# Scheduled::Poller waits INITIAL_WAIT, and Leader waits
|
|
106
|
+
# DEFAULT_INITIAL_WAIT — so starting them last costs a few thread
|
|
107
|
+
# spawns and nothing else. Both leader-
|
|
106
108
|
# gated pollers are also safe to start before leadership settles: a
|
|
107
109
|
# non-leader tick just returns early.
|
|
108
110
|
# 6. start the health probe server LAST so the listener doesn't
|
|
@@ -240,6 +242,7 @@ module Wurk
|
|
|
240
242
|
# Run on a background thread so /ready probe isn't delayed by a large
|
|
241
243
|
# orphan sweep (the reaper's reclaim is atomic, but can scan many entries).
|
|
242
244
|
@boot_reclaim_thread = safe_thread('boot-reclaim', &method(:boot_reclaim))
|
|
245
|
+
@config.apply_periodic!
|
|
243
246
|
start_periodic_components
|
|
244
247
|
end
|
|
245
248
|
@health_server&.start
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
-- Takes one orphaned payload off a private list and decides, before anyone can
|
|
2
|
+
-- see it, where it goes: back onto its public queue's fetch end, or into the
|
|
3
|
+
-- dead set once the poison-pill counter passes the threshold. Pushing first and
|
|
4
|
+
-- counting after handed a poison job to a worker parked in BLMOVE, then killed
|
|
5
|
+
-- it into the dead set while it ran (Fetcher::OrphanReclaim).
|
|
6
|
+
--
|
|
7
|
+
-- The payload is located from the tail, where the reaper peeked it. Gone means
|
|
8
|
+
-- someone else moved it, and nothing is touched. Push before remove, as in the
|
|
9
|
+
-- requeue scripts, and the counter last: a write refused part way through must
|
|
10
|
+
-- leave the job where it was, and a refused counter only leaves it short.
|
|
11
|
+
-- SET EX is INCR + EXPIRE in one write, so the counter never lacks its TTL.
|
|
12
|
+
-- KEYS = [private_list, public_queue, dead, counter (only for a payload with a jid)]
|
|
13
|
+
-- ARGV = [payload, threshold, ttl_seconds, now]
|
|
14
|
+
-- Returns the recovery count (0 without a counter), nil when the payload is gone.
|
|
15
|
+
if not redis.call("lpos", KEYS[1], ARGV[1], "RANK", -1) then
|
|
16
|
+
return nil
|
|
17
|
+
end
|
|
18
|
+
local count = 0
|
|
19
|
+
if KEYS[4] then
|
|
20
|
+
count = (tonumber(redis.call("get", KEYS[4])) or 0) + 1
|
|
21
|
+
end
|
|
22
|
+
if count > tonumber(ARGV[2]) then
|
|
23
|
+
redis.call("zadd", KEYS[3], ARGV[4], ARGV[1])
|
|
24
|
+
else
|
|
25
|
+
redis.call("rpush", KEYS[2], ARGV[1])
|
|
26
|
+
end
|
|
27
|
+
redis.call("lrem", KEYS[1], -1, ARGV[1])
|
|
28
|
+
if KEYS[4] then
|
|
29
|
+
redis.call("set", KEYS[4], count, "EX", ARGV[3])
|
|
30
|
+
end
|
|
31
|
+
return count
|
data/lib/wurk/lua.rb
CHANGED
|
@@ -128,10 +128,11 @@ module Wurk
|
|
|
128
128
|
# RPUSH (public tail) not LPUSH so the reclaimed job is fetched next — LMOVE
|
|
129
129
|
# pops the tail — ahead of fresh LPUSH'd enqueues.
|
|
130
130
|
#
|
|
131
|
-
# Push before remove, in this script
|
|
132
|
-
# script back, and a server can
|
|
133
|
-
#
|
|
134
|
-
# the job where it was,
|
|
131
|
+
# Push before remove, in this script, the one below and
|
|
132
|
+
# `lua/reclaim_orphan.lua`. Nothing rolls a script back, and a server can
|
|
133
|
+
# refuse the write mid-script (Dragonfly at maxmemory refuses the RPUSH
|
|
134
|
+
# after applying an LREM): the raise has to leave the job where it was,
|
|
135
|
+
# never in neither place.
|
|
135
136
|
# KEYS = [private_list, public_queue]
|
|
136
137
|
# ARGV = [job_json]
|
|
137
138
|
# Returns 1 when the job was moved, 0 when it was already acked.
|
|
@@ -154,21 +155,6 @@ module Wurk
|
|
|
154
155
|
return redis.call("lrem", KEYS[1], 1, ARGV[1])
|
|
155
156
|
LUA
|
|
156
157
|
|
|
157
|
-
# The reaper's move off a dead worker's private list: LMOVE private public
|
|
158
|
-
# RIGHT RIGHT, spelled out because Dragonfly at maxmemory answers an LMOVE
|
|
159
|
-
# between shards with the job having popped it and dropped the push. The
|
|
160
|
-
# reply would be the only copy left, and the reaper has no use for it.
|
|
161
|
-
# KEYS = [private_list, public_queue]
|
|
162
|
-
# Returns the job moved, nil when the list is empty.
|
|
163
|
-
RELIABLE_RECLAIM = <<~LUA
|
|
164
|
-
local job = redis.call("lindex", KEYS[1], -1)
|
|
165
|
-
if job then
|
|
166
|
-
redis.call("rpush", KEYS[2], job)
|
|
167
|
-
redis.call("rpop", KEYS[1])
|
|
168
|
-
end
|
|
169
|
-
return job
|
|
170
|
-
LUA
|
|
171
|
-
|
|
172
158
|
# Ent Unique (§3): atomic compare-and-delete of a lock key. Replaces the
|
|
173
159
|
# two-command GET-then-DEL — between those calls the key can expire and a
|
|
174
160
|
# fresh owner can grab it, and the bare DEL would then drop the new
|
|
@@ -276,7 +262,6 @@ module Wurk
|
|
|
276
262
|
scheduled_claim: SCHEDULED_CLAIM,
|
|
277
263
|
reliable_requeue: RELIABLE_REQUEUE,
|
|
278
264
|
requeue_unit: REQUEUE_UNIT,
|
|
279
|
-
reliable_reclaim: RELIABLE_RECLAIM,
|
|
280
265
|
fast_delete_job: FAST_DELETE_JOB,
|
|
281
266
|
fast_delete_by_class: FAST_DELETE_BY_CLASS,
|
|
282
267
|
release_if_owner: RELEASE_IF_OWNER,
|
|
@@ -9,19 +9,20 @@ require_relative '../dead_set'
|
|
|
9
9
|
module Wurk
|
|
10
10
|
module Middleware
|
|
11
11
|
# Pro parity (§3.2): poison-pill detection for reliable-fetch orphans.
|
|
12
|
-
# When a job is recovered out of a
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
16
|
-
# `jobs.poison` is emitted to statsd, and recovery callbacks fire so
|
|
12
|
+
# When a job is recovered out of a private list, a per-jid counter at
|
|
13
|
+
# `super_fetch:recovered:<jid>` goes up by one with a 72h TTL. Recoveries up
|
|
14
|
+
# to RECOVERY_THRESHOLD (3) requeue; the one that takes the counter past it
|
|
15
|
+
# (the 4th orphaning) is the poison pill: the payload is moved to the dead
|
|
16
|
+
# set, `jobs.poison` is emitted to statsd, and recovery callbacks fire so
|
|
17
17
|
# operators can be paged.
|
|
18
18
|
#
|
|
19
19
|
# Counter key TTL is wire-compat with Sidekiq Pro — third-party tooling
|
|
20
20
|
# that watches `super_fetch:recovered:*` expects 72h.
|
|
21
21
|
#
|
|
22
|
-
# No server-middleware registration: the producer is
|
|
23
|
-
#
|
|
24
|
-
#
|
|
22
|
+
# No server-middleware registration: the producer is Fetcher::OrphanReclaim,
|
|
23
|
+
# whose script bumps the counter and takes the requeue-or-kill decision in
|
|
24
|
+
# the same step as the move, then hands the outcome to {recovered!} for the
|
|
25
|
+
# notifications. The consumer is the ACK —
|
|
25
26
|
# `Fetcher::Reliable::UnitOfWork#acknowledge` pipelines {clear_in} next to
|
|
26
27
|
# its LREM, so an attempt that finished starts the next one at zero
|
|
27
28
|
# without costing a round trip of its own.
|
|
@@ -44,37 +45,34 @@ module Wurk
|
|
|
44
45
|
|
|
45
46
|
module_function
|
|
46
47
|
|
|
47
|
-
# Called per recovered orphan job
|
|
48
|
-
#
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
#
|
|
48
|
+
# Called per recovered orphan job, once Fetcher::OrphanReclaim has moved
|
|
49
|
+
# it: `count` is the recovery counter its script wrote (0 for a payload with
|
|
50
|
+
# no jid). Returns `:poison` when the count is past the threshold — the
|
|
51
|
+
# script already put the job in the dead set, never on its queue — and
|
|
52
|
+
# `:recovered` otherwise. Emits `jobs.recovered.fetch` for every parseable
|
|
53
|
+
# job, `jobs.poison` only on the kill path.
|
|
52
54
|
#
|
|
53
55
|
# Fires the Pro `super_fetch!` recovery callback (config.super_fetch_callback)
|
|
54
56
|
# exactly once per call: `(jobstr, nil)` on plain recovery, `(jobstr, pill)`
|
|
55
57
|
# on the kill path. The poison-only `on_poison` Hash callbacks fire
|
|
56
58
|
# independently inside #mark_poison.
|
|
57
59
|
#
|
|
58
|
-
# @param payload [String
|
|
60
|
+
# @param payload [String] the job JSON as it sat in the private list.
|
|
61
|
+
# @param job [Hash, nil] the payload parsed by {parse}.
|
|
62
|
+
# @param count [Integer] the recovery count after this recovery.
|
|
59
63
|
# @param queue [String, nil] the public queue name (without `queue:`).
|
|
60
|
-
# @param config [Configuration] config that owns the super_fetch! callback
|
|
61
|
-
# defaults to the global so non-reaper callers (tests) need not pass it.
|
|
64
|
+
# @param config [Configuration] config that owns the super_fetch! callback.
|
|
62
65
|
# @return [Symbol] :recovered | :poison
|
|
63
|
-
def
|
|
64
|
-
job = parse(payload)
|
|
66
|
+
def recovered!(payload, job, count:, queue: nil, config: Wurk.configuration)
|
|
65
67
|
unless job
|
|
66
68
|
fire_super_fetch(config, payload, nil)
|
|
67
69
|
return :recovered
|
|
68
70
|
end
|
|
69
71
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
emit('jobs.recovered.fetch', klass, queue)
|
|
73
|
-
|
|
74
|
-
count = bump_counter(jid) if jid && !jid.empty?
|
|
75
|
-
if count && count > RECOVERY_THRESHOLD
|
|
72
|
+
emit('jobs.recovered.fetch', job['class'], queue)
|
|
73
|
+
if count > RECOVERY_THRESHOLD
|
|
76
74
|
mark_poison(payload, job, queue: queue, count: count)
|
|
77
|
-
fire_super_fetch(config, payload, Pill.new(jid: jid, klass:
|
|
75
|
+
fire_super_fetch(config, payload, Pill.new(jid: job['jid'], klass: job['class'], count: count, queue: queue))
|
|
78
76
|
:poison
|
|
79
77
|
else
|
|
80
78
|
fire_super_fetch(config, payload, nil)
|
|
@@ -168,29 +166,14 @@ module Wurk
|
|
|
168
166
|
|
|
169
167
|
# ---- internals --------------------------------------------------
|
|
170
168
|
|
|
169
|
+
# The job Hash, or nil for anything that is not a JSON object.
|
|
171
170
|
def parse(payload)
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
Wurk.load_json(payload)
|
|
171
|
+
job = Wurk.load_json(payload)
|
|
172
|
+
job if job.is_a?(::Hash)
|
|
176
173
|
rescue ::JSON::ParserError
|
|
177
174
|
nil
|
|
178
175
|
end
|
|
179
176
|
|
|
180
|
-
# No apply-safety claim: INCR is additive, so a block replayed after a
|
|
181
|
-
# lost reply bumps twice and can carry a healthy job past
|
|
182
|
-
# RECOVERY_THRESHOLD into the dead set. Raising instead only leaves the
|
|
183
|
-
# count short — Reaper#drain rescues, and the job is already back on its
|
|
184
|
-
# public queue, so it just misses one poison check.
|
|
185
|
-
def bump_counter(jid)
|
|
186
|
-
key = counter_key(jid)
|
|
187
|
-
Wurk.redis do |conn|
|
|
188
|
-
count = conn.call('INCR', key).to_i
|
|
189
|
-
conn.call('EXPIRE', key, RECOVERY_TTL)
|
|
190
|
-
count
|
|
191
|
-
end
|
|
192
|
-
end
|
|
193
|
-
|
|
194
177
|
def emit(metric, klass, queue)
|
|
195
178
|
tags = []
|
|
196
179
|
tags << "class:#{klass}" if klass
|
|
@@ -198,6 +181,9 @@ module Wurk
|
|
|
198
181
|
Wurk::Metrics::Statsd.increment(metric, tags: tags.empty? ? nil : tags)
|
|
199
182
|
end
|
|
200
183
|
|
|
184
|
+
# The script already put `payload` in the dead set; DeadSet#kill writes the
|
|
185
|
+
# same member again, so the trim and the death handlers stay DeadSet's.
|
|
186
|
+
#
|
|
201
187
|
# Death handlers fire (`notify_failure` defaults to true). A poison kill
|
|
202
188
|
# is a death like any other exhaustion: suppressing it strands every
|
|
203
189
|
# batch that owns the job — Batch::DeathHandler is a death handler, so a
|
|
@@ -206,8 +192,7 @@ module Wurk
|
|
|
206
192
|
# (docs/target/sidekiq-pro.md §12); recorded in docs/idea/parity-divergences.md.
|
|
207
193
|
def mark_poison(payload, job, queue:, count:)
|
|
208
194
|
emit('jobs.poison', job['class'], queue)
|
|
209
|
-
|
|
210
|
-
Wurk::DeadSet.new.kill(json, ex: poisoned_error(job['class'], count))
|
|
195
|
+
Wurk::DeadSet.new.kill(payload, ex: poisoned_error(job['class'], count))
|
|
211
196
|
fire_callbacks(jid: job['jid'], klass: job['class'], count: count, queue: queue)
|
|
212
197
|
end
|
|
213
198
|
|
|
@@ -231,8 +216,7 @@ module Wurk
|
|
|
231
216
|
cb = config.super_fetch_callback
|
|
232
217
|
return unless cb
|
|
233
218
|
|
|
234
|
-
|
|
235
|
-
cb.call(jobstr, pill)
|
|
219
|
+
cb.call(payload, pill)
|
|
236
220
|
rescue StandardError => e
|
|
237
221
|
config.handle_exception(e, context: 'Wurk::Middleware::PoisonPill')
|
|
238
222
|
end
|
data/lib/wurk/rails_boot.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative 'rails_reloader'
|
|
4
|
+
require_relative 'boot_checks'
|
|
4
5
|
|
|
5
6
|
module Wurk
|
|
6
7
|
# Coordinates how Wurk boots inside a Rails host: whether this process should
|
|
@@ -60,7 +61,8 @@ module Wurk
|
|
|
60
61
|
|
|
61
62
|
# Invoked last of all initializers, once the host app has fully initialized.
|
|
62
63
|
def boot(app = ::Rails.application)
|
|
63
|
-
|
|
64
|
+
reason = skip_reason
|
|
65
|
+
return log_client_only(reason) if reason
|
|
64
66
|
|
|
65
67
|
case boot_action(app)
|
|
66
68
|
when :fork then boot_swarm
|
|
@@ -79,13 +81,36 @@ module Wurk
|
|
|
79
81
|
# run every job — every cron tick included — twice.
|
|
80
82
|
#
|
|
81
83
|
# Everything above is a denylist; `serving?` is the allowlist that backs it.
|
|
82
|
-
def skip_boot?
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
84
|
+
def skip_boot? = !skip_reason.nil?
|
|
85
|
+
|
|
86
|
+
# Why this process runs no jobs, or nil when it boots them.
|
|
87
|
+
def skip_reason
|
|
88
|
+
return 'WURK_DISABLED=1' if ENV['WURK_DISABLED'] == '1'
|
|
89
|
+
return 'the wurk runner owns this process' if Wurk.worker_boot_claimed?
|
|
90
|
+
return 'a build or rake task' if building?
|
|
91
|
+
return 'Rails console' if defined?(::Rails::Console)
|
|
92
|
+
return 'test environment' if ::Rails.env.test?
|
|
93
|
+
|
|
94
|
+
'not a web server; WURK_EMBED=1 runs jobs here' unless serving?
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# The runner announces its own role; a second line from here would describe
|
|
98
|
+
# the same process twice.
|
|
99
|
+
def log_client_only(reason)
|
|
100
|
+
return if Wurk.worker_boot_claimed?
|
|
101
|
+
|
|
102
|
+
BootChecks.log_client_only(logger, reason)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Named the way #serving? recognised it: a bundled gem defines its constant
|
|
106
|
+
# in processes that are not that server.
|
|
107
|
+
def server_name
|
|
108
|
+
return 'Puma' if puma_launcher_loaded?
|
|
109
|
+
return 'Passenger' if defined?(::PhusionPassenger)
|
|
110
|
+
return 'Unicorn' if unicorn_server?
|
|
111
|
+
return 'rails server' if defined?(::Rails::Server)
|
|
112
|
+
|
|
113
|
+
'this process (WURK_EMBED=1)'
|
|
89
114
|
end
|
|
90
115
|
|
|
91
116
|
# Boot only inside a web server, or where the host opted in. Every Rails
|
|
@@ -183,8 +208,11 @@ module Wurk
|
|
|
183
208
|
def boot_swarm # rubocop:disable Metrics/AbcSize
|
|
184
209
|
warn_on_production_fork
|
|
185
210
|
Health.options_from_env
|
|
186
|
-
|
|
187
|
-
|
|
211
|
+
config = Wurk.configuration
|
|
212
|
+
timeout = config[:timeout] || Swarm::DEFAULT_SHUTDOWN_TIMEOUT
|
|
213
|
+
BootChecks.log_swarm_role(config, "swarm parent in #{server_name}")
|
|
214
|
+
BootChecks.run(config, nil)
|
|
215
|
+
swarm = Wurk::Swarm.new(topology: config.topology, shutdown_timeout: timeout)
|
|
188
216
|
supervisor = nil
|
|
189
217
|
# Registered BEFORE boot, which forks the children one slot at a time: a
|
|
190
218
|
# fork that raises partway through would otherwise leave the children it
|
|
@@ -246,7 +274,8 @@ module Wurk
|
|
|
246
274
|
# already running with nothing to drain them on host exit.
|
|
247
275
|
at_exit { instance.stop }
|
|
248
276
|
instance.run
|
|
249
|
-
|
|
277
|
+
BootChecks.log_process_role(Wurk.configuration, "embedded in #{server_name}, threads only")
|
|
278
|
+
BootChecks.run(Wurk.configuration, nil)
|
|
250
279
|
instance
|
|
251
280
|
rescue StandardError => e
|
|
252
281
|
logger.error { "wurk: embedded boot failed: #{e.class}: #{e.message}" }
|
data/lib/wurk/unique.rb
CHANGED
|
@@ -202,8 +202,14 @@ module Wurk
|
|
|
202
202
|
end
|
|
203
203
|
|
|
204
204
|
# Coerce `unique_for` to a numeric seconds value. Accepts Integer,
|
|
205
|
-
# Numeric, ActiveSupport::Duration (any `to_i`-respondent),
|
|
206
|
-
# (skip). Returns nil when uniqueness should be skipped.
|
|
205
|
+
# Numeric, ActiveSupport::Duration (any `to_i`-respondent), a numeric
|
|
206
|
+
# String, or `false` (skip). Returns nil when uniqueness should be skipped.
|
|
207
|
+
#
|
|
208
|
+
# The String is what every reader after the push sees for a Duration:
|
|
209
|
+
# JSON.generate writes `5.minutes` as "300", in Sidekiq as here, so the
|
|
210
|
+
# server middleware and the death handler get "300" back off the queue.
|
|
211
|
+
# Refusing it left every `unique_for: 5.minutes` lock held until its TTL
|
|
212
|
+
# (tesote/tesote.com#519).
|
|
207
213
|
def self.coerce_ttl(value)
|
|
208
214
|
return nil if value.nil? || value == false
|
|
209
215
|
# `.to_i`, not `value`: ActiveSupport::Duration overrides `is_a?(Integer)`
|
|
@@ -212,11 +218,15 @@ module Wurk
|
|
|
212
218
|
# EX arg and raised TypeError at enqueue (#253).
|
|
213
219
|
return value.to_i if value.is_a?(Integer) && value.positive?
|
|
214
220
|
return value.to_i if value.is_a?(Numeric)
|
|
221
|
+
return value.to_f.to_i if value.is_a?(String) && value.match?(NUMERIC_TTL)
|
|
215
222
|
return value.to_i if duration_like?(value)
|
|
216
223
|
|
|
217
224
|
nil
|
|
218
225
|
end
|
|
219
226
|
|
|
227
|
+
# Duration#to_s: "300" for `5.minutes`, "90.0" for `1.5.minutes`.
|
|
228
|
+
NUMERIC_TTL = /\A\d+(?:\.\d+)?\z/
|
|
229
|
+
|
|
220
230
|
def self.duration_like?(value)
|
|
221
231
|
return false unless value.respond_to?(:to_i)
|
|
222
232
|
|