wurk 1.11.0 → 1.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/exe/wurk +4 -2
  4. data/lib/generators/wurk/install/templates/wurk.rb +5 -3
  5. data/lib/wurk/boot_checks.rb +130 -5
  6. data/lib/wurk/cli.rb +72 -10
  7. data/lib/wurk/configuration.rb +12 -3
  8. data/lib/wurk/cron.rb +66 -10
  9. data/lib/wurk/fetcher/capped.rb +8 -3
  10. data/lib/wurk/fetcher/oom_drain.rb +0 -5
  11. data/lib/wurk/fetcher/orphan_reclaim.rb +71 -0
  12. data/lib/wurk/fetcher/reaper.rb +18 -56
  13. data/lib/wurk/fetcher/self_reconcile.rb +8 -14
  14. data/lib/wurk/job_util.rb +18 -2
  15. data/lib/wurk/launcher.rb +7 -4
  16. data/lib/wurk/lua/reclaim_orphan.lua +31 -0
  17. data/lib/wurk/lua.rb +5 -20
  18. data/lib/wurk/middleware/poison_pill.rb +31 -47
  19. data/lib/wurk/rails_boot.rb +40 -11
  20. data/lib/wurk/unique.rb +12 -2
  21. data/lib/wurk/version.rb +1 -1
  22. data/lib/wurk/web/views/_job_info.erb +16 -0
  23. data/lib/wurk/web/views/_paging.erb +11 -0
  24. data/lib/wurk.rb +4 -0
  25. data/vendor/assets/dashboard/assets/{ArgsValue-CmaXArv8.js → ArgsValue-BJol9S0s.js} +1 -1
  26. data/vendor/assets/dashboard/assets/{BatchDetail-nc5yePZW.js → BatchDetail-D_zGQ9Sd.js} +1 -1
  27. data/vendor/assets/dashboard/assets/{Batches-DgEcyOZ_.js → Batches-Dt_xC660.js} +1 -1
  28. data/vendor/assets/dashboard/assets/{Busy-CUZncvCy.js → Busy-CWRthsmc.js} +1 -1
  29. data/vendor/assets/dashboard/assets/{Cron-CIZbaaZs.js → Cron-qeGr2YAm.js} +1 -1
  30. data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +1 -0
  31. data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +1 -0
  32. data/vendor/assets/dashboard/assets/{Extension-Doy4bJoA.js → Extension-DONXVLdA.js} +1 -1
  33. data/vendor/assets/dashboard/assets/{FilterBox-C4bO26_d.js → FilterBox-BbB6gXHp.js} +1 -1
  34. data/vendor/assets/dashboard/assets/{FlowDetail-Dh3SJdFh.js → FlowDetail-CrmnVRp0.js} +1 -1
  35. data/vendor/assets/dashboard/assets/{Flows-C5SMRuhq.js → Flows-g4GC8KmE.js} +1 -1
  36. data/vendor/assets/dashboard/assets/{JobDetailModal-Cg6sAsJB.js → JobDetailModal-C86k_1wj.js} +1 -1
  37. data/vendor/assets/dashboard/assets/{Limiters-BIxwDOJM.js → Limiters-zyO-iMlK.js} +1 -1
  38. data/vendor/assets/dashboard/assets/{Metrics-Qp9q0nF8.js → Metrics-Ckq5fNBv.js} +1 -1
  39. data/vendor/assets/dashboard/assets/{NotFound-DwjPxFjU.js → NotFound-C8NnyjlY.js} +1 -1
  40. data/vendor/assets/dashboard/assets/{Profiles-7dabYVe3.js → Profiles-DRl-FXPA.js} +1 -1
  41. data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +1 -0
  42. data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +1 -0
  43. data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +1 -0
  44. data/vendor/assets/dashboard/assets/{Search-BAsVE2qw.js → Search-LaEpj2fM.js} +1 -1
  45. data/vendor/assets/dashboard/assets/{charts-DZs138BV.js → charts-D_PQLoyc.js} +1 -1
  46. data/vendor/assets/dashboard/assets/index-FBKGtysy.js +141 -0
  47. data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-DxkLZCIm.js → useResetPageOnEmpty-BiwhHqgh.js} +1 -1
  48. data/vendor/assets/dashboard/assets/{utils-B0kQFNE8.js → utils-CnWZzW2L.js} +1 -1
  49. data/vendor/assets/dashboard/index.html +1 -1
  50. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  51. metadata +29 -25
  52. data/vendor/assets/dashboard/assets/Dashboard-BWmfAook.js +0 -1
  53. data/vendor/assets/dashboard/assets/Dead-CGKsTZHq.js +0 -1
  54. data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +0 -1
  55. data/vendor/assets/dashboard/assets/Retries-CNX8ZtTH.js +0 -1
  56. data/vendor/assets/dashboard/assets/Scheduled-ClVBj0nc.js +0 -1
  57. data/vendor/assets/dashboard/assets/index-CMzPFxNE.js +0 -141
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../keys'
4
+ require_relative '../lua'
5
+ require_relative '../middleware/poison_pill'
6
+ require_relative '../metrics/process_counters'
7
+ require_relative 'oom_drain'
8
+
9
+ module Wurk
10
+ class Fetcher
11
+ # Puts one payload from a private list back where it belongs, for both ways
12
+ # an orphan is found: the Reaper draining a dead owner's list, and
13
+ # SelfReconcile returning a payload its own live process lost track of.
14
+ #
15
+ # The poison-pill verdict is taken in the same script that moves the job
16
+ # (`lua/reclaim_orphan.lua`), so a job past the cap goes straight from the
17
+ # private list to the dead set and is never visible on its public queue. The
18
+ # notifications (death handlers, `super_fetch!`, `on_poison`) follow once
19
+ # the script has returned, for a job no worker can be running.
20
+ class OrphanReclaim
21
+ THRESHOLD = Middleware::PoisonPill::RECOVERY_THRESHOLD
22
+ TTL = Middleware::PoisonPill::RECOVERY_TTL
23
+
24
+ def initialize(config)
25
+ @config = config
26
+ end
27
+
28
+ # Returns true when this call took the payload off the private list, false
29
+ # when it had already left (another reaper, an ACK).
30
+ #
31
+ # Unclaimed apply-safety: the move is guarded, but a reply lost after a
32
+ # poison kill would leave the death handlers unfired, and a list holding
33
+ # two identical copies would move the second on a replay. A raise leaves
34
+ # the payload for the next pass instead.
35
+ def call(private_list, public_q, payload)
36
+ job = Middleware::PoisonPill.parse(payload)
37
+ count = @config.redis { |conn| move(conn, private_list, public_q, payload, job) }
38
+ return false unless count
39
+
40
+ Metrics::ProcessCounters.increment(:reaper_reclaimed)
41
+ unless count == :uncounted
42
+ Middleware::PoisonPill.recovered!(payload, job, count: count, queue: queue_name(public_q),
43
+ config: @config)
44
+ end
45
+ true
46
+ end
47
+
48
+ private
49
+
50
+ # Past maxmemory the counter cannot be written, nor the job killed: it goes
51
+ # back uncounted rather than waiting on the memory it would help free.
52
+ def move(conn, private_list, public_q, payload, job)
53
+ keys = [private_list, public_q, Keys::DEAD]
54
+ jid = job && job['jid']
55
+ keys << Middleware::PoisonPill.counter_key(jid) if jid.is_a?(String) && !jid.empty?
56
+ Wurk::Lua::Loader.eval_cached(conn, :reclaim_orphan, keys: keys, argv: [payload, THRESHOLD, TTL, now])
57
+ rescue RedisClient::OutOfMemoryError => e
58
+ :uncounted if OomDrain.requeue(conn, e, private_list, public_q, payload) == 1
59
+ end
60
+
61
+ # DeadSet#kill's score format, so the entry sorts with every other kill.
62
+ def now
63
+ ::Process.clock_gettime(::Process::CLOCK_REALTIME).to_s
64
+ end
65
+
66
+ def queue_name(public_q)
67
+ public_q.delete_prefix(Keys::QUEUE_PREFIX)
68
+ end
69
+ end
70
+ end
71
+ end
@@ -2,13 +2,12 @@
2
2
 
3
3
  require_relative '../component'
4
4
  require_relative '../keys'
5
- require_relative '../middleware/poison_pill'
6
- require_relative '../metrics/process_counters'
7
5
  require_relative '../timer_loop'
8
6
  require_relative 'private_list_key'
9
7
  require_relative 'unparseable_keys'
10
8
  require_relative 'orphan_grace'
11
9
  require_relative 'oom_drain'
10
+ require_relative 'orphan_reclaim'
12
11
  require_relative 'self_reconcile'
13
12
 
14
13
  module Wurk
@@ -45,10 +44,10 @@ module Wurk
45
44
  # process's own clock (OrphanGrace), and its identity must still have no
46
45
  # `info` hash. Our own incarnation skips both: kill(0) is not a snapshot.
47
46
  #
48
- # Re-pushed jobs run through Wurk::Middleware::PoisonPill, which caps a
49
- # job at RECOVERY_THRESHOLD recoveries within 72h: past the cap the job
50
- # is killed into the dead set instead of re-queued, so a job that crashes
51
- # its worker every time can't loop forever.
47
+ # Every move goes through OrphanReclaim, which caps a job at
48
+ # Middleware::PoisonPill::RECOVERY_THRESHOLD recoveries within 72h: past the
49
+ # cap the job goes from the private list straight to the dead set instead of
50
+ # its queue, so a job that crashes its worker every time can't loop forever.
52
51
  #
53
52
  # One sweep an interval: one SCAN of `queue:*|*` acting on every list, gated
54
53
  # by a cluster lock so about one process sweeps per interval. Every list, and
@@ -62,7 +61,7 @@ module Wurk
62
61
  # of a job.
63
62
  #
64
63
  # Spec: docs/target/sidekiq-pro.md §3.2.
65
- class Reaper # rubocop:disable Metrics/ClassLength
64
+ class Reaper
66
65
  include Component
67
66
 
68
67
  # Sweep cadence in seconds. 60s matches the heartbeat TTL — the floor below
@@ -103,8 +102,8 @@ module Wurk
103
102
  @sleeper = ::ConditionVariable.new
104
103
  @unparseable = UnparseableKeys.new(config)
105
104
  @next_sweep_at = 0.0
106
- @self_reconcile = SelfReconcile.new(config, confirm: reconcile_confirm, stop: -> { done? },
107
- on_moved: method(:recovered))
105
+ @orphan_reclaim = OrphanReclaim.new(config)
106
+ @self_reconcile = SelfReconcile.new(config, confirm: reconcile_confirm, stop: -> { done? })
108
107
  end
109
108
 
110
109
  # Spawns the sweep loop. Idempotent. The loop waits one interval before
@@ -313,27 +312,19 @@ module Wurk
313
312
  identity.split(':')[0..1].join(':')
314
313
  end
315
314
 
316
- # Drain one orphaned private list back to its public queue. Each job is
317
- # moved with an atomic LMOVE (private tail → public tail) BEFORE the
318
- # poison check, so a crash mid-drain leaves the job safely in the public
319
- # queue (at-least-once), never lost. Poison jobs are killed to the dead
320
- # set by PoisonPill.track! and then LREM'd out of the public queue.
315
+ # Drain one orphaned private list back to its public queue, a job at a
316
+ # time from the tail: peek it, then OrphanReclaim moves it, to the public
317
+ # tail or, past the poison cap, to the dead set. The decision is taken
318
+ # before the job is visible, so a worker parked in BLMOVE can never take a
319
+ # job that is being killed. A peeked job that has left by the time it is
320
+ # moved (another reaper drained it) costs one more peek.
321
321
  #
322
- # Unlike the fetcher's LMOVE, this one does *not* claim apply-safety: a
323
- # replay after a lost reply moves the next job and never poison-checks the
324
- # one already on the public tail, so a job that kills its worker every time
325
- # would get a free recovery past the cap. A raise instead lands in the
326
- # rescue below, and the next sweep re-drains what's left.
322
+ # A raise lands in the rescue below; the job stays in the private list
323
+ # and the next sweep re-drains what's left.
327
324
  def drain(private_list, public_q)
328
- queue_name = public_q.delete_prefix(Keys::QUEUE_PREFIX)
329
325
  count = 0
330
- loop do
331
- job = redis { |c| reclaim_one(c, private_list, public_q) }
332
- break unless job
333
-
334
- count += 1
335
- Metrics::ProcessCounters.increment(:reaper_reclaimed)
336
- poison_off(public_q, job, queue_name)
326
+ while (job = redis(idempotent: true) { |c| c.call('LINDEX', private_list, -1) })
327
+ count += 1 if @orphan_reclaim.call(private_list, public_q, job)
337
328
  end
338
329
  count
339
330
  rescue StandardError => e
@@ -341,35 +332,6 @@ module Wurk
341
332
  count
342
333
  end
343
334
 
344
- # A full Redis must not hold a dead worker's jobs back from the workers
345
- # still able to drain the queue.
346
- def reclaim_one(conn, private_list, public_q)
347
- Wurk::Lua::Loader.eval_cached(conn, :reliable_reclaim, keys: [private_list, public_q], argv: [])
348
- rescue RedisClient::OutOfMemoryError => e
349
- OomDrain.reclaim(conn, e, private_list, public_q)
350
- end
351
-
352
- # SelfReconcile put a payload back: the same accounting as a reclaim, so a
353
- # job that kills its processor thread every time still hits the poison cap.
354
- def recovered(public_q, job)
355
- Metrics::ProcessCounters.increment(:reaper_reclaimed)
356
- poison_off(public_q, job, public_q.delete_prefix(Keys::QUEUE_PREFIX))
357
- end
358
-
359
- # The job is back on its queue whatever happens here. Past maxmemory the
360
- # count can't be taken (nor the job killed, leaving it where it is): raising
361
- # would end the drain at one job per pass, with the rest of the list waiting
362
- # on memory they would help free.
363
- def poison_off(public_q, job, queue_name)
364
- return unless Middleware::PoisonPill.track!(job, queue: queue_name, config: @config) == :poison
365
-
366
- # track! already ZADDed the payload to the dead set; pull the copy we
367
- # just moved onto the public tail so it isn't also re-run.
368
- redis { |c| c.call('LREM', public_q, -1, job) }
369
- rescue RedisClient::OutOfMemoryError => e
370
- report(e)
371
- end
372
-
373
335
  def acquire_lock?
374
336
  ttl = (@interval * LOCK_TTL_RATIO * 1000).ceil
375
337
  redis do |c|
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative '../keys'
4
- require_relative '../lua'
4
+ require_relative 'orphan_reclaim'
5
5
 
6
6
  module Wurk
7
7
  class Fetcher
@@ -34,13 +34,14 @@ module Wurk
34
34
 
35
35
  Suspect = Struct.new(:copies, :since)
36
36
 
37
- # `on_moved` receives [public_queue, payload] for each payload put back, so
38
- # the Reaper can run it through the same poison accounting as a reclaim.
39
- def initialize(config, confirm: DEFAULT_CONFIRM, stop: -> { false }, on_moved: nil)
37
+ # A payload put back goes through the same poison accounting as a reclaim
38
+ # (OrphanReclaim), so a job that kills its processor thread every time
39
+ # still hits the poison cap.
40
+ def initialize(config, confirm: DEFAULT_CONFIRM, stop: -> { false })
40
41
  @config = config
41
42
  @confirm = confirm
42
43
  @stop = stop
43
- @on_moved = on_moved
44
+ @orphan_reclaim = OrphanReclaim.new(config)
44
45
  @suspects = {}
45
46
  end
46
47
 
@@ -154,16 +155,9 @@ module Wurk
154
155
  moved
155
156
  end
156
157
 
157
- # LPOS-guarded, so a replay after a lost reply moves nothing the second
158
- # time, and a payload settled since the last read stays settled.
158
+ # LPOS-guarded, so a payload settled since the last read stays settled.
159
159
  def requeue(list, public_q, payload)
160
- applied = @config.redis(idempotent: true) do |conn|
161
- Wurk::Lua::Loader.eval_cached(conn, :reliable_requeue, keys: [list, public_q], argv: [payload])
162
- rescue RedisClient::OutOfMemoryError => e
163
- OomDrain.requeue(conn, e, list, public_q, payload)
164
- end.to_i
165
- @on_moved&.call(public_q, payload) if applied == 1
166
- applied
160
+ @orphan_reclaim.call(list, public_q, payload) ? 1 : 0
167
161
  end
168
162
 
169
163
  def forget_all
data/lib/wurk/job_util.rb CHANGED
@@ -117,6 +117,17 @@ module Wurk
117
117
  finalize(normalized)
118
118
  end
119
119
 
120
+ # An ActiveSupport::Duration (`48.hours`) unwrapped to the plain number it
121
+ # stands for; anything else comes back as given. The Duration claims
122
+ # `is_a?(Numeric)` through its own override, but `case`/`when` (Module#===)
123
+ # does not ask it, and JSON.generate writes it as a String ("172800"), which
124
+ # no reader after the push can use as seconds.
125
+ def self.plain_seconds(value)
126
+ return value if !value.is_a?(::Numeric) || value.class <= ::Numeric
127
+
128
+ value.is_a?(::Integer) ? value.to_i : value.to_f
129
+ end
130
+
120
131
  def now_in_millis
121
132
  ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
122
133
  end
@@ -166,7 +177,7 @@ module Wurk
166
177
  end
167
178
 
168
179
  def numeric_retry_for(value)
169
- case value
180
+ case (value = JobUtil.plain_seconds(value))
170
181
  when Integer then value
171
182
  when Numeric then (value.to_i if value.finite? && value.real?)
172
183
  when String then (Integer(value, 10) if value.match?(/\A-?\d+\z/))
@@ -206,13 +217,18 @@ module Wurk
206
217
  def finalize(normalized)
207
218
  TRANSIENT_ATTRIBUTES.each { |k| normalized.delete(k) }
208
219
  normalized['jid'] ||= SecureRandom.hex(12)
209
- normalized['retry_for'] = numeric_retry_for(normalized['retry_for']) if normalized.key?('retry_for')
220
+ coerce_seconds!(normalized)
210
221
  normalized['created_at'] ||= now_in_millis
211
222
  stamp_expiry(normalized)
212
223
  stamp_deadline(normalized)
213
224
  normalized
214
225
  end
215
226
 
227
+ def coerce_seconds!(item)
228
+ item['retry_for'] = numeric_retry_for(item['retry_for']) if item.key?('retry_for')
229
+ BOUND_OPTIONS.each { |k| item[k] = JobUtil.plain_seconds(item[k]) if item.key?(k) }
230
+ end
231
+
216
232
  # Pro `expires_in:` → absolute epoch-float `expiry` resolved once at push,
217
233
  # so the server middleware doesn't redo the math. Spec: sidekiq-pro.md §7.
218
234
  # nil.respond_to?(:to_f) is true on modern Ruby (returns 0.0), so we must
data/lib/wurk/launcher.rb CHANGED
@@ -99,10 +99,12 @@ module Wurk
99
99
  # 4. start the reaper and the boot-time reclaim sweep — kill-9 recovery,
100
100
  # not decoration: a SIGKILLed sibling's in-flight jobs wait on this,
101
101
  # so it stays ahead of the periodic loops.
102
- # 5. start the periodic loops. None of them ticks at zero — TimerLoop#run
103
- # waits an interval before its first yield, Scheduled::Poller waits
104
- # INITIAL_WAIT, and Leader waits DEFAULT_INITIAL_WAIT — so starting
105
- # them last costs a few thread spawns and nothing else. Both leader-
102
+ # 5. write the `config.periodic` registrations (Ent registers loops on
103
+ # startup), then start the periodic loops. None of them ticks at
104
+ # zero — TimerLoop#run waits an interval before its first yield,
105
+ # Scheduled::Poller waits INITIAL_WAIT, and Leader waits
106
+ # DEFAULT_INITIAL_WAIT — so starting them last costs a few thread
107
+ # spawns and nothing else. Both leader-
106
108
  # gated pollers are also safe to start before leadership settles: a
107
109
  # non-leader tick just returns early.
108
110
  # 6. start the health probe server LAST so the listener doesn't
@@ -240,6 +242,7 @@ module Wurk
240
242
  # Run on a background thread so /ready probe isn't delayed by a large
241
243
  # orphan sweep (the reaper's reclaim is atomic, but can scan many entries).
242
244
  @boot_reclaim_thread = safe_thread('boot-reclaim', &method(:boot_reclaim))
245
+ @config.apply_periodic!
243
246
  start_periodic_components
244
247
  end
245
248
  @health_server&.start
@@ -0,0 +1,31 @@
1
+ -- Takes one orphaned payload off a private list and decides, before anyone can
2
+ -- see it, where it goes: back onto its public queue's fetch end, or into the
3
+ -- dead set once the poison-pill counter passes the threshold. Pushing first and
4
+ -- counting after handed a poison job to a worker parked in BLMOVE, then killed
5
+ -- it into the dead set while it ran (Fetcher::OrphanReclaim).
6
+ --
7
+ -- The payload is located from the tail, where the reaper peeked it. Gone means
8
+ -- someone else moved it, and nothing is touched. Push before remove, as in the
9
+ -- requeue scripts, and the counter last: a write refused part way through must
10
+ -- leave the job where it was, and a refused counter only leaves it short.
11
+ -- SET EX is INCR + EXPIRE in one write, so the counter never lacks its TTL.
12
+ -- KEYS = [private_list, public_queue, dead, counter (only for a payload with a jid)]
13
+ -- ARGV = [payload, threshold, ttl_seconds, now]
14
+ -- Returns the recovery count (0 without a counter), nil when the payload is gone.
15
+ if not redis.call("lpos", KEYS[1], ARGV[1], "RANK", -1) then
16
+ return nil
17
+ end
18
+ local count = 0
19
+ if KEYS[4] then
20
+ count = (tonumber(redis.call("get", KEYS[4])) or 0) + 1
21
+ end
22
+ if count > tonumber(ARGV[2]) then
23
+ redis.call("zadd", KEYS[3], ARGV[4], ARGV[1])
24
+ else
25
+ redis.call("rpush", KEYS[2], ARGV[1])
26
+ end
27
+ redis.call("lrem", KEYS[1], -1, ARGV[1])
28
+ if KEYS[4] then
29
+ redis.call("set", KEYS[4], count, "EX", ARGV[3])
30
+ end
31
+ return count
data/lib/wurk/lua.rb CHANGED
@@ -128,10 +128,11 @@ module Wurk
128
128
  # RPUSH (public tail) not LPUSH so the reclaimed job is fetched next — LMOVE
129
129
  # pops the tail — ahead of fresh LPUSH'd enqueues.
130
130
  #
131
- # Push before remove, in this script and the two below. Nothing rolls a
132
- # script back, and a server can refuse the write mid-script (Dragonfly at
133
- # maxmemory refuses the RPUSH after applying an LREM): the raise has to leave
134
- # the job where it was, never in neither place.
131
+ # Push before remove, in this script, the one below and
132
+ # `lua/reclaim_orphan.lua`. Nothing rolls a script back, and a server can
133
+ # refuse the write mid-script (Dragonfly at maxmemory refuses the RPUSH
134
+ # after applying an LREM): the raise has to leave the job where it was,
135
+ # never in neither place.
135
136
  # KEYS = [private_list, public_queue]
136
137
  # ARGV = [job_json]
137
138
  # Returns 1 when the job was moved, 0 when it was already acked.
@@ -154,21 +155,6 @@ module Wurk
154
155
  return redis.call("lrem", KEYS[1], 1, ARGV[1])
155
156
  LUA
156
157
 
157
- # The reaper's move off a dead worker's private list: LMOVE private public
158
- # RIGHT RIGHT, spelled out because Dragonfly at maxmemory answers an LMOVE
159
- # between shards with the job having popped it and dropped the push. The
160
- # reply would be the only copy left, and the reaper has no use for it.
161
- # KEYS = [private_list, public_queue]
162
- # Returns the job moved, nil when the list is empty.
163
- RELIABLE_RECLAIM = <<~LUA
164
- local job = redis.call("lindex", KEYS[1], -1)
165
- if job then
166
- redis.call("rpush", KEYS[2], job)
167
- redis.call("rpop", KEYS[1])
168
- end
169
- return job
170
- LUA
171
-
172
158
  # Ent Unique (§3): atomic compare-and-delete of a lock key. Replaces the
173
159
  # two-command GET-then-DEL — between those calls the key can expire and a
174
160
  # fresh owner can grab it, and the bare DEL would then drop the new
@@ -276,7 +262,6 @@ module Wurk
276
262
  scheduled_claim: SCHEDULED_CLAIM,
277
263
  reliable_requeue: RELIABLE_REQUEUE,
278
264
  requeue_unit: REQUEUE_UNIT,
279
- reliable_reclaim: RELIABLE_RECLAIM,
280
265
  fast_delete_job: FAST_DELETE_JOB,
281
266
  fast_delete_by_class: FAST_DELETE_BY_CLASS,
282
267
  release_if_owner: RELEASE_IF_OWNER,
@@ -9,19 +9,20 @@ require_relative '../dead_set'
9
9
  module Wurk
10
10
  module Middleware
11
11
  # Pro parity (§3.2): poison-pill detection for reliable-fetch orphans.
12
- # When a job is recovered out of a dead process's private list, we
13
- # INCR a per-jid counter at `super_fetch:recovered:<jid>` with a 72h
14
- # TTL. Recoveries up to RECOVERY_THRESHOLD (3) requeue; the one that takes
15
- # the counter past it (the 4th orphaning) is the poison pill: the payload is moved to the dead set,
16
- # `jobs.poison` is emitted to statsd, and recovery callbacks fire so
12
+ # When a job is recovered out of a private list, a per-jid counter at
13
+ # `super_fetch:recovered:<jid>` goes up by one with a 72h TTL. Recoveries up
14
+ # to RECOVERY_THRESHOLD (3) requeue; the one that takes the counter past it
15
+ # (the 4th orphaning) is the poison pill: the payload is moved to the dead
16
+ # set, `jobs.poison` is emitted to statsd, and recovery callbacks fire so
17
17
  # operators can be paged.
18
18
  #
19
19
  # Counter key TTL is wire-compat with Sidekiq Pro — third-party tooling
20
20
  # that watches `super_fetch:recovered:*` expects 72h.
21
21
  #
22
- # No server-middleware registration: the producer is Reaper#drain, which
23
- # drives the lifecycle directly via `track!(payload, queue:)` on each
24
- # orphan it moves back to the public queue. The consumer is the ACK —
22
+ # No server-middleware registration: the producer is Fetcher::OrphanReclaim,
23
+ # whose script bumps the counter and takes the requeue-or-kill decision in
24
+ # the same step as the move, then hands the outcome to {recovered!} for the
25
+ # notifications. The consumer is the ACK —
25
26
  # `Fetcher::Reliable::UnitOfWork#acknowledge` pipelines {clear_in} next to
26
27
  # its LREM, so an attempt that finished starts the next one at zero
27
28
  # without costing a round trip of its own.
@@ -44,37 +45,34 @@ module Wurk
44
45
 
45
46
  module_function
46
47
 
47
- # Called per recovered orphan job. Returns `:poison` when the threshold
48
- # was crossed and the job was killed; `:recovered` when the job is
49
- # being re-pushed (caller's responsibility — we don't touch the queue
50
- # here). Emits `jobs.recovered.fetch` on every call, `jobs.poison`
51
- # only on the kill path.
48
+ # Called per recovered orphan job, once Fetcher::OrphanReclaim has moved
49
+ # it: `count` is the recovery counter its script wrote (0 for a payload with
50
+ # no jid). Returns `:poison` when the count is past the threshold — the
51
+ # script already put the job in the dead set, never on its queue — and
52
+ # `:recovered` otherwise. Emits `jobs.recovered.fetch` for every parseable
53
+ # job, `jobs.poison` only on the kill path.
52
54
  #
53
55
  # Fires the Pro `super_fetch!` recovery callback (config.super_fetch_callback)
54
56
  # exactly once per call: `(jobstr, nil)` on plain recovery, `(jobstr, pill)`
55
57
  # on the kill path. The poison-only `on_poison` Hash callbacks fire
56
58
  # independently inside #mark_poison.
57
59
  #
58
- # @param payload [String, Hash] the job JSON or pre-parsed hash.
60
+ # @param payload [String] the job JSON as it sat in the private list.
61
+ # @param job [Hash, nil] the payload parsed by {parse}.
62
+ # @param count [Integer] the recovery count after this recovery.
59
63
  # @param queue [String, nil] the public queue name (without `queue:`).
60
- # @param config [Configuration] config that owns the super_fetch! callback;
61
- # defaults to the global so non-reaper callers (tests) need not pass it.
64
+ # @param config [Configuration] config that owns the super_fetch! callback.
62
65
  # @return [Symbol] :recovered | :poison
63
- def track!(payload, queue: nil, config: Wurk.configuration)
64
- job = parse(payload)
66
+ def recovered!(payload, job, count:, queue: nil, config: Wurk.configuration)
65
67
  unless job
66
68
  fire_super_fetch(config, payload, nil)
67
69
  return :recovered
68
70
  end
69
71
 
70
- jid = job['jid']
71
- klass = job['class']
72
- emit('jobs.recovered.fetch', klass, queue)
73
-
74
- count = bump_counter(jid) if jid && !jid.empty?
75
- if count && count > RECOVERY_THRESHOLD
72
+ emit('jobs.recovered.fetch', job['class'], queue)
73
+ if count > RECOVERY_THRESHOLD
76
74
  mark_poison(payload, job, queue: queue, count: count)
77
- fire_super_fetch(config, payload, Pill.new(jid: jid, klass: klass, count: count, queue: queue))
75
+ fire_super_fetch(config, payload, Pill.new(jid: job['jid'], klass: job['class'], count: count, queue: queue))
78
76
  :poison
79
77
  else
80
78
  fire_super_fetch(config, payload, nil)
@@ -168,29 +166,14 @@ module Wurk
168
166
 
169
167
  # ---- internals --------------------------------------------------
170
168
 
169
+ # The job Hash, or nil for anything that is not a JSON object.
171
170
  def parse(payload)
172
- return payload if payload.is_a?(::Hash)
173
- return nil unless payload.is_a?(::String)
174
-
175
- Wurk.load_json(payload)
171
+ job = Wurk.load_json(payload)
172
+ job if job.is_a?(::Hash)
176
173
  rescue ::JSON::ParserError
177
174
  nil
178
175
  end
179
176
 
180
- # No apply-safety claim: INCR is additive, so a block replayed after a
181
- # lost reply bumps twice and can carry a healthy job past
182
- # RECOVERY_THRESHOLD into the dead set. Raising instead only leaves the
183
- # count short — Reaper#drain rescues, and the job is already back on its
184
- # public queue, so it just misses one poison check.
185
- def bump_counter(jid)
186
- key = counter_key(jid)
187
- Wurk.redis do |conn|
188
- count = conn.call('INCR', key).to_i
189
- conn.call('EXPIRE', key, RECOVERY_TTL)
190
- count
191
- end
192
- end
193
-
194
177
  def emit(metric, klass, queue)
195
178
  tags = []
196
179
  tags << "class:#{klass}" if klass
@@ -198,6 +181,9 @@ module Wurk
198
181
  Wurk::Metrics::Statsd.increment(metric, tags: tags.empty? ? nil : tags)
199
182
  end
200
183
 
184
+ # The script already put `payload` in the dead set; DeadSet#kill writes the
185
+ # same member again, so the trim and the death handlers stay DeadSet's.
186
+ #
201
187
  # Death handlers fire (`notify_failure` defaults to true). A poison kill
202
188
  # is a death like any other exhaustion: suppressing it strands every
203
189
  # batch that owns the job — Batch::DeathHandler is a death handler, so a
@@ -206,8 +192,7 @@ module Wurk
206
192
  # (docs/target/sidekiq-pro.md §12); recorded in docs/idea/parity-divergences.md.
207
193
  def mark_poison(payload, job, queue:, count:)
208
194
  emit('jobs.poison', job['class'], queue)
209
- json = payload.is_a?(String) ? payload : Wurk.dump_json(job)
210
- Wurk::DeadSet.new.kill(json, ex: poisoned_error(job['class'], count))
195
+ Wurk::DeadSet.new.kill(payload, ex: poisoned_error(job['class'], count))
211
196
  fire_callbacks(jid: job['jid'], klass: job['class'], count: count, queue: queue)
212
197
  end
213
198
 
@@ -231,8 +216,7 @@ module Wurk
231
216
  cb = config.super_fetch_callback
232
217
  return unless cb
233
218
 
234
- jobstr = payload.is_a?(::String) ? payload : Wurk.dump_json(payload)
235
- cb.call(jobstr, pill)
219
+ cb.call(payload, pill)
236
220
  rescue StandardError => e
237
221
  config.handle_exception(e, context: 'Wurk::Middleware::PoisonPill')
238
222
  end
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative 'rails_reloader'
4
+ require_relative 'boot_checks'
4
5
 
5
6
  module Wurk
6
7
  # Coordinates how Wurk boots inside a Rails host: whether this process should
@@ -60,7 +61,8 @@ module Wurk
60
61
 
61
62
  # Invoked last of all initializers, once the host app has fully initialized.
62
63
  def boot(app = ::Rails.application)
63
- return if skip_boot?
64
+ reason = skip_reason
65
+ return log_client_only(reason) if reason
64
66
 
65
67
  case boot_action(app)
66
68
  when :fork then boot_swarm
@@ -79,13 +81,36 @@ module Wurk
79
81
  # run every job — every cron tick included — twice.
80
82
  #
81
83
  # Everything above is a denylist; `serving?` is the allowlist that backs it.
82
- def skip_boot?
83
- ENV['WURK_DISABLED'] == '1' ||
84
- Wurk.worker_boot_claimed? ||
85
- building? ||
86
- defined?(::Rails::Console) ||
87
- ::Rails.env.test? ||
88
- !serving?
84
+ def skip_boot? = !skip_reason.nil?
85
+
86
+ # Why this process runs no jobs, or nil when it boots them.
87
+ def skip_reason
88
+ return 'WURK_DISABLED=1' if ENV['WURK_DISABLED'] == '1'
89
+ return 'the wurk runner owns this process' if Wurk.worker_boot_claimed?
90
+ return 'a build or rake task' if building?
91
+ return 'Rails console' if defined?(::Rails::Console)
92
+ return 'test environment' if ::Rails.env.test?
93
+
94
+ 'not a web server; WURK_EMBED=1 runs jobs here' unless serving?
95
+ end
96
+
97
+ # The runner announces its own role; a second line from here would describe
98
+ # the same process twice.
99
+ def log_client_only(reason)
100
+ return if Wurk.worker_boot_claimed?
101
+
102
+ BootChecks.log_client_only(logger, reason)
103
+ end
104
+
105
+ # Named the way #serving? recognised it: a bundled gem defines its constant
106
+ # in processes that are not that server.
107
+ def server_name
108
+ return 'Puma' if puma_launcher_loaded?
109
+ return 'Passenger' if defined?(::PhusionPassenger)
110
+ return 'Unicorn' if unicorn_server?
111
+ return 'rails server' if defined?(::Rails::Server)
112
+
113
+ 'this process (WURK_EMBED=1)'
89
114
  end
90
115
 
91
116
  # Boot only inside a web server, or where the host opted in. Every Rails
@@ -183,8 +208,11 @@ module Wurk
183
208
  def boot_swarm # rubocop:disable Metrics/AbcSize
184
209
  warn_on_production_fork
185
210
  Health.options_from_env
186
- timeout = Wurk.configuration[:timeout] || Swarm::DEFAULT_SHUTDOWN_TIMEOUT
187
- swarm = Wurk::Swarm.new(topology: Wurk.configuration.topology, shutdown_timeout: timeout)
211
+ config = Wurk.configuration
212
+ timeout = config[:timeout] || Swarm::DEFAULT_SHUTDOWN_TIMEOUT
213
+ BootChecks.log_swarm_role(config, "swarm parent in #{server_name}")
214
+ BootChecks.run(config, nil)
215
+ swarm = Wurk::Swarm.new(topology: config.topology, shutdown_timeout: timeout)
188
216
  supervisor = nil
189
217
  # Registered BEFORE boot, which forks the children one slot at a time: a
190
218
  # fork that raises partway through would otherwise leave the children it
@@ -246,7 +274,8 @@ module Wurk
246
274
  # already running with nothing to drain them on host exit.
247
275
  at_exit { instance.stop }
248
276
  instance.run
249
- logger.info { 'wurk: running embedded in the web process (config.wurk.embed_in_web) — threads only, no fork' }
277
+ BootChecks.log_process_role(Wurk.configuration, "embedded in #{server_name}, threads only")
278
+ BootChecks.run(Wurk.configuration, nil)
250
279
  instance
251
280
  rescue StandardError => e
252
281
  logger.error { "wurk: embedded boot failed: #{e.class}: #{e.message}" }
data/lib/wurk/unique.rb CHANGED
@@ -202,8 +202,14 @@ module Wurk
202
202
  end
203
203
 
204
204
  # Coerce `unique_for` to a numeric seconds value. Accepts Integer,
205
- # Numeric, ActiveSupport::Duration (any `to_i`-respondent), or `false`
206
- # (skip). Returns nil when uniqueness should be skipped.
205
+ # Numeric, ActiveSupport::Duration (any `to_i`-respondent), a numeric
206
+ # String, or `false` (skip). Returns nil when uniqueness should be skipped.
207
+ #
208
+ # The String is what every reader after the push sees for a Duration:
209
+ # JSON.generate writes `5.minutes` as "300", in Sidekiq as here, so the
210
+ # server middleware and the death handler get "300" back off the queue.
211
+ # Refusing it left every `unique_for: 5.minutes` lock held until its TTL
212
+ # (tesote/tesote.com#519).
207
213
  def self.coerce_ttl(value)
208
214
  return nil if value.nil? || value == false
209
215
  # `.to_i`, not `value`: ActiveSupport::Duration overrides `is_a?(Integer)`
@@ -212,11 +218,15 @@ module Wurk
212
218
  # EX arg and raised TypeError at enqueue (#253).
213
219
  return value.to_i if value.is_a?(Integer) && value.positive?
214
220
  return value.to_i if value.is_a?(Numeric)
221
+ return value.to_f.to_i if value.is_a?(String) && value.match?(NUMERIC_TTL)
215
222
  return value.to_i if duration_like?(value)
216
223
 
217
224
  nil
218
225
  end
219
226
 
227
+ # Duration#to_s: "300" for `5.minutes`, "90.0" for `1.5.minutes`.
228
+ NUMERIC_TTL = /\A\d+(?:\.\d+)?\z/
229
+
220
230
  def self.duration_like?(value)
221
231
  return false unless value.respond_to?(:to_i)
222
232