wurk 1.9.0 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +4 -4
  2. data/app/controllers/wurk/api_controller.rb +4 -1
  3. data/lib/wurk/boot_checks.rb +87 -0
  4. data/lib/wurk/capsule.rb +33 -8
  5. data/lib/wurk/cli.rb +73 -8
  6. data/lib/wurk/client/buffered.rb +240 -26
  7. data/lib/wurk/compat.rb +11 -0
  8. data/lib/wurk/configuration.rb +114 -16
  9. data/lib/wurk/cron.rb +45 -0
  10. data/lib/wurk/debounce.rb +2 -2
  11. data/lib/wurk/error_coalescer.rb +35 -0
  12. data/lib/wurk/fetcher/at_maxmemory.rb +79 -0
  13. data/lib/wurk/fetcher/capped.rb +4 -0
  14. data/lib/wurk/fetcher/many_queues.rb +17 -1
  15. data/lib/wurk/fetcher/oom_drain.rb +102 -0
  16. data/lib/wurk/fetcher/orphan_grace.rb +60 -44
  17. data/lib/wurk/fetcher/private_list_key.rb +1 -16
  18. data/lib/wurk/fetcher/reaper.rb +169 -127
  19. data/lib/wurk/fetcher/reliable.rb +26 -4
  20. data/lib/wurk/fetcher/self_reconcile.rb +175 -0
  21. data/lib/wurk/fetcher/unit_of_work.rb +32 -8
  22. data/lib/wurk/health.rb +23 -4
  23. data/lib/wurk/health_probe.rb +64 -0
  24. data/lib/wurk/heartbeat.rb +55 -3
  25. data/lib/wurk/job/options.rb +9 -1
  26. data/lib/wurk/job.rb +2 -24
  27. data/lib/wurk/job_record.rb +17 -9
  28. data/lib/wurk/job_retry.rb +38 -10
  29. data/lib/wurk/job_set.rb +13 -5
  30. data/lib/wurk/launcher.rb +134 -52
  31. data/lib/wurk/leader.rb +9 -3
  32. data/lib/wurk/limiter/base.rb +17 -1
  33. data/lib/wurk/lua/debounce.lua +11 -6
  34. data/lib/wurk/lua/loader.rb +13 -1
  35. data/lib/wurk/lua.rb +71 -8
  36. data/lib/wurk/manager.rb +1 -1
  37. data/lib/wurk/metrics/process_counters.rb +27 -0
  38. data/lib/wurk/metrics/prometheus.rb +35 -14
  39. data/lib/wurk/middleware/current_attributes.rb +26 -10
  40. data/lib/wurk/middleware/poison_pill.rb +1 -2
  41. data/lib/wurk/process_set.rb +3 -1
  42. data/lib/wurk/processor.rb +82 -18
  43. data/lib/wurk/queue_consumers.rb +134 -0
  44. data/lib/wurk/rails_boot.rb +40 -2
  45. data/lib/wurk/rails_reloader.rb +51 -0
  46. data/lib/wurk/railtie.rb +20 -1
  47. data/lib/wurk/redact.rb +12 -0
  48. data/lib/wurk/redis_pool.rb +34 -9
  49. data/lib/wurk/scheduled.rb +142 -67
  50. data/lib/wurk/sentry/death_handler.rb +29 -0
  51. data/lib/wurk/sentry/error_handler.rb +52 -10
  52. data/lib/wurk/sentry/middleware.rb +8 -1
  53. data/lib/wurk/sentry/retry_policy.rb +12 -3
  54. data/lib/wurk/sentry.rb +42 -5
  55. data/lib/wurk/sorted_entry.rb +38 -13
  56. data/lib/wurk/swarm/child_boot.rb +42 -31
  57. data/lib/wurk/swarm/crash_watch.rb +65 -0
  58. data/lib/wurk/swarm.rb +79 -11
  59. data/lib/wurk/testing.rb +66 -0
  60. data/lib/wurk/version.rb +1 -1
  61. data/lib/wurk/web/extension.rb +6 -1
  62. data/lib/wurk/work_set.rb +6 -0
  63. data/lib/wurk/worker/setter.rb +11 -6
  64. data/lib/wurk/worker.rb +17 -1
  65. data/lib/wurk.rb +1 -1
  66. data/vendor/assets/dashboard/assets/{ArgsValue-DDqX6XWH.js → ArgsValue-CmaXArv8.js} +1 -1
  67. data/vendor/assets/dashboard/assets/{BatchDetail-lIytcn2C.js → BatchDetail-nc5yePZW.js} +1 -1
  68. data/vendor/assets/dashboard/assets/{Batches-BEumt-1O.js → Batches-DgEcyOZ_.js} +1 -1
  69. data/vendor/assets/dashboard/assets/{Busy-Vk9HmZcN.js → Busy-CUZncvCy.js} +1 -1
  70. data/vendor/assets/dashboard/assets/{Cron-CWcM-Cfr.js → Cron-CIZbaaZs.js} +1 -1
  71. data/vendor/assets/dashboard/assets/{Dashboard-Bgv1kusv.js → Dashboard-BWmfAook.js} +1 -1
  72. data/vendor/assets/dashboard/assets/{Dead-Di5Rct2H.js → Dead-CGKsTZHq.js} +1 -1
  73. data/vendor/assets/dashboard/assets/{Extension-LZY5-imc.js → Extension-Doy4bJoA.js} +1 -1
  74. data/vendor/assets/dashboard/assets/{FilterBox-BDo-7IAy.js → FilterBox-C4bO26_d.js} +1 -1
  75. data/vendor/assets/dashboard/assets/{FlowDetail-CehwRLTt.js → FlowDetail-Dh3SJdFh.js} +1 -1
  76. data/vendor/assets/dashboard/assets/{FlowState-glK2Tnf8.js → FlowState-yb7virqR.js} +1 -1
  77. data/vendor/assets/dashboard/assets/{Flows-cgy-Fsbp.js → Flows-C5SMRuhq.js} +1 -1
  78. data/vendor/assets/dashboard/assets/{JobDetailModal-Cuk3jruU.js → JobDetailModal-Cg6sAsJB.js} +1 -1
  79. data/vendor/assets/dashboard/assets/{Limiters-Ci8G2yZG.js → Limiters-BIxwDOJM.js} +1 -1
  80. data/vendor/assets/dashboard/assets/{Metrics-CQFUeZTR.js → Metrics-Qp9q0nF8.js} +1 -1
  81. data/vendor/assets/dashboard/assets/{NotFound-CYjhuDsI.js → NotFound-DwjPxFjU.js} +1 -1
  82. data/vendor/assets/dashboard/assets/{PageHeader-DjIAFTyX.js → PageHeader-vWeBbSPF.js} +1 -1
  83. data/vendor/assets/dashboard/assets/{Profiles-DTowqIRg.js → Profiles-7dabYVe3.js} +1 -1
  84. data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +1 -0
  85. data/vendor/assets/dashboard/assets/{Retries-SJ3BaKC0.js → Retries-CNX8ZtTH.js} +1 -1
  86. data/vendor/assets/dashboard/assets/{Scheduled-DxSRpmFZ.js → Scheduled-ClVBj0nc.js} +1 -1
  87. data/vendor/assets/dashboard/assets/{Search-bBbi5JNK.js → Search-BAsVE2qw.js} +1 -1
  88. data/vendor/assets/dashboard/assets/{charts-Bg2YBcaD.js → charts-DZs138BV.js} +1 -1
  89. data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +1 -0
  90. data/vendor/assets/dashboard/assets/index--xvvNlYx.css +1 -0
  91. data/vendor/assets/dashboard/assets/{index-BOTVSP9M.js → index-CMzPFxNE.js} +3 -3
  92. data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-BRPNyV2M.js → useResetPageOnEmpty-DxkLZCIm.js} +1 -1
  93. data/vendor/assets/dashboard/assets/{useSort-B-BLJik7.js → useSort-jBYVwXvg.js} +1 -1
  94. data/vendor/assets/dashboard/assets/{utils-CGyrKpTh.js → utils-B0kQFNE8.js} +1 -1
  95. data/vendor/assets/dashboard/index.html +3 -3
  96. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  97. metadata +41 -30
  98. data/vendor/assets/dashboard/assets/Queues-DUGRPjME.js +0 -1
  99. data/vendor/assets/dashboard/assets/i18n-BZClP62Q.js +0 -1
  100. data/vendor/assets/dashboard/assets/index-Rewk4eSE.css +0 -1
data/lib/wurk/lua.rb CHANGED
@@ -70,7 +70,13 @@ module Wurk
70
70
  # Redis, so an unbatched promote of a post-outage backlog (100k+ due
71
71
  # members) would block every client for the whole sweep. The Ruby caller
72
72
  # loops until a short batch comes back.
73
+ # The first line is a Lua comment to Redis and Valkey. Dragonfly reads it as
74
+ # a per-script grant, because this script alone derives its target key from
75
+ # the payload (`ARGV[2] .. q`) and that server refuses undeclared keys by
76
+ # default. It spares operators the server-wide allow-undeclared-keys flag,
77
+ # which serializes every script and cost about 3x script throughput.
73
78
  RELIABLE_SCHEDULE_PROMOTE = <<~LUA
79
+ --!df flags=allow-undeclared-keys
74
80
  local jobs = redis.call("zrangebyscore", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, tonumber(ARGV[4]))
75
81
  for i = 1, #jobs do
76
82
  local job = jobs[i]
@@ -89,26 +95,80 @@ module Wurk
89
95
  return #jobs
90
96
  LUA
91
97
 
98
+ # Default scheduler claim: lease up to ARGV[3] due members instead of popping
99
+ # them. Each stays in its set, rescored past any due window, so a lost reply, a
100
+ # failed push or a crash costs a later re-promotion and never the job. The
101
+ # caller ZREMs a member only once it has been pushed.
102
+ #
103
+ # Redis's clock decides both ends, so a host with a skewed clock can neither
104
+ # cut a lease short nor see another host's lease as due: the window is the
105
+ # earlier of the caller's and the server's time, and the expiry counts from the
106
+ # server's. tostring() would round a 10-digit epoch, hence the format.
107
+ # KEYS = [sorted_set]
108
+ # ARGV = [now, lease_seconds, limit]
109
+ # Returns the claimed members, lowest score first.
110
+ SCHEDULED_CLAIM = <<~LUA
111
+ local t = redis.call("TIME")
112
+ local server_now = tonumber(t[1]) + (tonumber(t[2]) / 1000000)
113
+ local window = math.min(tonumber(ARGV[1]), server_now)
114
+ local jobs = redis.call("zrange", KEYS[1], "-inf", string.format("%.6f", window), "byscore", "limit", 0, tonumber(ARGV[3]))
115
+ local expiry = string.format("%.6f", server_now + tonumber(ARGV[2]))
116
+ for i = 1, #jobs do
117
+ redis.call("zadd", KEYS[1], "XX", expiry, jobs[i])
118
+ end
119
+ return jobs
120
+ LUA
121
+
92
122
  # Reliable fetch (Pro super_fetch §3) shutdown requeue: atomically move
93
123
  # one in-flight job from a per-process private list back to its public
94
- # queue. The LREM guard is the whole point — RPUSH runs only when the job
95
- # was still in the private list (LREM removed exactly 1). A job the
96
- # Processor ACKed in the window between hard_shutdown's cross-thread `job`
97
- # read and this move (LREM removes 0) is NOT re-pushed, so the job lands in
98
- # exactly one place and can't double-execute. RPUSH (public tail) not LPUSH
99
- # so the reclaimed job is fetched next — LMOVE pops the tail — ahead of
100
- # fresh LPUSH'd enqueues.
124
+ # queue. The LPOS guard is the whole point — RPUSH runs only when the job
125
+ # is still in the private list. A job the Processor ACKed in the window
126
+ # between hard_shutdown's cross-thread `job` read and this move is NOT
127
+ # re-pushed, so the job lands in exactly one place and can't double-execute.
128
+ # RPUSH (public tail) not LPUSH so the reclaimed job is fetched next — LMOVE
129
+ # pops the tail — ahead of fresh LPUSH'd enqueues.
130
+ #
131
+ # Push before remove, in this script and the two below. Nothing rolls a
132
+ # script back, and a server can refuse the write mid-script (Dragonfly at
133
+ # maxmemory refuses the RPUSH after applying an LREM): the raise has to leave
134
+ # the job where it was, never in neither place.
101
135
  # KEYS = [private_list, public_queue]
102
136
  # ARGV = [job_json]
103
137
  # Returns 1 when the job was moved, 0 when it was already acked.
104
138
  RELIABLE_REQUEUE = <<~LUA
105
- if redis.call("lrem", KEYS[1], 1, ARGV[1]) == 1 then
139
+ if redis.call("lpos", KEYS[1], ARGV[1]) then
106
140
  redis.call("rpush", KEYS[2], ARGV[1])
141
+ redis.call("lrem", KEYS[1], 1, ARGV[1])
107
142
  return 1
108
143
  end
109
144
  return 0
110
145
  LUA
111
146
 
147
+ # UnitOfWork#requeue: unguarded, as the caller owns the unit and a job whose
148
+ # ACK already went out must still go back. Not replay-safe, so it is only
149
+ # ever sent once (Loader.eval_once).
150
+ # KEYS = [private_list, public_queue]
151
+ # ARGV = [job_json]
152
+ REQUEUE_UNIT = <<~LUA
153
+ redis.call("rpush", KEYS[2], ARGV[1])
154
+ return redis.call("lrem", KEYS[1], 1, ARGV[1])
155
+ LUA
156
+
157
+ # The reaper's move off a dead worker's private list: LMOVE private public
158
+ # RIGHT RIGHT, spelled out because Dragonfly at maxmemory answers an LMOVE
159
+ # between shards with the job having popped it and dropped the push. The
160
+ # reply would be the only copy left, and the reaper has no use for it.
161
+ # KEYS = [private_list, public_queue]
162
+ # Returns the job moved, nil when the list is empty.
163
+ RELIABLE_RECLAIM = <<~LUA
164
+ local job = redis.call("lindex", KEYS[1], -1)
165
+ if job then
166
+ redis.call("rpush", KEYS[2], job)
167
+ redis.call("rpop", KEYS[1])
168
+ end
169
+ return job
170
+ LUA
171
+
112
172
  # Ent Unique (§3): atomic compare-and-delete of a lock key. Replaces the
113
173
  # two-command GET-then-DEL — between those calls the key can expire and a
114
174
  # fresh owner can grab it, and the bare DEL would then drop the new
@@ -213,7 +273,10 @@ module Wurk
213
273
  SCRIPTS = {
214
274
  zpopbyscore: ZPOPBYSCORE,
215
275
  reliable_schedule_promote: RELIABLE_SCHEDULE_PROMOTE,
276
+ scheduled_claim: SCHEDULED_CLAIM,
216
277
  reliable_requeue: RELIABLE_REQUEUE,
278
+ requeue_unit: REQUEUE_UNIT,
279
+ reliable_reclaim: RELIABLE_RECLAIM,
217
280
  fast_delete_job: FAST_DELETE_JOB,
218
281
  fast_delete_by_class: FAST_DELETE_BY_CLASS,
219
282
  release_if_owner: RELEASE_IF_OWNER,
data/lib/wurk/manager.rb CHANGED
@@ -140,7 +140,7 @@ module Wurk
140
140
  # raising Wurk::Shutdown into the threads, so a job killed mid-perform is
141
141
  # re-run once (Sidekiq's at-least-once contract). `job` is read off another
142
142
  # thread, so a Processor can ACK between this map and the requeue — but
143
- # bulk_requeue's LREM guard skips the RPUSH on a miss, so a job that
143
+ # bulk_requeue's guard skips the RPUSH on a miss, so a job that
144
144
  # finished in that window is not resurrected onto the public queue.
145
145
  #
146
146
  # The kill sits in an `ensure`: a bulk_requeue that raises (Redis down
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wurk
4
+ module Metrics
5
+ # In-memory counters for events with no Redis key of their own, exposed by
6
+ # Metrics::Prometheus. Per process: under a swarm, a scrape sees only the
7
+ # child that owns the health port.
8
+ module ProcessCounters
9
+ @lock = ::Mutex.new
10
+ @counts = Hash.new(0)
11
+
12
+ class << self
13
+ def increment(name, by = 1)
14
+ @lock.synchronize { @counts[name] += by }
15
+ end
16
+
17
+ def value(name)
18
+ @lock.synchronize { @counts[name] }
19
+ end
20
+
21
+ def reset!
22
+ @lock.synchronize { @counts.clear }
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -3,7 +3,10 @@
3
3
  require 'json'
4
4
  require_relative '../component'
5
5
  require_relative '../keys'
6
+ require_relative '../leader'
6
7
  require_relative '../version'
8
+ require_relative '../queue_consumers'
9
+ require_relative 'process_counters'
7
10
 
8
11
  module Wurk
9
12
  module Metrics
@@ -18,6 +21,11 @@ module Wurk
18
21
  # heartbeat identities sharing this process's nonce (the swarm parent
19
22
  # stamps one nonce that every child inherits), so a scrape of one pod
20
23
  # covers all of its children no matter which child owns the port.
24
+ # `wurk_leader` is the same: 1 on every scrape target in the group that
25
+ # holds the lease.
26
+ # * `wurk_fetch_errors_total` / `wurk_reaper_reclaimed_total` count in the
27
+ # memory of the process that owns the port, so under a swarm they are a
28
+ # sample of one child. `rate` them; a child restart resets them.
21
29
  #
22
30
  # One Redis checkout per refresh — two pipelines, because the second needs
23
31
  # the queue and process names the first returns — and the snapshot is cached
@@ -29,7 +37,7 @@ module Wurk
29
37
  TTL = 1.0
30
38
  LABEL_ESCAPES = { '\\' => '\\\\', '"' => '\\"', "\n" => '\\n' }.freeze
31
39
  LABEL_ESCAPES_RE = /[\\"\n]/
32
- PROCESS_FIELDS = %w[busy concurrency beat rss quiet].freeze
40
+ PROCESS_FIELDS = %w[busy concurrency beat rss quiet info].freeze
33
41
  SCALARS = {
34
42
  processed: ['GET', Keys::STAT_PROCESSED],
35
43
  failed: ['GET', 'stat:failed'],
@@ -38,11 +46,13 @@ module Wurk
38
46
  dead: ['ZCARD', Keys::DEAD]
39
47
  }.freeze
40
48
  # Names come last: the detail pass is built from them.
41
- FIRST_PASS = [*SCALARS.values, ['SMEMBERS', Keys::QUEUES_SET], ['SMEMBERS', Keys::PROCESSES]].freeze
49
+ FIRST_PASS = [*SCALARS.values, ['GET', Wurk::Leader::DEFAULT_KEY],
50
+ ['SMEMBERS', Keys::QUEUES_SET], ['SMEMBERS', Keys::PROCESSES]].freeze
42
51
 
43
52
  QUEUE_GAUGES = [
44
53
  ['wurk_queue_size', 'Jobs enqueued per queue.', :depth.to_proc],
45
- ['wurk_queue_latency_seconds', 'Age of the oldest job per queue.', :latency.to_proc]
54
+ ['wurk_queue_latency_seconds', 'Age of the oldest job per queue.', :latency.to_proc],
55
+ ['wurk_queue_consumers', 'Live processes subscribed to the queue, quiet ones included.', :consumers.to_proc]
46
56
  ].freeze
47
57
  CLUSTER_GAUGES = [
48
58
  ['wurk_scheduled_size', 'Jobs in the schedule set.', :scheduled.to_proc],
@@ -61,10 +71,10 @@ module Wurk
61
71
  ['wurk_process_quiet', '1 when the process is quiet (TSTP).', ->(row, _now) { row.quiet ? 1 : 0 }]
62
72
  ].freeze
63
73
 
64
- QueueRow = Struct.new(:name, :depth, :latency)
65
- ProcessRow = Struct.new(:identity, :pid, :busy, :concurrency, :beat, :rss_kb, :quiet)
74
+ QueueRow = Struct.new(:name, :depth, :latency, :consumers)
75
+ ProcessRow = Struct.new(:identity, :pid, :busy, :concurrency, :beat, :rss_kb, :quiet, :queues)
66
76
  Snapshot = Struct.new(:ok, :taken_at, :processed, :failed, :scheduled, :retries, :dead, :queues, :processes,
67
- keyword_init: true)
77
+ :leader, keyword_init: true)
68
78
 
69
79
  def initialize(config, ttl: TTL, nonce: Component::PROCESS_NONCE, hostname: Component.hostname,
70
80
  clock: -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) })
@@ -101,6 +111,10 @@ module Wurk
101
111
  out = +''
102
112
  build_info(out)
103
113
  gauge(out, 'wurk_redis_up', 'Whether the last metrics refresh could read Redis.', [[nil, snap.ok ? 1 : 0]])
114
+ counter(out, 'wurk_fetch_errors_total', 'Fetch-loop errors in the process serving this scrape.',
115
+ ProcessCounters.value(:fetch_errors))
116
+ counter(out, 'wurk_reaper_reclaimed_total', 'Orphaned jobs reclaimed by the process serving this scrape.',
117
+ ProcessCounters.value(:reaper_reclaimed))
104
118
  if snap.ok
105
119
  cluster(out, snap)
106
120
  local(out, snap, expected_children, fresh_window)
@@ -114,12 +128,12 @@ module Wurk
114
128
  identity.end_with?(@local_suffix) && identity.start_with?(@local_prefix)
115
129
  end
116
130
 
117
- def fetch(now)
131
+ def fetch(now) # rubocop:disable Metrics/AbcSize
118
132
  first, detail = read_redis
119
133
  queues, processes = names(first)
120
- Snapshot.new(ok: true, taken_at: now, **scalars(first),
121
- queues: queue_rows(queues, detail),
122
- processes: process_rows(processes, detail.drop(queues.size * 2)))
134
+ rows = process_rows(processes, detail.drop(queues.size * 2))
135
+ Snapshot.new(ok: true, taken_at: now, **scalars(first), leader: first[SCALARS.size],
136
+ queues: queue_rows(queues, detail, rows), processes: rows)
123
137
  rescue StandardError => e
124
138
  @config.logger&.warn { "Wurk::Metrics::Prometheus refresh failed: #{e.class}: #{e.message}" }
125
139
  Snapshot.new(ok: false, taken_at: now, queues: [], processes: [])
@@ -148,20 +162,21 @@ module Wurk
148
162
  processes.each { |id| pipe.call('HMGET', id, *PROCESS_FIELDS) }
149
163
  end
150
164
 
151
- def queue_rows(queues, detail)
165
+ def queue_rows(queues, detail, processes)
166
+ consumers = processes.flat_map(&:queues).tally
152
167
  queues.each_with_index.map do |name, i|
153
- QueueRow.new(name, detail[i * 2].to_i, latency(detail[(i * 2) + 1]))
168
+ QueueRow.new(name, detail[i * 2].to_i, latency(detail[(i * 2) + 1]), consumers.fetch(name, 0))
154
169
  end
155
170
  end
156
171
 
157
172
  # An expired identity (dead process not yet pruned from `processes`)
158
173
  # answers HMGET with all nils; it is not a process, so it is dropped.
159
174
  def process_rows(processes, fields)
160
- processes.zip(fields).filter_map do |identity, (busy, concurrency, beat, rss, quiet)|
175
+ processes.zip(fields).filter_map do |identity, (busy, concurrency, beat, rss, quiet, info)|
161
176
  next if beat.nil? && busy.nil?
162
177
 
163
178
  ProcessRow.new(identity, identity.split(':')[-2].to_i, busy.to_i, concurrency.to_i,
164
- beat&.to_f, rss.to_i, quiet == 'true')
179
+ beat&.to_f, rss.to_i, quiet == 'true', QueueConsumers.subscribed(info))
165
180
  end
166
181
  end
167
182
 
@@ -190,6 +205,7 @@ module Wurk
190
205
 
191
206
  def local(out, snap, expected, window)
192
207
  now = ::Time.now.to_f
208
+ leader_gauge(out, snap)
193
209
  if expected
194
210
  gauge(out, 'wurk_swarm_children_expected', 'Children this swarm is configured to run.', [[nil, expected]])
195
211
  end
@@ -201,6 +217,11 @@ module Wurk
201
217
  end
202
218
  end
203
219
 
220
+ def leader_gauge(out, snap)
221
+ gauge(out, 'wurk_leader', '1 when this process group holds the leader lease.',
222
+ [[nil, snap.leader && local?(snap.leader) ? 1 : 0]])
223
+ end
224
+
204
225
  def counter(out, name, help, value)
205
226
  family(out, name, help, 'counter', [[nil, value]])
206
227
  end
@@ -12,6 +12,10 @@ module Wurk
12
12
  # Multiple → `"cattr"`, `"cattr_1"`, `"cattr_2"`, … (keys mirror Sidekiq's
13
13
  # naming exactly: wire-compat sacred).
14
14
  #
15
+ # Classes are kept by name and resolved per job, as a String registered from
16
+ # an initializer has to be. It also means a class object passed here doesn't
17
+ # go stale once development code reloading replaces the constant.
18
+ #
15
19
  # Spec: docs/target/sidekiq-free.md §10.3 and §2.2.
16
20
  module CurrentAttributes
17
21
  PERSISTENT_KEY = 'cattr'
@@ -21,12 +25,22 @@ module Wurk
21
25
  # no-op: `add` already dedupes by klass, so calling `persist` twice
22
26
  # with the same set replaces the old entry with the new args.
23
27
  def persist(klass_or_array, config = Wurk.configuration)
24
- classes = Array(klass_or_array)
28
+ classes = Array(klass_or_array).map { |klass| reference_for(klass) }
25
29
  raise ArgumentError, 'persist requires at least one CurrentAttributes class' if classes.empty?
26
30
 
27
31
  config.client_middleware.add(Save, classes)
28
32
  config.client_middleware.add(Load, classes)
29
- config.server_middleware.add(Load, classes)
33
+ # Outermost, so every other server middleware sees the job's attributes.
34
+ config.server_middleware.prepend(Load, classes)
35
+ end
36
+
37
+ # An anonymous class has no name to look up again, so it stays as given.
38
+ def reference_for(klass)
39
+ klass.is_a?(Module) ? klass.name || klass : klass.to_s
40
+ end
41
+
42
+ def resolve(reference)
43
+ reference.is_a?(String) ? Object.const_get(reference) : reference
30
44
  end
31
45
 
32
46
  # Composes the wire key for the Nth registered class. Sidekiq numbers
@@ -50,7 +64,8 @@ module Wurk
50
64
  end
51
65
 
52
66
  # Client-side: snapshot each registered CurrentAttributes class into
53
- # the job hash. Caller-supplied keys take precedence (`||=`).
67
+ # the job hash. Caller-supplied keys take precedence (`||=`), and a class
68
+ # with nothing set adds no key, so the payload doesn't grow an empty one.
54
69
  class Save
55
70
  include Wurk::Middleware::ClientMiddleware
56
71
 
@@ -59,9 +74,9 @@ module Wurk
59
74
  end
60
75
 
61
76
  def call(_job_class, job, _queue, _redis_pool)
62
- @classes.each_with_index do |klass, idx|
63
- key = CurrentAttributes.key_for(idx)
64
- job[key] ||= CurrentAttributes.snapshot(klass)
77
+ @classes.each_with_index do |reference, idx|
78
+ attrs = CurrentAttributes.snapshot(CurrentAttributes.resolve(reference))
79
+ job[CurrentAttributes.key_for(idx)] ||= attrs unless attrs.empty?
65
80
  end
66
81
  yield
67
82
  end
@@ -87,15 +102,16 @@ module Wurk
87
102
  end
88
103
 
89
104
  def call(_job_or_class, job, _queue, _redis_pool = nil)
90
- previous = @classes.map { |klass| CurrentAttributes.snapshot(klass) }
91
- @classes.each_with_index do |klass, idx|
105
+ classes = @classes.map { |reference| CurrentAttributes.resolve(reference) }
106
+ previous = classes.map { |klass| CurrentAttributes.snapshot(klass) }
107
+ classes.each_with_index do |klass, idx|
92
108
  CurrentAttributes.restore(klass, job[CurrentAttributes.key_for(idx)])
93
109
  end
94
110
  yield
95
111
  ensure
96
112
  previous&.each_with_index do |attrs, idx|
97
- @classes[idx].reset
98
- CurrentAttributes.restore(@classes[idx], attrs)
113
+ classes[idx].reset
114
+ CurrentAttributes.restore(classes[idx], attrs)
99
115
  end
100
116
  end
101
117
  end
@@ -12,8 +12,7 @@ module Wurk
12
12
  # When a job is recovered out of a dead process's private list, we
13
13
  # INCR a per-jid counter at `super_fetch:recovered:<jid>` with a 72h
14
14
  # TTL. Recoveries up to RECOVERY_THRESHOLD (3) requeue; the one that takes
15
- # the counter past it (the 4th orphaning) is the poison pill, matching
16
- # super_fetch's `count <= max_recoveries` recover_orphan script: the payload is moved to the dead set,
15
+ # the counter past it (the 4th orphaning) is the poison pill: the payload is moved to the dead set,
17
16
  # `jobs.poison` is emitted to statsd, and recovery callbacks fire so
18
17
  # operators can be paged.
19
18
  #
@@ -121,7 +121,9 @@ module Wurk
121
121
  # claimed (heartbeat is fresher).
122
122
  def build_process(attrs)
123
123
  hash = Wurk.load_json(attrs.fetch('info')).merge(beat_overrides(attrs))
124
- hash['concurrency'] = attrs['concurrency'].to_i if attrs.key?('concurrency')
124
+ # HMGET answers nil for a field the process never wrote: a stock Sidekiq
125
+ # process keeps concurrency only in `info`.
126
+ hash['concurrency'] = attrs['concurrency'].to_i if attrs['concurrency']
125
127
  Process.new(hash)
126
128
  end
127
129
 
@@ -6,6 +6,7 @@ require_relative 'dead_set'
6
6
  require_relative 'job_logger'
7
7
  require_relative 'job_retry'
8
8
  require_relative 'profiler'
9
+ require_relative 'rails_reloader'
9
10
 
10
11
  module Wurk
11
12
  # Inside each Manager, N Processors run in parallel. Each owns one thread,
@@ -23,7 +24,7 @@ module Wurk
23
24
  # payload survives in the private list and is reclaimed on next boot.
24
25
  #
25
26
  # Spec: docs/target/sidekiq-free.md §14.
26
- class Processor
27
+ class Processor # rubocop:disable Metrics/ClassLength
27
28
  include Component
28
29
 
29
30
  # Interrupt masks for the two `Thread.handle_interrupt` scopes every job
@@ -35,6 +36,20 @@ module Wurk
35
36
  ALLOW_SHUTDOWN_INTERRUPTS = { Wurk::Shutdown => :immediate }.freeze
36
37
  private_constant :ALLOW_SHUTDOWN_INTERRUPTS
37
38
 
39
+ # An internal failure that says something about Redis or the pool and nothing
40
+ # about the job, so the job goes back to run again. Any other StandardError
41
+ # escaping the retry layer comes from the payload itself (a `retry_count`
42
+ # that is not a number, args that cannot be re-serialized into a retry) and
43
+ # will fail the same way every time.
44
+ TRANSIENT_FAILURES = [RedisClient::Error, ConnectionPool::Error, ConnectionPool::TimeoutError].freeze
45
+ private_constant :TRANSIENT_FAILURES
46
+
47
+ # Pause after putting a job back on an internal failure. The job goes to the
48
+ # tail, which is the next fetch, so without one a failure that persists
49
+ # re-runs `perform` as fast as the thread can loop.
50
+ REQUEUE_PAUSE = 1
51
+ private_constant :REQUEUE_PAUSE
52
+
38
53
  # Stand-in for the default reloader, `proc { |&b| b.call }`. That default
39
54
  # is an identity wrapper, but a block param (`|&b|`) forces MRI to reify
40
55
  # the dispatch block into a Proc on every job just to call it straight
@@ -165,6 +180,11 @@ module Wurk
165
180
  end
166
181
  end
167
182
 
183
+ @vm_exiting = false
184
+ at_exit { @vm_exiting = true }
185
+
186
+ def self.vm_exiting? = @vm_exiting
187
+
168
188
  PROCESSED = Counter.new
169
189
  FAILURE = Counter.new
170
190
  EXPIRED = Counter.new
@@ -173,14 +193,15 @@ module Wurk
173
193
  private
174
194
 
175
195
  # Only the untouched framework default is swapped out — a host-supplied
176
- # reloader (Rails wraps every job in `Rails.application.reloader`) is used
177
- # exactly as given. `equal?` against DEFAULTS is sound because
178
- # Configuration's deep-dup copies Hashes/Arrays only, so an unset
179
- # `:reloader` is still the very Proc object DEFAULTS holds.
196
+ # reloader is used exactly as given, and nil opts out of wrapping. `equal?`
197
+ # against DEFAULTS is sound because Configuration's deep-dup copies
198
+ # Hashes/Arrays only, so an unset `:reloader` is still the very Proc object
199
+ # DEFAULTS holds.
180
200
  def resolve_reloader(configured)
181
- return IdentityReloader if configured.nil? || configured.equal?(Configuration::DEFAULTS[:reloader])
201
+ return IdentityReloader if configured.nil?
202
+ return configured unless configured.equal?(Configuration::DEFAULTS[:reloader])
182
203
 
183
- configured
204
+ RailsReloader.default || IdentityReloader
184
205
  end
185
206
 
186
207
  def run
@@ -196,16 +217,20 @@ module Wurk
196
217
  flush_acks
197
218
  Thread.current[:wurk_capsule] = outer_capsule
198
219
  end
199
- @callback&.call(self)
200
220
  rescue Wurk::Shutdown
201
- @callback&.call(self)
221
+ nil
202
222
  rescue Exception => e # rubocop:disable Lint/RescueException
203
223
  # Sidekiq hands the error to the manager (processor.rb `@callback.call(
204
224
  # self, ex)`); reporting it here keeps it visible, and the re-raise ends
205
225
  # the thread so the Manager's replacement starts clean.
206
226
  handle_exception(e, { context: '!shutdown' })
207
- @callback&.call(self)
208
227
  raise
228
+ ensure
229
+ # In an ensure because `Thread.exit` from a job unwinds through here with
230
+ # no exception at all, and a Manager never told would run one thread short
231
+ # for the life of the process. The VM unwinds threads the same way as it
232
+ # exits, and asking for a replacement then only raises ThreadError.
233
+ @callback&.call(self) unless Processor.vm_exiting?
209
234
  end
210
235
 
211
236
  # The fetcher holds each finished job's LREM until a fetch can pipeline it
@@ -231,7 +256,7 @@ module Wurk
231
256
  rescue Wurk::Shutdown
232
257
  nil
233
258
  rescue StandardError => e
234
- handle_exception(e, { context: 'Error fetching job' })
259
+ @capsule.config.report_fetch_error(e)
235
260
  sleep(1)
236
261
  nil
237
262
  end
@@ -245,6 +270,8 @@ module Wurk
245
270
  # outcome is booked.
246
271
  def process(uow)
247
272
  Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) { process_deferred(uow) }
273
+ ensure
274
+ uow.release_claim if uow.respond_to?(:release_claim)
248
275
  end
249
276
 
250
277
  def process_deferred(uow)
@@ -271,12 +298,10 @@ module Wurk
271
298
  rescue Wurk::Shutdown
272
299
  # Don't ack — UoW stays in private list and is reclaimed on reboot.
273
300
  rescue Exception => e # rubocop:disable Lint/RescueException
274
- # Escaped the retry layer (its ZADD hit a Redis blip, a logger raised in
275
- # JobLogger#prepare): no retry was booked. This owner is alive, so the
276
- # reaper will never reclaim its private list — put the job back now or
277
- # it sits there until the process restarts.
301
+ # Escaped the retry layer, so no retry was booked. This owner is alive,
302
+ # so the reaper will never reclaim its private list.
278
303
  handle_exception(e, { context: 'Internal exception!', job: job_hash, jobstr: jobstr })
279
- requeue(uow)
304
+ ack = settle_internal_failure(uow, job_hash, jobstr, e)
280
305
  ensure
281
306
  # This frame is the one every exit path passes through, which is why a
282
307
  # global-concurrency slot is given back from here and nowhere else: a
@@ -306,12 +331,45 @@ module Wurk
306
331
  handle_exception(handled.cause || handled, { context: 'Job raised exception', job: job_hash })
307
332
  end
308
333
 
334
+ # Whether the job is finished with (ACK it). Redis or the pool failing, or a
335
+ # process-level condition (NoMemoryError and friends), is no verdict on the
336
+ # job: it goes back to run again. A job that cannot be booked for any other
337
+ # reason would loop forever, re-running `perform` every time, so it goes to
338
+ # the morgue instead.
339
+ def settle_internal_failure(uow, job_hash, jobstr, error)
340
+ return kill_unbookable(job_hash, jobstr, error) if unbookable?(error)
341
+
342
+ requeue(uow)
343
+ sleep(REQUEUE_PAUSE) unless @done
344
+ false
345
+ end
346
+
347
+ def unbookable?(error)
348
+ error.is_a?(::StandardError) && TRANSIENT_FAILURES.none? { |klass| error.is_a?(klass) }
349
+ end
350
+
351
+ # Death handlers fire when there is a job to tell them about, so a batch
352
+ # records the death and a unique lock is released, as for any other job that
353
+ # died on its own. A kill that fails leaves the job in the private list, which
354
+ # is where it was before this ran; Fetcher::SelfReconcile picks it up.
355
+ def kill_unbookable(job_hash, jobstr, error)
356
+ dead = DeadSet.new
357
+ job_hash.is_a?(::Hash) ? dead.kill(jobstr, ex: error) : dead.kill_raw(jobstr)
358
+ true
359
+ rescue StandardError => e
360
+ handle_exception(e, { context: 'Error killing a job after an internal exception' })
361
+ false
362
+ end
363
+
309
364
  # Reported, never raised: we are already handling an internal error, and
310
365
  # a Redis that refused the retry ZADD may refuse this too — the job then
311
- # stays in the private list, which is where it was before this ran.
366
+ # stays in the private list, which is where it was before this ran, until
367
+ # Fetcher::SelfReconcile finds it. ERROR because nothing else will say that a
368
+ # job is parked out of sight for the next couple of minutes.
312
369
  def requeue(uow)
313
370
  uow.requeue
314
371
  rescue StandardError => e
372
+ logger.error { "Could not requeue #{uow.queue_name} job after an internal exception: #{e.class}: #{e.message}" }
315
373
  handle_exception(e, { context: 'Error requeueing a job after an internal exception' })
316
374
  end
317
375
 
@@ -346,8 +404,14 @@ module Wurk
346
404
  # morgue exactly like send_to_morgue does — spec §31.8/§31.9).
347
405
  # Returns nil to signal "no further processing" — the ACK that retires the
348
406
  # payload belongs to #process's ensure, which is the one cleanup path.
407
+ #
408
+ # JSON that parses to something other than an object (`[1]`, `true`) is the
409
+ # same thing: there is no job in it to run.
349
410
  def parse_or_kill(jobstr)
350
- Wurk.load_json(jobstr)
411
+ job_hash = Wurk.load_json(jobstr)
412
+ raise ::JSON::ParserError, 'job payload is not a JSON object' unless job_hash.is_a?(::Hash)
413
+
414
+ job_hash
351
415
  rescue ::JSON::ParserError => e
352
416
  handle_exception(e, { context: 'Invalid JSON', jobstr: jobstr })
353
417
  DeadSet.new.kill_raw(jobstr)