wurk 1.9.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/wurk/api_controller.rb +4 -1
- data/lib/wurk/boot_checks.rb +87 -0
- data/lib/wurk/capsule.rb +33 -8
- data/lib/wurk/cli.rb +73 -8
- data/lib/wurk/client/buffered.rb +240 -26
- data/lib/wurk/compat.rb +11 -0
- data/lib/wurk/configuration.rb +114 -16
- data/lib/wurk/cron.rb +45 -0
- data/lib/wurk/debounce.rb +2 -2
- data/lib/wurk/error_coalescer.rb +35 -0
- data/lib/wurk/fetcher/at_maxmemory.rb +79 -0
- data/lib/wurk/fetcher/capped.rb +4 -0
- data/lib/wurk/fetcher/many_queues.rb +17 -1
- data/lib/wurk/fetcher/oom_drain.rb +102 -0
- data/lib/wurk/fetcher/orphan_grace.rb +60 -44
- data/lib/wurk/fetcher/private_list_key.rb +1 -16
- data/lib/wurk/fetcher/reaper.rb +169 -127
- data/lib/wurk/fetcher/reliable.rb +26 -4
- data/lib/wurk/fetcher/self_reconcile.rb +175 -0
- data/lib/wurk/fetcher/unit_of_work.rb +32 -8
- data/lib/wurk/health.rb +23 -4
- data/lib/wurk/health_probe.rb +64 -0
- data/lib/wurk/heartbeat.rb +55 -3
- data/lib/wurk/job/options.rb +9 -1
- data/lib/wurk/job.rb +2 -24
- data/lib/wurk/job_record.rb +17 -9
- data/lib/wurk/job_retry.rb +38 -10
- data/lib/wurk/job_set.rb +13 -5
- data/lib/wurk/launcher.rb +134 -52
- data/lib/wurk/leader.rb +9 -3
- data/lib/wurk/limiter/base.rb +17 -1
- data/lib/wurk/lua/debounce.lua +11 -6
- data/lib/wurk/lua/loader.rb +13 -1
- data/lib/wurk/lua.rb +71 -8
- data/lib/wurk/manager.rb +1 -1
- data/lib/wurk/metrics/process_counters.rb +27 -0
- data/lib/wurk/metrics/prometheus.rb +35 -14
- data/lib/wurk/middleware/current_attributes.rb +26 -10
- data/lib/wurk/middleware/poison_pill.rb +1 -2
- data/lib/wurk/process_set.rb +3 -1
- data/lib/wurk/processor.rb +82 -18
- data/lib/wurk/queue_consumers.rb +134 -0
- data/lib/wurk/rails_boot.rb +40 -2
- data/lib/wurk/rails_reloader.rb +51 -0
- data/lib/wurk/railtie.rb +20 -1
- data/lib/wurk/redact.rb +12 -0
- data/lib/wurk/redis_pool.rb +34 -9
- data/lib/wurk/scheduled.rb +142 -67
- data/lib/wurk/sentry/death_handler.rb +29 -0
- data/lib/wurk/sentry/error_handler.rb +52 -10
- data/lib/wurk/sentry/middleware.rb +8 -1
- data/lib/wurk/sentry/retry_policy.rb +12 -3
- data/lib/wurk/sentry.rb +42 -5
- data/lib/wurk/sorted_entry.rb +38 -13
- data/lib/wurk/swarm/child_boot.rb +42 -31
- data/lib/wurk/swarm/crash_watch.rb +65 -0
- data/lib/wurk/swarm.rb +79 -11
- data/lib/wurk/testing.rb +66 -0
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/extension.rb +6 -1
- data/lib/wurk/work_set.rb +6 -0
- data/lib/wurk/worker/setter.rb +11 -6
- data/lib/wurk/worker.rb +17 -1
- data/lib/wurk.rb +1 -1
- data/vendor/assets/dashboard/assets/{ArgsValue-DDqX6XWH.js → ArgsValue-CmaXArv8.js} +1 -1
- data/vendor/assets/dashboard/assets/{BatchDetail-lIytcn2C.js → BatchDetail-nc5yePZW.js} +1 -1
- data/vendor/assets/dashboard/assets/{Batches-BEumt-1O.js → Batches-DgEcyOZ_.js} +1 -1
- data/vendor/assets/dashboard/assets/{Busy-Vk9HmZcN.js → Busy-CUZncvCy.js} +1 -1
- data/vendor/assets/dashboard/assets/{Cron-CWcM-Cfr.js → Cron-CIZbaaZs.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dashboard-Bgv1kusv.js → Dashboard-BWmfAook.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dead-Di5Rct2H.js → Dead-CGKsTZHq.js} +1 -1
- data/vendor/assets/dashboard/assets/{Extension-LZY5-imc.js → Extension-Doy4bJoA.js} +1 -1
- data/vendor/assets/dashboard/assets/{FilterBox-BDo-7IAy.js → FilterBox-C4bO26_d.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowDetail-CehwRLTt.js → FlowDetail-Dh3SJdFh.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowState-glK2Tnf8.js → FlowState-yb7virqR.js} +1 -1
- data/vendor/assets/dashboard/assets/{Flows-cgy-Fsbp.js → Flows-C5SMRuhq.js} +1 -1
- data/vendor/assets/dashboard/assets/{JobDetailModal-Cuk3jruU.js → JobDetailModal-Cg6sAsJB.js} +1 -1
- data/vendor/assets/dashboard/assets/{Limiters-Ci8G2yZG.js → Limiters-BIxwDOJM.js} +1 -1
- data/vendor/assets/dashboard/assets/{Metrics-CQFUeZTR.js → Metrics-Qp9q0nF8.js} +1 -1
- data/vendor/assets/dashboard/assets/{NotFound-CYjhuDsI.js → NotFound-DwjPxFjU.js} +1 -1
- data/vendor/assets/dashboard/assets/{PageHeader-DjIAFTyX.js → PageHeader-vWeBbSPF.js} +1 -1
- data/vendor/assets/dashboard/assets/{Profiles-DTowqIRg.js → Profiles-7dabYVe3.js} +1 -1
- data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +1 -0
- data/vendor/assets/dashboard/assets/{Retries-SJ3BaKC0.js → Retries-CNX8ZtTH.js} +1 -1
- data/vendor/assets/dashboard/assets/{Scheduled-DxSRpmFZ.js → Scheduled-ClVBj0nc.js} +1 -1
- data/vendor/assets/dashboard/assets/{Search-bBbi5JNK.js → Search-BAsVE2qw.js} +1 -1
- data/vendor/assets/dashboard/assets/{charts-Bg2YBcaD.js → charts-DZs138BV.js} +1 -1
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +1 -0
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +1 -0
- data/vendor/assets/dashboard/assets/{index-BOTVSP9M.js → index-CMzPFxNE.js} +3 -3
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-BRPNyV2M.js → useResetPageOnEmpty-DxkLZCIm.js} +1 -1
- data/vendor/assets/dashboard/assets/{useSort-B-BLJik7.js → useSort-jBYVwXvg.js} +1 -1
- data/vendor/assets/dashboard/assets/{utils-CGyrKpTh.js → utils-B0kQFNE8.js} +1 -1
- data/vendor/assets/dashboard/index.html +3 -3
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +41 -30
- data/vendor/assets/dashboard/assets/Queues-DUGRPjME.js +0 -1
- data/vendor/assets/dashboard/assets/i18n-BZClP62Q.js +0 -1
- data/vendor/assets/dashboard/assets/index-Rewk4eSE.css +0 -1
data/lib/wurk/lua.rb
CHANGED
|
@@ -70,7 +70,13 @@ module Wurk
|
|
|
70
70
|
# Redis, so an unbatched promote of a post-outage backlog (100k+ due
|
|
71
71
|
# members) would block every client for the whole sweep. The Ruby caller
|
|
72
72
|
# loops until a short batch comes back.
|
|
73
|
+
# The first line is a Lua comment to Redis and Valkey. Dragonfly reads it as
|
|
74
|
+
# a per-script grant, because this script alone derives its target key from
|
|
75
|
+
# the payload (`ARGV[2] .. q`) and that server refuses undeclared keys by
|
|
76
|
+
# default. It spares operators the server-wide allow-undeclared-keys flag,
|
|
77
|
+
# which serializes every script and cost about 3x script throughput.
|
|
73
78
|
RELIABLE_SCHEDULE_PROMOTE = <<~LUA
|
|
79
|
+
--!df flags=allow-undeclared-keys
|
|
74
80
|
local jobs = redis.call("zrangebyscore", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, tonumber(ARGV[4]))
|
|
75
81
|
for i = 1, #jobs do
|
|
76
82
|
local job = jobs[i]
|
|
@@ -89,26 +95,80 @@ module Wurk
|
|
|
89
95
|
return #jobs
|
|
90
96
|
LUA
|
|
91
97
|
|
|
98
|
+
# Default scheduler claim: lease up to ARGV[3] due members instead of popping
|
|
99
|
+
# them. Each stays in its set, rescored past any due window, so a lost reply, a
|
|
100
|
+
# failed push or a crash costs a later re-promotion and never the job. The
|
|
101
|
+
# caller ZREMs a member only once it has been pushed.
|
|
102
|
+
#
|
|
103
|
+
# Redis's clock decides both ends, so a host with a skewed clock can neither
|
|
104
|
+
# cut a lease short nor see another host's lease as due: the window is the
|
|
105
|
+
# earlier of the caller's and the server's time, and the expiry counts from the
|
|
106
|
+
# server's. tostring() would round a 10-digit epoch, hence the format.
|
|
107
|
+
# KEYS = [sorted_set]
|
|
108
|
+
# ARGV = [now, lease_seconds, limit]
|
|
109
|
+
# Returns the claimed members, lowest score first.
|
|
110
|
+
SCHEDULED_CLAIM = <<~LUA
|
|
111
|
+
local t = redis.call("TIME")
|
|
112
|
+
local server_now = tonumber(t[1]) + (tonumber(t[2]) / 1000000)
|
|
113
|
+
local window = math.min(tonumber(ARGV[1]), server_now)
|
|
114
|
+
local jobs = redis.call("zrange", KEYS[1], "-inf", string.format("%.6f", window), "byscore", "limit", 0, tonumber(ARGV[3]))
|
|
115
|
+
local expiry = string.format("%.6f", server_now + tonumber(ARGV[2]))
|
|
116
|
+
for i = 1, #jobs do
|
|
117
|
+
redis.call("zadd", KEYS[1], "XX", expiry, jobs[i])
|
|
118
|
+
end
|
|
119
|
+
return jobs
|
|
120
|
+
LUA
|
|
121
|
+
|
|
92
122
|
# Reliable fetch (Pro super_fetch §3) shutdown requeue: atomically move
|
|
93
123
|
# one in-flight job from a per-process private list back to its public
|
|
94
|
-
# queue. The
|
|
95
|
-
#
|
|
96
|
-
#
|
|
97
|
-
#
|
|
98
|
-
#
|
|
99
|
-
#
|
|
100
|
-
#
|
|
124
|
+
# queue. The LPOS guard is the whole point — RPUSH runs only when the job
|
|
125
|
+
# is still in the private list. A job the Processor ACKed in the window
|
|
126
|
+
# between hard_shutdown's cross-thread `job` read and this move is NOT
|
|
127
|
+
# re-pushed, so the job lands in exactly one place and can't double-execute.
|
|
128
|
+
# RPUSH (public tail) not LPUSH so the reclaimed job is fetched next — LMOVE
|
|
129
|
+
# pops the tail — ahead of fresh LPUSH'd enqueues.
|
|
130
|
+
#
|
|
131
|
+
# Push before remove, in this script and the two below. Nothing rolls a
|
|
132
|
+
# script back, and a server can refuse the write mid-script (Dragonfly at
|
|
133
|
+
# maxmemory refuses the RPUSH after applying an LREM): the raise has to leave
|
|
134
|
+
# the job where it was, never in neither place.
|
|
101
135
|
# KEYS = [private_list, public_queue]
|
|
102
136
|
# ARGV = [job_json]
|
|
103
137
|
# Returns 1 when the job was moved, 0 when it was already acked.
|
|
104
138
|
RELIABLE_REQUEUE = <<~LUA
|
|
105
|
-
if redis.call("
|
|
139
|
+
if redis.call("lpos", KEYS[1], ARGV[1]) then
|
|
106
140
|
redis.call("rpush", KEYS[2], ARGV[1])
|
|
141
|
+
redis.call("lrem", KEYS[1], 1, ARGV[1])
|
|
107
142
|
return 1
|
|
108
143
|
end
|
|
109
144
|
return 0
|
|
110
145
|
LUA
|
|
111
146
|
|
|
147
|
+
# UnitOfWork#requeue: unguarded, as the caller owns the unit and a job whose
|
|
148
|
+
# ACK already went out must still go back. Not replay-safe, so it is only
|
|
149
|
+
# ever sent once (Loader.eval_once).
|
|
150
|
+
# KEYS = [private_list, public_queue]
|
|
151
|
+
# ARGV = [job_json]
|
|
152
|
+
REQUEUE_UNIT = <<~LUA
|
|
153
|
+
redis.call("rpush", KEYS[2], ARGV[1])
|
|
154
|
+
return redis.call("lrem", KEYS[1], 1, ARGV[1])
|
|
155
|
+
LUA
|
|
156
|
+
|
|
157
|
+
# The reaper's move off a dead worker's private list: LMOVE private public
|
|
158
|
+
# RIGHT RIGHT, spelled out because Dragonfly at maxmemory answers an LMOVE
|
|
159
|
+
# between shards with the job having popped it and dropped the push. The
|
|
160
|
+
# reply would be the only copy left, and the reaper has no use for it.
|
|
161
|
+
# KEYS = [private_list, public_queue]
|
|
162
|
+
# Returns the job moved, nil when the list is empty.
|
|
163
|
+
RELIABLE_RECLAIM = <<~LUA
|
|
164
|
+
local job = redis.call("lindex", KEYS[1], -1)
|
|
165
|
+
if job then
|
|
166
|
+
redis.call("rpush", KEYS[2], job)
|
|
167
|
+
redis.call("rpop", KEYS[1])
|
|
168
|
+
end
|
|
169
|
+
return job
|
|
170
|
+
LUA
|
|
171
|
+
|
|
112
172
|
# Ent Unique (§3): atomic compare-and-delete of a lock key. Replaces the
|
|
113
173
|
# two-command GET-then-DEL — between those calls the key can expire and a
|
|
114
174
|
# fresh owner can grab it, and the bare DEL would then drop the new
|
|
@@ -213,7 +273,10 @@ module Wurk
|
|
|
213
273
|
SCRIPTS = {
|
|
214
274
|
zpopbyscore: ZPOPBYSCORE,
|
|
215
275
|
reliable_schedule_promote: RELIABLE_SCHEDULE_PROMOTE,
|
|
276
|
+
scheduled_claim: SCHEDULED_CLAIM,
|
|
216
277
|
reliable_requeue: RELIABLE_REQUEUE,
|
|
278
|
+
requeue_unit: REQUEUE_UNIT,
|
|
279
|
+
reliable_reclaim: RELIABLE_RECLAIM,
|
|
217
280
|
fast_delete_job: FAST_DELETE_JOB,
|
|
218
281
|
fast_delete_by_class: FAST_DELETE_BY_CLASS,
|
|
219
282
|
release_if_owner: RELEASE_IF_OWNER,
|
data/lib/wurk/manager.rb
CHANGED
|
@@ -140,7 +140,7 @@ module Wurk
|
|
|
140
140
|
# raising Wurk::Shutdown into the threads, so a job killed mid-perform is
|
|
141
141
|
# re-run once (Sidekiq's at-least-once contract). `job` is read off another
|
|
142
142
|
# thread, so a Processor can ACK between this map and the requeue — but
|
|
143
|
-
# bulk_requeue's
|
|
143
|
+
# bulk_requeue's guard skips the RPUSH on a miss, so a job that
|
|
144
144
|
# finished in that window is not resurrected onto the public queue.
|
|
145
145
|
#
|
|
146
146
|
# The kill sits in an `ensure`: a bulk_requeue that raises (Redis down
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
module Metrics
|
|
5
|
+
# In-memory counters for events with no Redis key of their own, exposed by
|
|
6
|
+
# Metrics::Prometheus. Per process: under a swarm, a scrape sees only the
|
|
7
|
+
# child that owns the health port.
|
|
8
|
+
module ProcessCounters
|
|
9
|
+
@lock = ::Mutex.new
|
|
10
|
+
@counts = Hash.new(0)
|
|
11
|
+
|
|
12
|
+
class << self
|
|
13
|
+
def increment(name, by = 1)
|
|
14
|
+
@lock.synchronize { @counts[name] += by }
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def value(name)
|
|
18
|
+
@lock.synchronize { @counts[name] }
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def reset!
|
|
22
|
+
@lock.synchronize { @counts.clear }
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
require 'json'
|
|
4
4
|
require_relative '../component'
|
|
5
5
|
require_relative '../keys'
|
|
6
|
+
require_relative '../leader'
|
|
6
7
|
require_relative '../version'
|
|
8
|
+
require_relative '../queue_consumers'
|
|
9
|
+
require_relative 'process_counters'
|
|
7
10
|
|
|
8
11
|
module Wurk
|
|
9
12
|
module Metrics
|
|
@@ -18,6 +21,11 @@ module Wurk
|
|
|
18
21
|
# heartbeat identities sharing this process's nonce (the swarm parent
|
|
19
22
|
# stamps one nonce that every child inherits), so a scrape of one pod
|
|
20
23
|
# covers all of its children no matter which child owns the port.
|
|
24
|
+
# `wurk_leader` is the same: 1 on every scrape target in the group that
|
|
25
|
+
# holds the lease.
|
|
26
|
+
# * `wurk_fetch_errors_total` / `wurk_reaper_reclaimed_total` count in the
|
|
27
|
+
# memory of the process that owns the port, so under a swarm they are a
|
|
28
|
+
# sample of one child. `rate` them; a child restart resets them.
|
|
21
29
|
#
|
|
22
30
|
# One Redis checkout per refresh — two pipelines, because the second needs
|
|
23
31
|
# the queue and process names the first returns — and the snapshot is cached
|
|
@@ -29,7 +37,7 @@ module Wurk
|
|
|
29
37
|
TTL = 1.0
|
|
30
38
|
LABEL_ESCAPES = { '\\' => '\\\\', '"' => '\\"', "\n" => '\\n' }.freeze
|
|
31
39
|
LABEL_ESCAPES_RE = /[\\"\n]/
|
|
32
|
-
PROCESS_FIELDS = %w[busy concurrency beat rss quiet].freeze
|
|
40
|
+
PROCESS_FIELDS = %w[busy concurrency beat rss quiet info].freeze
|
|
33
41
|
SCALARS = {
|
|
34
42
|
processed: ['GET', Keys::STAT_PROCESSED],
|
|
35
43
|
failed: ['GET', 'stat:failed'],
|
|
@@ -38,11 +46,13 @@ module Wurk
|
|
|
38
46
|
dead: ['ZCARD', Keys::DEAD]
|
|
39
47
|
}.freeze
|
|
40
48
|
# Names come last: the detail pass is built from them.
|
|
41
|
-
FIRST_PASS = [*SCALARS.values, ['
|
|
49
|
+
FIRST_PASS = [*SCALARS.values, ['GET', Wurk::Leader::DEFAULT_KEY],
|
|
50
|
+
['SMEMBERS', Keys::QUEUES_SET], ['SMEMBERS', Keys::PROCESSES]].freeze
|
|
42
51
|
|
|
43
52
|
QUEUE_GAUGES = [
|
|
44
53
|
['wurk_queue_size', 'Jobs enqueued per queue.', :depth.to_proc],
|
|
45
|
-
['wurk_queue_latency_seconds', 'Age of the oldest job per queue.', :latency.to_proc]
|
|
54
|
+
['wurk_queue_latency_seconds', 'Age of the oldest job per queue.', :latency.to_proc],
|
|
55
|
+
['wurk_queue_consumers', 'Live processes subscribed to the queue, quiet ones included.', :consumers.to_proc]
|
|
46
56
|
].freeze
|
|
47
57
|
CLUSTER_GAUGES = [
|
|
48
58
|
['wurk_scheduled_size', 'Jobs in the schedule set.', :scheduled.to_proc],
|
|
@@ -61,10 +71,10 @@ module Wurk
|
|
|
61
71
|
['wurk_process_quiet', '1 when the process is quiet (TSTP).', ->(row, _now) { row.quiet ? 1 : 0 }]
|
|
62
72
|
].freeze
|
|
63
73
|
|
|
64
|
-
QueueRow = Struct.new(:name, :depth, :latency)
|
|
65
|
-
ProcessRow = Struct.new(:identity, :pid, :busy, :concurrency, :beat, :rss_kb, :quiet)
|
|
74
|
+
QueueRow = Struct.new(:name, :depth, :latency, :consumers)
|
|
75
|
+
ProcessRow = Struct.new(:identity, :pid, :busy, :concurrency, :beat, :rss_kb, :quiet, :queues)
|
|
66
76
|
Snapshot = Struct.new(:ok, :taken_at, :processed, :failed, :scheduled, :retries, :dead, :queues, :processes,
|
|
67
|
-
keyword_init: true)
|
|
77
|
+
:leader, keyword_init: true)
|
|
68
78
|
|
|
69
79
|
def initialize(config, ttl: TTL, nonce: Component::PROCESS_NONCE, hostname: Component.hostname,
|
|
70
80
|
clock: -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) })
|
|
@@ -101,6 +111,10 @@ module Wurk
|
|
|
101
111
|
out = +''
|
|
102
112
|
build_info(out)
|
|
103
113
|
gauge(out, 'wurk_redis_up', 'Whether the last metrics refresh could read Redis.', [[nil, snap.ok ? 1 : 0]])
|
|
114
|
+
counter(out, 'wurk_fetch_errors_total', 'Fetch-loop errors in the process serving this scrape.',
|
|
115
|
+
ProcessCounters.value(:fetch_errors))
|
|
116
|
+
counter(out, 'wurk_reaper_reclaimed_total', 'Orphaned jobs reclaimed by the process serving this scrape.',
|
|
117
|
+
ProcessCounters.value(:reaper_reclaimed))
|
|
104
118
|
if snap.ok
|
|
105
119
|
cluster(out, snap)
|
|
106
120
|
local(out, snap, expected_children, fresh_window)
|
|
@@ -114,12 +128,12 @@ module Wurk
|
|
|
114
128
|
identity.end_with?(@local_suffix) && identity.start_with?(@local_prefix)
|
|
115
129
|
end
|
|
116
130
|
|
|
117
|
-
def fetch(now)
|
|
131
|
+
def fetch(now) # rubocop:disable Metrics/AbcSize
|
|
118
132
|
first, detail = read_redis
|
|
119
133
|
queues, processes = names(first)
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
134
|
+
rows = process_rows(processes, detail.drop(queues.size * 2))
|
|
135
|
+
Snapshot.new(ok: true, taken_at: now, **scalars(first), leader: first[SCALARS.size],
|
|
136
|
+
queues: queue_rows(queues, detail, rows), processes: rows)
|
|
123
137
|
rescue StandardError => e
|
|
124
138
|
@config.logger&.warn { "Wurk::Metrics::Prometheus refresh failed: #{e.class}: #{e.message}" }
|
|
125
139
|
Snapshot.new(ok: false, taken_at: now, queues: [], processes: [])
|
|
@@ -148,20 +162,21 @@ module Wurk
|
|
|
148
162
|
processes.each { |id| pipe.call('HMGET', id, *PROCESS_FIELDS) }
|
|
149
163
|
end
|
|
150
164
|
|
|
151
|
-
def queue_rows(queues, detail)
|
|
165
|
+
def queue_rows(queues, detail, processes)
|
|
166
|
+
consumers = processes.flat_map(&:queues).tally
|
|
152
167
|
queues.each_with_index.map do |name, i|
|
|
153
|
-
QueueRow.new(name, detail[i * 2].to_i, latency(detail[(i * 2) + 1]))
|
|
168
|
+
QueueRow.new(name, detail[i * 2].to_i, latency(detail[(i * 2) + 1]), consumers.fetch(name, 0))
|
|
154
169
|
end
|
|
155
170
|
end
|
|
156
171
|
|
|
157
172
|
# An expired identity (dead process not yet pruned from `processes`)
|
|
158
173
|
# answers HMGET with all nils; it is not a process, so it is dropped.
|
|
159
174
|
def process_rows(processes, fields)
|
|
160
|
-
processes.zip(fields).filter_map do |identity, (busy, concurrency, beat, rss, quiet)|
|
|
175
|
+
processes.zip(fields).filter_map do |identity, (busy, concurrency, beat, rss, quiet, info)|
|
|
161
176
|
next if beat.nil? && busy.nil?
|
|
162
177
|
|
|
163
178
|
ProcessRow.new(identity, identity.split(':')[-2].to_i, busy.to_i, concurrency.to_i,
|
|
164
|
-
beat&.to_f, rss.to_i, quiet == 'true')
|
|
179
|
+
beat&.to_f, rss.to_i, quiet == 'true', QueueConsumers.subscribed(info))
|
|
165
180
|
end
|
|
166
181
|
end
|
|
167
182
|
|
|
@@ -190,6 +205,7 @@ module Wurk
|
|
|
190
205
|
|
|
191
206
|
def local(out, snap, expected, window)
|
|
192
207
|
now = ::Time.now.to_f
|
|
208
|
+
leader_gauge(out, snap)
|
|
193
209
|
if expected
|
|
194
210
|
gauge(out, 'wurk_swarm_children_expected', 'Children this swarm is configured to run.', [[nil, expected]])
|
|
195
211
|
end
|
|
@@ -201,6 +217,11 @@ module Wurk
|
|
|
201
217
|
end
|
|
202
218
|
end
|
|
203
219
|
|
|
220
|
+
def leader_gauge(out, snap)
|
|
221
|
+
gauge(out, 'wurk_leader', '1 when this process group holds the leader lease.',
|
|
222
|
+
[[nil, snap.leader && local?(snap.leader) ? 1 : 0]])
|
|
223
|
+
end
|
|
224
|
+
|
|
204
225
|
def counter(out, name, help, value)
|
|
205
226
|
family(out, name, help, 'counter', [[nil, value]])
|
|
206
227
|
end
|
|
@@ -12,6 +12,10 @@ module Wurk
|
|
|
12
12
|
# Multiple → `"cattr"`, `"cattr_1"`, `"cattr_2"`, … (keys mirror Sidekiq's
|
|
13
13
|
# naming exactly: wire-compat sacred).
|
|
14
14
|
#
|
|
15
|
+
# Classes are kept by name and resolved per job, as a String registered from
|
|
16
|
+
# an initializer has to be. It also means a class object passed here doesn't
|
|
17
|
+
# go stale once development code reloading replaces the constant.
|
|
18
|
+
#
|
|
15
19
|
# Spec: docs/target/sidekiq-free.md §10.3 and §2.2.
|
|
16
20
|
module CurrentAttributes
|
|
17
21
|
PERSISTENT_KEY = 'cattr'
|
|
@@ -21,12 +25,22 @@ module Wurk
|
|
|
21
25
|
# no-op: `add` already dedupes by klass, so calling `persist` twice
|
|
22
26
|
# with the same set replaces the old entry with the new args.
|
|
23
27
|
def persist(klass_or_array, config = Wurk.configuration)
|
|
24
|
-
classes = Array(klass_or_array)
|
|
28
|
+
classes = Array(klass_or_array).map { |klass| reference_for(klass) }
|
|
25
29
|
raise ArgumentError, 'persist requires at least one CurrentAttributes class' if classes.empty?
|
|
26
30
|
|
|
27
31
|
config.client_middleware.add(Save, classes)
|
|
28
32
|
config.client_middleware.add(Load, classes)
|
|
29
|
-
|
|
33
|
+
# Outermost, so every other server middleware sees the job's attributes.
|
|
34
|
+
config.server_middleware.prepend(Load, classes)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# An anonymous class has no name to look up again, so it stays as given.
|
|
38
|
+
def reference_for(klass)
|
|
39
|
+
klass.is_a?(Module) ? klass.name || klass : klass.to_s
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def resolve(reference)
|
|
43
|
+
reference.is_a?(String) ? Object.const_get(reference) : reference
|
|
30
44
|
end
|
|
31
45
|
|
|
32
46
|
# Composes the wire key for the Nth registered class. Sidekiq numbers
|
|
@@ -50,7 +64,8 @@ module Wurk
|
|
|
50
64
|
end
|
|
51
65
|
|
|
52
66
|
# Client-side: snapshot each registered CurrentAttributes class into
|
|
53
|
-
# the job hash. Caller-supplied keys take precedence (`||=`)
|
|
67
|
+
# the job hash. Caller-supplied keys take precedence (`||=`), and a class
|
|
68
|
+
# with nothing set adds no key, so the payload doesn't grow an empty one.
|
|
54
69
|
class Save
|
|
55
70
|
include Wurk::Middleware::ClientMiddleware
|
|
56
71
|
|
|
@@ -59,9 +74,9 @@ module Wurk
|
|
|
59
74
|
end
|
|
60
75
|
|
|
61
76
|
def call(_job_class, job, _queue, _redis_pool)
|
|
62
|
-
@classes.each_with_index do |
|
|
63
|
-
|
|
64
|
-
job[
|
|
77
|
+
@classes.each_with_index do |reference, idx|
|
|
78
|
+
attrs = CurrentAttributes.snapshot(CurrentAttributes.resolve(reference))
|
|
79
|
+
job[CurrentAttributes.key_for(idx)] ||= attrs unless attrs.empty?
|
|
65
80
|
end
|
|
66
81
|
yield
|
|
67
82
|
end
|
|
@@ -87,15 +102,16 @@ module Wurk
|
|
|
87
102
|
end
|
|
88
103
|
|
|
89
104
|
def call(_job_or_class, job, _queue, _redis_pool = nil)
|
|
90
|
-
|
|
91
|
-
|
|
105
|
+
classes = @classes.map { |reference| CurrentAttributes.resolve(reference) }
|
|
106
|
+
previous = classes.map { |klass| CurrentAttributes.snapshot(klass) }
|
|
107
|
+
classes.each_with_index do |klass, idx|
|
|
92
108
|
CurrentAttributes.restore(klass, job[CurrentAttributes.key_for(idx)])
|
|
93
109
|
end
|
|
94
110
|
yield
|
|
95
111
|
ensure
|
|
96
112
|
previous&.each_with_index do |attrs, idx|
|
|
97
|
-
|
|
98
|
-
CurrentAttributes.restore(
|
|
113
|
+
classes[idx].reset
|
|
114
|
+
CurrentAttributes.restore(classes[idx], attrs)
|
|
99
115
|
end
|
|
100
116
|
end
|
|
101
117
|
end
|
|
@@ -12,8 +12,7 @@ module Wurk
|
|
|
12
12
|
# When a job is recovered out of a dead process's private list, we
|
|
13
13
|
# INCR a per-jid counter at `super_fetch:recovered:<jid>` with a 72h
|
|
14
14
|
# TTL. Recoveries up to RECOVERY_THRESHOLD (3) requeue; the one that takes
|
|
15
|
-
# the counter past it (the 4th orphaning) is the poison pill,
|
|
16
|
-
# super_fetch's `count <= max_recoveries` recover_orphan script: the payload is moved to the dead set,
|
|
15
|
+
# the counter past it (the 4th orphaning) is the poison pill: the payload is moved to the dead set,
|
|
17
16
|
# `jobs.poison` is emitted to statsd, and recovery callbacks fire so
|
|
18
17
|
# operators can be paged.
|
|
19
18
|
#
|
data/lib/wurk/process_set.rb
CHANGED
|
@@ -121,7 +121,9 @@ module Wurk
|
|
|
121
121
|
# claimed (heartbeat is fresher).
|
|
122
122
|
def build_process(attrs)
|
|
123
123
|
hash = Wurk.load_json(attrs.fetch('info')).merge(beat_overrides(attrs))
|
|
124
|
-
|
|
124
|
+
# HMGET answers nil for a field the process never wrote: a stock Sidekiq
|
|
125
|
+
# process keeps concurrency only in `info`.
|
|
126
|
+
hash['concurrency'] = attrs['concurrency'].to_i if attrs['concurrency']
|
|
125
127
|
Process.new(hash)
|
|
126
128
|
end
|
|
127
129
|
|
data/lib/wurk/processor.rb
CHANGED
|
@@ -6,6 +6,7 @@ require_relative 'dead_set'
|
|
|
6
6
|
require_relative 'job_logger'
|
|
7
7
|
require_relative 'job_retry'
|
|
8
8
|
require_relative 'profiler'
|
|
9
|
+
require_relative 'rails_reloader'
|
|
9
10
|
|
|
10
11
|
module Wurk
|
|
11
12
|
# Inside each Manager, N Processors run in parallel. Each owns one thread,
|
|
@@ -23,7 +24,7 @@ module Wurk
|
|
|
23
24
|
# payload survives in the private list and is reclaimed on next boot.
|
|
24
25
|
#
|
|
25
26
|
# Spec: docs/target/sidekiq-free.md §14.
|
|
26
|
-
class Processor
|
|
27
|
+
class Processor # rubocop:disable Metrics/ClassLength
|
|
27
28
|
include Component
|
|
28
29
|
|
|
29
30
|
# Interrupt masks for the two `Thread.handle_interrupt` scopes every job
|
|
@@ -35,6 +36,20 @@ module Wurk
|
|
|
35
36
|
ALLOW_SHUTDOWN_INTERRUPTS = { Wurk::Shutdown => :immediate }.freeze
|
|
36
37
|
private_constant :ALLOW_SHUTDOWN_INTERRUPTS
|
|
37
38
|
|
|
39
|
+
# An internal failure that says something about Redis or the pool and nothing
|
|
40
|
+
# about the job, so the job goes back to run again. Any other StandardError
|
|
41
|
+
# escaping the retry layer comes from the payload itself (a `retry_count`
|
|
42
|
+
# that is not a number, args that cannot be re-serialized into a retry) and
|
|
43
|
+
# will fail the same way every time.
|
|
44
|
+
TRANSIENT_FAILURES = [RedisClient::Error, ConnectionPool::Error, ConnectionPool::TimeoutError].freeze
|
|
45
|
+
private_constant :TRANSIENT_FAILURES
|
|
46
|
+
|
|
47
|
+
# Pause after putting a job back on an internal failure. The job goes to the
|
|
48
|
+
# tail, which is the next fetch, so without one a failure that persists
|
|
49
|
+
# re-runs `perform` as fast as the thread can loop.
|
|
50
|
+
REQUEUE_PAUSE = 1
|
|
51
|
+
private_constant :REQUEUE_PAUSE
|
|
52
|
+
|
|
38
53
|
# Stand-in for the default reloader, `proc { |&b| b.call }`. That default
|
|
39
54
|
# is an identity wrapper, but a block param (`|&b|`) forces MRI to reify
|
|
40
55
|
# the dispatch block into a Proc on every job just to call it straight
|
|
@@ -165,6 +180,11 @@ module Wurk
|
|
|
165
180
|
end
|
|
166
181
|
end
|
|
167
182
|
|
|
183
|
+
@vm_exiting = false
|
|
184
|
+
at_exit { @vm_exiting = true }
|
|
185
|
+
|
|
186
|
+
def self.vm_exiting? = @vm_exiting
|
|
187
|
+
|
|
168
188
|
PROCESSED = Counter.new
|
|
169
189
|
FAILURE = Counter.new
|
|
170
190
|
EXPIRED = Counter.new
|
|
@@ -173,14 +193,15 @@ module Wurk
|
|
|
173
193
|
private
|
|
174
194
|
|
|
175
195
|
# Only the untouched framework default is swapped out — a host-supplied
|
|
176
|
-
# reloader
|
|
177
|
-
#
|
|
178
|
-
#
|
|
179
|
-
#
|
|
196
|
+
# reloader is used exactly as given, and nil opts out of wrapping. `equal?`
|
|
197
|
+
# against DEFAULTS is sound because Configuration's deep-dup copies
|
|
198
|
+
# Hashes/Arrays only, so an unset `:reloader` is still the very Proc object
|
|
199
|
+
# DEFAULTS holds.
|
|
180
200
|
def resolve_reloader(configured)
|
|
181
|
-
return IdentityReloader if configured.nil?
|
|
201
|
+
return IdentityReloader if configured.nil?
|
|
202
|
+
return configured unless configured.equal?(Configuration::DEFAULTS[:reloader])
|
|
182
203
|
|
|
183
|
-
|
|
204
|
+
RailsReloader.default || IdentityReloader
|
|
184
205
|
end
|
|
185
206
|
|
|
186
207
|
def run
|
|
@@ -196,16 +217,20 @@ module Wurk
|
|
|
196
217
|
flush_acks
|
|
197
218
|
Thread.current[:wurk_capsule] = outer_capsule
|
|
198
219
|
end
|
|
199
|
-
@callback&.call(self)
|
|
200
220
|
rescue Wurk::Shutdown
|
|
201
|
-
|
|
221
|
+
nil
|
|
202
222
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
203
223
|
# Sidekiq hands the error to the manager (processor.rb `@callback.call(
|
|
204
224
|
# self, ex)`); reporting it here keeps it visible, and the re-raise ends
|
|
205
225
|
# the thread so the Manager's replacement starts clean.
|
|
206
226
|
handle_exception(e, { context: '!shutdown' })
|
|
207
|
-
@callback&.call(self)
|
|
208
227
|
raise
|
|
228
|
+
ensure
|
|
229
|
+
# In an ensure because `Thread.exit` from a job unwinds through here with
|
|
230
|
+
# no exception at all, and a Manager never told would run one thread short
|
|
231
|
+
# for the life of the process. The VM unwinds threads the same way as it
|
|
232
|
+
# exits, and asking for a replacement then only raises ThreadError.
|
|
233
|
+
@callback&.call(self) unless Processor.vm_exiting?
|
|
209
234
|
end
|
|
210
235
|
|
|
211
236
|
# The fetcher holds each finished job's LREM until a fetch can pipeline it
|
|
@@ -231,7 +256,7 @@ module Wurk
|
|
|
231
256
|
rescue Wurk::Shutdown
|
|
232
257
|
nil
|
|
233
258
|
rescue StandardError => e
|
|
234
|
-
|
|
259
|
+
@capsule.config.report_fetch_error(e)
|
|
235
260
|
sleep(1)
|
|
236
261
|
nil
|
|
237
262
|
end
|
|
@@ -245,6 +270,8 @@ module Wurk
|
|
|
245
270
|
# outcome is booked.
|
|
246
271
|
def process(uow)
|
|
247
272
|
Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) { process_deferred(uow) }
|
|
273
|
+
ensure
|
|
274
|
+
uow.release_claim if uow.respond_to?(:release_claim)
|
|
248
275
|
end
|
|
249
276
|
|
|
250
277
|
def process_deferred(uow)
|
|
@@ -271,12 +298,10 @@ module Wurk
|
|
|
271
298
|
rescue Wurk::Shutdown
|
|
272
299
|
# Don't ack — UoW stays in private list and is reclaimed on reboot.
|
|
273
300
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
274
|
-
# Escaped the retry layer
|
|
275
|
-
#
|
|
276
|
-
# reaper will never reclaim its private list — put the job back now or
|
|
277
|
-
# it sits there until the process restarts.
|
|
301
|
+
# Escaped the retry layer, so no retry was booked. This owner is alive,
|
|
302
|
+
# so the reaper will never reclaim its private list.
|
|
278
303
|
handle_exception(e, { context: 'Internal exception!', job: job_hash, jobstr: jobstr })
|
|
279
|
-
|
|
304
|
+
ack = settle_internal_failure(uow, job_hash, jobstr, e)
|
|
280
305
|
ensure
|
|
281
306
|
# This frame is the one every exit path passes through, which is why a
|
|
282
307
|
# global-concurrency slot is given back from here and nowhere else: a
|
|
@@ -306,12 +331,45 @@ module Wurk
|
|
|
306
331
|
handle_exception(handled.cause || handled, { context: 'Job raised exception', job: job_hash })
|
|
307
332
|
end
|
|
308
333
|
|
|
334
|
+
# Whether the job is finished with (ACK it). Redis or the pool failing, or a
|
|
335
|
+
# process-level condition (NoMemoryError and friends), is no verdict on the
|
|
336
|
+
# job: it goes back to run again. A job that cannot be booked for any other
|
|
337
|
+
# reason would loop forever, re-running `perform` every time, so it goes to
|
|
338
|
+
# the morgue instead.
|
|
339
|
+
def settle_internal_failure(uow, job_hash, jobstr, error)
|
|
340
|
+
return kill_unbookable(job_hash, jobstr, error) if unbookable?(error)
|
|
341
|
+
|
|
342
|
+
requeue(uow)
|
|
343
|
+
sleep(REQUEUE_PAUSE) unless @done
|
|
344
|
+
false
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
def unbookable?(error)
|
|
348
|
+
error.is_a?(::StandardError) && TRANSIENT_FAILURES.none? { |klass| error.is_a?(klass) }
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
# Death handlers fire when there is a job to tell them about, so a batch
|
|
352
|
+
# records the death and a unique lock is released, as for any other job that
|
|
353
|
+
# died on its own. A kill that fails leaves the job in the private list, which
|
|
354
|
+
# is where it was before this ran; Fetcher::SelfReconcile picks it up.
|
|
355
|
+
def kill_unbookable(job_hash, jobstr, error)
|
|
356
|
+
dead = DeadSet.new
|
|
357
|
+
job_hash.is_a?(::Hash) ? dead.kill(jobstr, ex: error) : dead.kill_raw(jobstr)
|
|
358
|
+
true
|
|
359
|
+
rescue StandardError => e
|
|
360
|
+
handle_exception(e, { context: 'Error killing a job after an internal exception' })
|
|
361
|
+
false
|
|
362
|
+
end
|
|
363
|
+
|
|
309
364
|
# Reported, never raised: we are already handling an internal error, and
|
|
310
365
|
# a Redis that refused the retry ZADD may refuse this too — the job then
|
|
311
|
-
# stays in the private list, which is where it was before this ran
|
|
366
|
+
# stays in the private list, which is where it was before this ran, until
|
|
367
|
+
# Fetcher::SelfReconcile finds it. ERROR because nothing else will say that a
|
|
368
|
+
# job is parked out of sight for the next couple of minutes.
|
|
312
369
|
def requeue(uow)
|
|
313
370
|
uow.requeue
|
|
314
371
|
rescue StandardError => e
|
|
372
|
+
logger.error { "Could not requeue #{uow.queue_name} job after an internal exception: #{e.class}: #{e.message}" }
|
|
315
373
|
handle_exception(e, { context: 'Error requeueing a job after an internal exception' })
|
|
316
374
|
end
|
|
317
375
|
|
|
@@ -346,8 +404,14 @@ module Wurk
|
|
|
346
404
|
# morgue exactly like send_to_morgue does — spec §31.8/§31.9).
|
|
347
405
|
# Returns nil to signal "no further processing" — the ACK that retires the
|
|
348
406
|
# payload belongs to #process's ensure, which is the one cleanup path.
|
|
407
|
+
#
|
|
408
|
+
# JSON that parses to something other than an object (`[1]`, `true`) is the
|
|
409
|
+
# same thing: there is no job in it to run.
|
|
349
410
|
def parse_or_kill(jobstr)
|
|
350
|
-
Wurk.load_json(jobstr)
|
|
411
|
+
job_hash = Wurk.load_json(jobstr)
|
|
412
|
+
raise ::JSON::ParserError, 'job payload is not a JSON object' unless job_hash.is_a?(::Hash)
|
|
413
|
+
|
|
414
|
+
job_hash
|
|
351
415
|
rescue ::JSON::ParserError => e
|
|
352
416
|
handle_exception(e, { context: 'Invalid JSON', jobstr: jobstr })
|
|
353
417
|
DeadSet.new.kill_raw(jobstr)
|