wurk 1.9.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/wurk/api_controller.rb +4 -1
- data/lib/wurk/boot_checks.rb +87 -0
- data/lib/wurk/capsule.rb +33 -8
- data/lib/wurk/cli.rb +73 -8
- data/lib/wurk/client/buffered.rb +240 -26
- data/lib/wurk/compat.rb +11 -0
- data/lib/wurk/configuration.rb +114 -16
- data/lib/wurk/cron.rb +45 -0
- data/lib/wurk/debounce.rb +2 -2
- data/lib/wurk/error_coalescer.rb +35 -0
- data/lib/wurk/fetcher/at_maxmemory.rb +79 -0
- data/lib/wurk/fetcher/capped.rb +4 -0
- data/lib/wurk/fetcher/many_queues.rb +17 -1
- data/lib/wurk/fetcher/oom_drain.rb +102 -0
- data/lib/wurk/fetcher/orphan_grace.rb +60 -44
- data/lib/wurk/fetcher/private_list_key.rb +1 -16
- data/lib/wurk/fetcher/reaper.rb +169 -127
- data/lib/wurk/fetcher/reliable.rb +26 -4
- data/lib/wurk/fetcher/self_reconcile.rb +175 -0
- data/lib/wurk/fetcher/unit_of_work.rb +32 -8
- data/lib/wurk/health.rb +23 -4
- data/lib/wurk/health_probe.rb +64 -0
- data/lib/wurk/heartbeat.rb +55 -3
- data/lib/wurk/job/options.rb +9 -1
- data/lib/wurk/job.rb +2 -24
- data/lib/wurk/job_record.rb +17 -9
- data/lib/wurk/job_retry.rb +38 -10
- data/lib/wurk/job_set.rb +13 -5
- data/lib/wurk/launcher.rb +134 -52
- data/lib/wurk/leader.rb +9 -3
- data/lib/wurk/limiter/base.rb +17 -1
- data/lib/wurk/lua/debounce.lua +11 -6
- data/lib/wurk/lua/loader.rb +13 -1
- data/lib/wurk/lua.rb +71 -8
- data/lib/wurk/manager.rb +1 -1
- data/lib/wurk/metrics/process_counters.rb +27 -0
- data/lib/wurk/metrics/prometheus.rb +35 -14
- data/lib/wurk/middleware/current_attributes.rb +26 -10
- data/lib/wurk/middleware/poison_pill.rb +1 -2
- data/lib/wurk/process_set.rb +3 -1
- data/lib/wurk/processor.rb +82 -18
- data/lib/wurk/queue_consumers.rb +134 -0
- data/lib/wurk/rails_boot.rb +40 -2
- data/lib/wurk/rails_reloader.rb +51 -0
- data/lib/wurk/railtie.rb +20 -1
- data/lib/wurk/redact.rb +12 -0
- data/lib/wurk/redis_pool.rb +34 -9
- data/lib/wurk/scheduled.rb +142 -67
- data/lib/wurk/sentry/death_handler.rb +29 -0
- data/lib/wurk/sentry/error_handler.rb +52 -10
- data/lib/wurk/sentry/middleware.rb +8 -1
- data/lib/wurk/sentry/retry_policy.rb +12 -3
- data/lib/wurk/sentry.rb +42 -5
- data/lib/wurk/sorted_entry.rb +38 -13
- data/lib/wurk/swarm/child_boot.rb +42 -31
- data/lib/wurk/swarm/crash_watch.rb +65 -0
- data/lib/wurk/swarm.rb +79 -11
- data/lib/wurk/testing.rb +66 -0
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/extension.rb +6 -1
- data/lib/wurk/work_set.rb +6 -0
- data/lib/wurk/worker/setter.rb +11 -6
- data/lib/wurk/worker.rb +17 -1
- data/lib/wurk.rb +1 -1
- data/vendor/assets/dashboard/assets/{ArgsValue-DDqX6XWH.js → ArgsValue-CmaXArv8.js} +1 -1
- data/vendor/assets/dashboard/assets/{BatchDetail-lIytcn2C.js → BatchDetail-nc5yePZW.js} +1 -1
- data/vendor/assets/dashboard/assets/{Batches-BEumt-1O.js → Batches-DgEcyOZ_.js} +1 -1
- data/vendor/assets/dashboard/assets/{Busy-Vk9HmZcN.js → Busy-CUZncvCy.js} +1 -1
- data/vendor/assets/dashboard/assets/{Cron-CWcM-Cfr.js → Cron-CIZbaaZs.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dashboard-Bgv1kusv.js → Dashboard-BWmfAook.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dead-Di5Rct2H.js → Dead-CGKsTZHq.js} +1 -1
- data/vendor/assets/dashboard/assets/{Extension-LZY5-imc.js → Extension-Doy4bJoA.js} +1 -1
- data/vendor/assets/dashboard/assets/{FilterBox-BDo-7IAy.js → FilterBox-C4bO26_d.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowDetail-CehwRLTt.js → FlowDetail-Dh3SJdFh.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowState-glK2Tnf8.js → FlowState-yb7virqR.js} +1 -1
- data/vendor/assets/dashboard/assets/{Flows-cgy-Fsbp.js → Flows-C5SMRuhq.js} +1 -1
- data/vendor/assets/dashboard/assets/{JobDetailModal-Cuk3jruU.js → JobDetailModal-Cg6sAsJB.js} +1 -1
- data/vendor/assets/dashboard/assets/{Limiters-Ci8G2yZG.js → Limiters-BIxwDOJM.js} +1 -1
- data/vendor/assets/dashboard/assets/{Metrics-CQFUeZTR.js → Metrics-Qp9q0nF8.js} +1 -1
- data/vendor/assets/dashboard/assets/{NotFound-CYjhuDsI.js → NotFound-DwjPxFjU.js} +1 -1
- data/vendor/assets/dashboard/assets/{PageHeader-DjIAFTyX.js → PageHeader-vWeBbSPF.js} +1 -1
- data/vendor/assets/dashboard/assets/{Profiles-DTowqIRg.js → Profiles-7dabYVe3.js} +1 -1
- data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +1 -0
- data/vendor/assets/dashboard/assets/{Retries-SJ3BaKC0.js → Retries-CNX8ZtTH.js} +1 -1
- data/vendor/assets/dashboard/assets/{Scheduled-DxSRpmFZ.js → Scheduled-ClVBj0nc.js} +1 -1
- data/vendor/assets/dashboard/assets/{Search-bBbi5JNK.js → Search-BAsVE2qw.js} +1 -1
- data/vendor/assets/dashboard/assets/{charts-Bg2YBcaD.js → charts-DZs138BV.js} +1 -1
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +1 -0
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +1 -0
- data/vendor/assets/dashboard/assets/{index-BOTVSP9M.js → index-CMzPFxNE.js} +3 -3
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-BRPNyV2M.js → useResetPageOnEmpty-DxkLZCIm.js} +1 -1
- data/vendor/assets/dashboard/assets/{useSort-B-BLJik7.js → useSort-jBYVwXvg.js} +1 -1
- data/vendor/assets/dashboard/assets/{utils-CGyrKpTh.js → utils-B0kQFNE8.js} +1 -1
- data/vendor/assets/dashboard/index.html +3 -3
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +41 -30
- data/vendor/assets/dashboard/assets/Queues-DUGRPjME.js +0 -1
- data/vendor/assets/dashboard/assets/i18n-BZClP62Q.js +0 -1
- data/vendor/assets/dashboard/assets/index-Rewk4eSE.css +0 -1
|
@@ -8,6 +8,7 @@ require_relative '../lua'
|
|
|
8
8
|
require_relative '../fetcher'
|
|
9
9
|
require_relative 'capped'
|
|
10
10
|
require_relative 'many_queues'
|
|
11
|
+
require_relative 'at_maxmemory'
|
|
11
12
|
|
|
12
13
|
module Wurk
|
|
13
14
|
class Fetcher
|
|
@@ -45,6 +46,7 @@ module Wurk
|
|
|
45
46
|
# path is unchanged than a second loop kept in step by hand.
|
|
46
47
|
prepend Capped
|
|
47
48
|
include ManyQueues
|
|
49
|
+
include AtMaxmemory
|
|
48
50
|
|
|
49
51
|
# Default BLMOVE block timeout; overridable via config.fetch_poll_interval.
|
|
50
52
|
TIMEOUT = 2
|
|
@@ -89,6 +91,12 @@ module Wurk
|
|
|
89
91
|
# misses a bump by microseconds picks it up on its next pass.
|
|
90
92
|
PAUSED_GENERATION_LOCK = Mutex.new
|
|
91
93
|
|
|
94
|
+
# Thread variable holding the unit this thread claimed and has not settled;
|
|
95
|
+
# what Fetcher::SelfReconcile reads to tell a running job from a stranded
|
|
96
|
+
# one. A thread variable, not `Thread#[]`: those are per fiber, and a job
|
|
97
|
+
# running inside a Fiber would read as holding nothing.
|
|
98
|
+
CLAIM_KEY = :wurk_fetcher_claim
|
|
99
|
+
|
|
92
100
|
@paused_generation = 0
|
|
93
101
|
|
|
94
102
|
class << self
|
|
@@ -118,6 +126,8 @@ module Wurk
|
|
|
118
126
|
@prefixed_queues = nil
|
|
119
127
|
@prefixed_source = nil
|
|
120
128
|
@claim_keys = nil
|
|
129
|
+
@rest_empty_until = 0.0
|
|
130
|
+
@oom_reported_at = nil
|
|
121
131
|
end
|
|
122
132
|
|
|
123
133
|
# Take custody of a finished job's LREM instead of sending it now. One
|
|
@@ -140,7 +150,7 @@ module Wurk
|
|
|
140
150
|
# them — and from #bulk_requeue, where it is a correctness requirement
|
|
141
151
|
# rather than an optimization: a finished job whose LREM is still pending
|
|
142
152
|
# is not in Manager#hard_shutdown's in-flight list, so the requeue Lua's
|
|
143
|
-
#
|
|
153
|
+
# guard would still find its payload, RPUSH it onto the public
|
|
144
154
|
# queue, and run it a second time on every graceful shutdown.
|
|
145
155
|
def flush_pending_acks
|
|
146
156
|
pending = claim_pending_acks
|
|
@@ -186,11 +196,19 @@ module Wurk
|
|
|
186
196
|
end
|
|
187
197
|
|
|
188
198
|
walk(queues)
|
|
199
|
+
rescue RedisClient::OutOfMemoryError => e
|
|
200
|
+
claim_at_maxmemory(queues, e)
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# The units whose ACK is waiting here for a fetch to ride. A snapshot, for
|
|
204
|
+
# Fetcher::SelfReconcile.
|
|
205
|
+
def held_acks
|
|
206
|
+
@pending_lock.synchronize { @pending_acks.values.flatten }
|
|
189
207
|
end
|
|
190
208
|
|
|
191
209
|
# Called on shutdown for jobs the Processor couldn't finish in time.
|
|
192
210
|
# Atomically moves each still-private UoW back to its public queue via
|
|
193
|
-
# the RELIABLE_REQUEUE Lua (
|
|
211
|
+
# the RELIABLE_REQUEUE Lua (guarded RPUSH, then LREM): the job leaves the
|
|
194
212
|
# per-process private list and reappears on the public queue in one hop,
|
|
195
213
|
# so it's visible immediately after a deploy instead of waiting for the
|
|
196
214
|
# next boot's reaper. The guard makes the move idempotent against the
|
|
@@ -208,6 +226,8 @@ module Wurk
|
|
|
208
226
|
return if in_progress.nil? || in_progress.empty?
|
|
209
227
|
|
|
210
228
|
config.redis { |conn| requeue_pipelined(conn, in_progress) }
|
|
229
|
+
rescue RedisClient::OutOfMemoryError => e
|
|
230
|
+
requeue_past_maxmemory(in_progress, e)
|
|
211
231
|
end
|
|
212
232
|
|
|
213
233
|
# Prefixed queue keys (`queue:<name>`) in fetch order. Strict mode
|
|
@@ -440,8 +460,10 @@ module Wurk
|
|
|
440
460
|
# Both fetch paths hand the unit every string its Processor and its ACK
|
|
441
461
|
# will need, so neither re-derives one per job.
|
|
442
462
|
def unit_of_work(public_q, priv, name, job)
|
|
443
|
-
UnitOfWork.new(queue: public_q, queue_name: name, private_queue: priv,
|
|
444
|
-
|
|
463
|
+
uow = UnitOfWork.new(queue: public_q, queue_name: name, private_queue: priv,
|
|
464
|
+
job: job, config: config, fetcher: self)
|
|
465
|
+
::Thread.current.thread_variable_set(CLAIM_KEY, uow)
|
|
466
|
+
uow
|
|
445
467
|
end
|
|
446
468
|
|
|
447
469
|
# BLMOVE block timeout for an empty poll. `config.fetch_poll_interval`
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../keys'
|
|
4
|
+
require_relative '../lua'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
class Fetcher
|
|
8
|
+
# A live worker's check of its own private lists.
|
|
9
|
+
#
|
|
10
|
+
# The Reaper leaves a live owner's list alone, and a live owner only ever
|
|
11
|
+
# settles the jobs its threads know about. So a payload no thread knows about
|
|
12
|
+
# would wait there until the process exits. Two ways in:
|
|
13
|
+
# * a claim applied but its reply was lost (Redis restart, a stall past the
|
|
14
|
+
# read timeout, a dropped socket): the client re-sends and claims another
|
|
15
|
+
# job, so one job per thread per incident;
|
|
16
|
+
# * a job that failed while Redis was down, where the retry booking and the
|
|
17
|
+
# requeue both raised.
|
|
18
|
+
#
|
|
19
|
+
# A payload counts as stranded only if it is in the list and held by nothing:
|
|
20
|
+
# no thread's claim (Fetcher::Reliable::CLAIM_KEY, which covers the whole of
|
|
21
|
+
# a running job however long it runs) and no pending ACK. The list is read
|
|
22
|
+
# first and the holders second, so a claim landing between the two reads as
|
|
23
|
+
# held. What the reads cannot see is a unit in transit: claimed but not yet
|
|
24
|
+
# stamped, or an ACK on the wire. Those last microseconds to seconds, which
|
|
25
|
+
# is why one sighting proves nothing: it has to be seen stranded again at
|
|
26
|
+
# least `confirm` seconds later before it moves.
|
|
27
|
+
#
|
|
28
|
+
# Counted rather than compared as sets, because a client re-send can leave two
|
|
29
|
+
# byte-identical payloads, and a running job must not vouch for its twin.
|
|
30
|
+
class SelfReconcile
|
|
31
|
+
# Above anything a unit can spend in transit: the longest a held ACK sits on
|
|
32
|
+
# the wire is the pool's blip backoff plus its socket timeouts, ~18s.
|
|
33
|
+
DEFAULT_CONFIRM = 30
|
|
34
|
+
|
|
35
|
+
Suspect = Struct.new(:copies, :since)
|
|
36
|
+
|
|
37
|
+
# `on_moved` receives [public_queue, payload] for each payload put back, so
|
|
38
|
+
# the Reaper can run it through the same poison accounting as a reclaim.
|
|
39
|
+
def initialize(config, confirm: DEFAULT_CONFIRM, stop: -> { false }, on_moved: nil)
|
|
40
|
+
@config = config
|
|
41
|
+
@confirm = confirm
|
|
42
|
+
@stop = stop
|
|
43
|
+
@on_moved = on_moved
|
|
44
|
+
@suspects = {}
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# One pass. Returns how many payloads went back to their public queues.
|
|
48
|
+
def sweep
|
|
49
|
+
lists = own_lists
|
|
50
|
+
return 0 if lists.empty?
|
|
51
|
+
|
|
52
|
+
crowded = crowded_lists(lists)
|
|
53
|
+
return forget_all if crowded.empty?
|
|
54
|
+
|
|
55
|
+
contents = read(crowded.keys)
|
|
56
|
+
settle(crowded, contents, held_by_list)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
# { private_list => public_queue } for every queue a Reliable fetcher here
|
|
62
|
+
# serves. Two capsules on one queue share a list, hence the Hash. A custom
|
|
63
|
+
# `config[:fetch_class]` keeps its own books, so its queues are left alone.
|
|
64
|
+
def own_lists
|
|
65
|
+
@config.capsules.each_value.with_object({}) do |capsule, lists|
|
|
66
|
+
next unless capsule.fetcher.is_a?(Reliable)
|
|
67
|
+
|
|
68
|
+
capsule.queues.uniq.each do |name|
|
|
69
|
+
lists[Reliable.private_queue_name(Keys.queue(name))] = Keys.queue(name)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# A list no longer than what its holders account for has nothing stranded
|
|
75
|
+
# in it, which is every list on every healthy sweep, and LLEN is what keeps
|
|
76
|
+
# a healthy sweep from reading every in-flight payload back over the wire.
|
|
77
|
+
def crowded_lists(lists)
|
|
78
|
+
lengths = read_lengths(lists.keys)
|
|
79
|
+
held = held_by_list
|
|
80
|
+
lists.select { |list, _| lengths[list] > held_count(held, list) }
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def read_lengths(names)
|
|
84
|
+
replies = @config.redis(idempotent: true) do |conn|
|
|
85
|
+
conn.pipelined { |pipe| names.each { |list| pipe.call('LLEN', list) } }
|
|
86
|
+
end
|
|
87
|
+
names.zip(replies).to_h
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def held_count(held, list)
|
|
91
|
+
held.fetch(list, {}).each_value.sum
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def read(names)
|
|
95
|
+
replies = @config.redis(idempotent: true) do |conn|
|
|
96
|
+
conn.pipelined { |pipe| names.each { |list| pipe.call('LRANGE', list, 0, -1) } }
|
|
97
|
+
end
|
|
98
|
+
names.zip(replies).to_h
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# { private_list => { payload => count } } across every thread and fetcher
|
|
102
|
+
# in this process. Thread.list, not a registry: nothing is written per job,
|
|
103
|
+
# and a thread that died mid-job stops vouching for its payload by itself.
|
|
104
|
+
def held_by_list
|
|
105
|
+
held_units.each_with_object({}) do |unit, held|
|
|
106
|
+
tally = (held[unit.private_queue] ||= Hash.new(0))
|
|
107
|
+
tally[unit.job.b] += 1
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def held_units
|
|
112
|
+
claims = ::Thread.list.filter_map { |thread| thread.thread_variable_get(Reliable::CLAIM_KEY) }
|
|
113
|
+
fetchers = @config.capsules.each_value.map(&:fetcher).grep(Reliable)
|
|
114
|
+
claims + fetchers.flat_map(&:held_acks)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def settle(crowded, contents, held)
|
|
118
|
+
now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
119
|
+
following = {}
|
|
120
|
+
moved = contents.sum do |list, payloads|
|
|
121
|
+
stranded(payloads, held[list]).sum do |payload, count|
|
|
122
|
+
judge(list, crowded[list], payload, count, now, following)
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
@suspects = following
|
|
126
|
+
moved
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Moves what has been seen stranded for long enough, remembers the rest.
|
|
130
|
+
def judge(list, public_q, payload, count, now, following) # rubocop:disable Metrics/ParameterLists
|
|
131
|
+
suspect = @suspects.dig(list, payload)
|
|
132
|
+
confirmed = suspect && now - suspect.since >= @confirm
|
|
133
|
+
return put_back(list, public_q, payload, [count, suspect.copies].min) if confirmed
|
|
134
|
+
|
|
135
|
+
(following[list] ||= {})[payload] = Suspect.new(count, suspect&.since || now)
|
|
136
|
+
0
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def stranded(payloads, held)
|
|
140
|
+
held ||= {}
|
|
141
|
+
payloads.map(&:b).tally.filter_map do |payload, count|
|
|
142
|
+
excess = count - held.fetch(payload, 0)
|
|
143
|
+
[payload, excess] if excess.positive?
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def put_back(list, public_q, payload, count)
|
|
148
|
+
moved = 0
|
|
149
|
+
count.times do
|
|
150
|
+
break if @stop.call
|
|
151
|
+
|
|
152
|
+
moved += requeue(list, public_q, payload)
|
|
153
|
+
end
|
|
154
|
+
moved
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# LPOS-guarded, so a replay after a lost reply moves nothing the second
|
|
158
|
+
# time, and a payload settled since the last read stays settled.
|
|
159
|
+
def requeue(list, public_q, payload)
|
|
160
|
+
applied = @config.redis(idempotent: true) do |conn|
|
|
161
|
+
Wurk::Lua::Loader.eval_cached(conn, :reliable_requeue, keys: [list, public_q], argv: [payload])
|
|
162
|
+
rescue RedisClient::OutOfMemoryError => e
|
|
163
|
+
OomDrain.requeue(conn, e, list, public_q, payload)
|
|
164
|
+
end.to_i
|
|
165
|
+
@on_moved&.call(public_q, payload) if applied == 1
|
|
166
|
+
applied
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def forget_all
|
|
170
|
+
@suspects = {}
|
|
171
|
+
0
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
end
|
|
@@ -41,6 +41,19 @@ module Wurk
|
|
|
41
41
|
|
|
42
42
|
UnitOfWork = Struct.new(:queue, :queue_name, :private_queue, :job, :config, :jid, :fetcher,
|
|
43
43
|
:slot_key, :slot_token, keyword_init: true) do
|
|
44
|
+
# The positional form is upstream's documented `UnitOfWork.new(queue,
|
|
45
|
+
# job, config)`, which gem test suites build directly. No fetcher means
|
|
46
|
+
# no private list, so #acknowledge has nothing to do. Anonymous `**` is
|
|
47
|
+
# the cheap spelling for the fetch path, which always passes keywords:
|
|
48
|
+
# one allocation and ~230ns per job (a named **fields costs two).
|
|
49
|
+
def initialize(queue = nil, job = nil, config = nil, **)
|
|
50
|
+
if queue
|
|
51
|
+
super(queue: queue, queue_name: queue.delete_prefix('queue:'), job: job, config: config, **)
|
|
52
|
+
else
|
|
53
|
+
super(**)
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
44
57
|
# Deferred, never skipped: the LREM goes back to the fetcher, which
|
|
45
58
|
# pipelines it in front of the next fetch's LMOVE instead of spending a
|
|
46
59
|
# round trip of its own. Ordering against the job is unchanged — the
|
|
@@ -48,7 +61,7 @@ module Wurk
|
|
|
48
61
|
# so all that moves is the wall clock. Every path that stops fetching
|
|
49
62
|
# flushes first; see #flush_pending_acks.
|
|
50
63
|
def acknowledge
|
|
51
|
-
fetcher
|
|
64
|
+
fetcher&.defer_ack(self)
|
|
52
65
|
end
|
|
53
66
|
|
|
54
67
|
# Queue this unit's ACK into an already-open pipeline. The counter DEL
|
|
@@ -107,18 +120,29 @@ module Wurk
|
|
|
107
120
|
# both lists and runs twice — once from the public queue, once when the
|
|
108
121
|
# reaper reclaims this process's private list.
|
|
109
122
|
#
|
|
110
|
-
#
|
|
123
|
+
# RPUSH then LREM in one script, unconditionally: unlike bulk_requeue
|
|
111
124
|
# (which races a Processor ACKing the same UoW and so guards the push on
|
|
112
|
-
# the
|
|
113
|
-
# went out must still be re-queued rather than dropped.
|
|
125
|
+
# the private copy), the caller here owns the unit, and a job whose ACK
|
|
126
|
+
# already went out must still be re-queued rather than dropped. A script
|
|
127
|
+
# because a MULTI would apply the LREM after a refused RPUSH, losing the
|
|
128
|
+
# job. Unconditional is also why it goes by call_once: redis-client
|
|
129
|
+
# re-sends on a dropped socket, and the replay would push the job a second
|
|
130
|
+
# time. A raise leaves the outcome unknown, and both outcomes are safe:
|
|
131
|
+
# moved, or still in the private list, where Fetcher::SelfReconcile finds
|
|
132
|
+
# it.
|
|
114
133
|
def requeue
|
|
115
134
|
config.redis do |conn|
|
|
116
|
-
conn.
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
end
|
|
135
|
+
next conn.call_once('RPUSH', queue, job) unless private_queue
|
|
136
|
+
|
|
137
|
+
Wurk::Lua::Loader.eval_once(conn, :requeue_unit, keys: [private_queue, queue], argv: [job])
|
|
120
138
|
end
|
|
121
139
|
end
|
|
140
|
+
|
|
141
|
+
# The Processor has settled this unit, one way or the other: stop
|
|
142
|
+
# vouching for its payload. See Fetcher::SelfReconcile.
|
|
143
|
+
def release_claim
|
|
144
|
+
::Thread.current.thread_variable_set(CLAIM_KEY, nil)
|
|
145
|
+
end
|
|
122
146
|
end
|
|
123
147
|
end
|
|
124
148
|
end
|
data/lib/wurk/health.rb
CHANGED
|
@@ -6,14 +6,14 @@ require_relative 'metrics/prometheus'
|
|
|
6
6
|
|
|
7
7
|
module Wurk
|
|
8
8
|
# Thin HTTP listener for k8s liveness/readiness probes. Optional, off by
|
|
9
|
-
# default — opt in with `config.health_check(port: 7433)
|
|
10
|
-
# GET /metrics on that
|
|
9
|
+
# default — opt in with `config.health_check(port: 7433)` or
|
|
10
|
+
# WURK_HEALTH_PORT=7433, then scrape GET /metrics on that port.
|
|
11
11
|
# Mounting Wurk::Engine does not expose Prometheus metrics: this deliberate
|
|
12
12
|
# separation keeps internal metrics off the host application's HTTP surface
|
|
13
13
|
# so access can be restricted independently on the dedicated listener port.
|
|
14
14
|
#
|
|
15
15
|
# Endpoints:
|
|
16
|
-
# * GET /live → 200
|
|
16
|
+
# * GET /live → 200 until the Launcher is shutting down; quiet stays 200.
|
|
17
17
|
# * GET /ready → 200 only when Redis is reachable AND the heartbeat has
|
|
18
18
|
# fired within `ready_window` seconds AND, inside a swarm
|
|
19
19
|
# child, enough of the swarm's children have (see
|
|
@@ -43,6 +43,22 @@ module Wurk
|
|
|
43
43
|
# fleet of one and /ready judges them on their own heartbeat alone.
|
|
44
44
|
attr_accessor :fleet_size
|
|
45
45
|
|
|
46
|
+
# WURK_HEALTH_PORT turns the listener on from the environment, so whoever
|
|
47
|
+
# owns the deployment manifest can add a probe without an application
|
|
48
|
+
# change. `config.health_check` wins when both are set. A bad value raises:
|
|
49
|
+
# a probe silently not listening would restart the pod forever instead.
|
|
50
|
+
def options_from_env(env = ENV)
|
|
51
|
+
raw = env['WURK_HEALTH_PORT'].to_s.strip
|
|
52
|
+
return nil if raw.empty?
|
|
53
|
+
|
|
54
|
+
port = Integer(raw, 10)
|
|
55
|
+
raise ArgumentError unless port.between?(0, 65_535)
|
|
56
|
+
|
|
57
|
+
{ port: port }
|
|
58
|
+
rescue ArgumentError
|
|
59
|
+
raise ArgumentError, "WURK_HEALTH_PORT must be a port number (got #{raw.inspect})"
|
|
60
|
+
end
|
|
61
|
+
|
|
46
62
|
def json(status, **extra)
|
|
47
63
|
::JSON.generate({ status: status }.merge(extra))
|
|
48
64
|
end
|
|
@@ -278,8 +294,11 @@ module Wurk
|
|
|
278
294
|
@fleet ||= Fleet.new(@config, @ready_window)
|
|
279
295
|
end
|
|
280
296
|
|
|
297
|
+
# A quiet process is alive and is finishing its jobs. Failing here would have
|
|
298
|
+
# the orchestrator restart a pod an operator just quieted, and the
|
|
299
|
+
# replacement would fetch again. /ready is the probe that speaks to traffic.
|
|
281
300
|
def live_response
|
|
282
|
-
if @launcher.
|
|
301
|
+
if @launcher.shutting_down?
|
|
283
302
|
[json('down', check: 'live', reason: 'stopping'), 503]
|
|
284
303
|
else
|
|
285
304
|
[json('ok', check: 'live'), 200]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'component'
|
|
4
|
+
require_relative 'keys'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
# `wurk health`: an exec-style liveness probe, for a pod with no HTTP port. It
|
|
8
|
+
# boots no application, so it is fast and only needs to reach Redis (REDIS_URL
|
|
9
|
+
# or --redis-url). Passes when enough processes on this host have a recent
|
|
10
|
+
# heartbeat.
|
|
11
|
+
#
|
|
12
|
+
# A Redis it cannot reach is not this pod's fault, and restarting workers does
|
|
13
|
+
# not bring Redis back, so that passes too unless `--strict`.
|
|
14
|
+
class HealthProbe
|
|
15
|
+
HEALTHY = 0
|
|
16
|
+
UNHEALTHY = 1
|
|
17
|
+
DEFAULT_WINDOW = 60
|
|
18
|
+
|
|
19
|
+
# At most connect + two round trips: about 2s in all against a hung server.
|
|
20
|
+
CLIENT_LIMITS = { connect_timeout: 0.5, read_timeout: 0.6, write_timeout: 0.6, reconnect_attempts: 0 }.freeze
|
|
21
|
+
|
|
22
|
+
def initialize(config, window: DEFAULT_WINDOW, min: 1, strict: false, hostname: Component.hostname, out: $stdout)
|
|
23
|
+
@config = config
|
|
24
|
+
@window = window
|
|
25
|
+
@min = min
|
|
26
|
+
@strict = strict
|
|
27
|
+
@prefix = "#{hostname}:"
|
|
28
|
+
@out = out
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def call
|
|
32
|
+
fresh = fresh_processes
|
|
33
|
+
@out.puts "wurk health: #{fresh} process(es) on this host beat within #{@window}s (need #{@min})"
|
|
34
|
+
fresh >= @min ? HEALTHY : UNHEALTHY
|
|
35
|
+
rescue RedisClient::Error, ConnectionPool::TimeoutError => e
|
|
36
|
+
@out.puts "wurk health: cannot read Redis (#{e.class}: #{e.message})"
|
|
37
|
+
@strict ? UNHEALTHY : HEALTHY
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def fresh_processes
|
|
43
|
+
now = ::Time.now.to_f
|
|
44
|
+
read_local_beats.count { |beat| beat && now - beat.to_f < @window }
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def read_local_beats
|
|
48
|
+
client = build_client
|
|
49
|
+
ids = client.call('SMEMBERS', Keys::PROCESSES).select { |id| id.start_with?(@prefix) }
|
|
50
|
+
ids.empty? ? [] : client.pipelined { |pipe| ids.each { |id| pipe.call('HGET', id, 'beat') } }
|
|
51
|
+
ensure
|
|
52
|
+
client&.close
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Directly on redis-client rather than through RedisPool: the pool's retries
|
|
56
|
+
# and backoff are right for a worker and fatal here. They took 4s on a refused
|
|
57
|
+
# connection and 19s on a hung server, past a kubelet's probe timeout, so a
|
|
58
|
+
# store outage restarted every pod.
|
|
59
|
+
def build_client
|
|
60
|
+
opts = RedisOptions.normalize(@config.redis_config).except(*RedisOptions::POOL_KEYS).merge(CLIENT_LIMITS)
|
|
61
|
+
(RedisOptions.sentinel?(opts) ? RedisClient.sentinel(**opts) : RedisClient.config(**opts)).new_client
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
data/lib/wurk/heartbeat.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'etc'
|
|
4
4
|
|
|
5
5
|
require_relative 'component'
|
|
6
|
+
require_relative 'fetcher/oom_drain'
|
|
6
7
|
require_relative 'keys'
|
|
7
8
|
require_relative 'lua'
|
|
8
9
|
require_relative 'processor'
|
|
@@ -38,12 +39,30 @@ module Wurk
|
|
|
38
39
|
# network partition (rearmed when beat! rescues). `:beat` fires every tick.
|
|
39
40
|
#
|
|
40
41
|
# Spec: docs/target/sidekiq-free.md §12 (Launcher#❤️).
|
|
41
|
-
class Heartbeat
|
|
42
|
+
class Heartbeat # rubocop:disable Metrics/ClassLength
|
|
42
43
|
include Component
|
|
43
44
|
|
|
44
45
|
# Cadence in seconds. Key TTL is 60s — a process is dead after ~6 misses.
|
|
45
46
|
BEAT_PAUSE = 10
|
|
46
47
|
TTL_SECONDS = 60
|
|
48
|
+
|
|
49
|
+
OOM_REFRESH_SLOTS = "#{Fetcher::OomDrain::HEADER}#{Lua::SCRIPTS.fetch(:refresh_slots)}".freeze
|
|
50
|
+
|
|
51
|
+
# KEYS = [processes, identity, work], ARGV = [info_json, beat, ttl_seconds, tid, work_json, ...]
|
|
52
|
+
OOM_BEAT = <<~LUA.freeze
|
|
53
|
+
#{Fetcher::OomDrain::HEADER}if redis.call('EXISTS', KEYS[2]) == 0 then
|
|
54
|
+
redis.call('HSET', KEYS[2], 'info', ARGV[1])
|
|
55
|
+
redis.call('SADD', KEYS[1], KEYS[2])
|
|
56
|
+
end
|
|
57
|
+
redis.call('HSET', KEYS[2], 'beat', ARGV[2])
|
|
58
|
+
redis.call('EXPIRE', KEYS[2], ARGV[3])
|
|
59
|
+
redis.call('UNLINK', KEYS[3])
|
|
60
|
+
for i = 4, #ARGV, 2 do
|
|
61
|
+
redis.call('HSET', KEYS[3], ARGV[i], ARGV[i + 1])
|
|
62
|
+
end
|
|
63
|
+
redis.call('EXPIRE', KEYS[3], ARGV[3])
|
|
64
|
+
return 1
|
|
65
|
+
LUA
|
|
47
66
|
FALLBACK_PAGE_SIZE = 4096
|
|
48
67
|
|
|
49
68
|
# statm counts pages, and a page is not 4 KB everywhere: arm64 kernels
|
|
@@ -131,11 +150,38 @@ module Wurk
|
|
|
131
150
|
t0 = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
|
|
132
151
|
redis(idempotent: true) do |conn|
|
|
133
152
|
Wurk::Lua::Loader.pipelined_eval(conn) { |pipe, eval_method| write_beat(pipe, work_snapshot, eval_method) }
|
|
153
|
+
rescue RedisClient::OutOfMemoryError => e
|
|
154
|
+
beat_at_maxmemory(conn, e, work_snapshot)
|
|
134
155
|
end
|
|
135
156
|
rtt = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond) - t0
|
|
136
157
|
[drain_signals, rtt]
|
|
137
158
|
end
|
|
138
159
|
|
|
160
|
+
# At maxmemory the beat's HSET and SADD are refused. A worker with every thread
|
|
161
|
+
# in a long job claims nothing, so nothing else proves it alive: its `beat`
|
|
162
|
+
# would stand still and Swarm::Liveness would TERM it, just when the fleet has
|
|
163
|
+
# to stay up. Its global-concurrency holds would lapse the same way, and the
|
|
164
|
+
# cap then over-admit. Shebang scripts, so by EVAL for the reason
|
|
165
|
+
# Fetcher::OomDrain gives. A server refusing one raises the original OOM, the
|
|
166
|
+
# failure without this.
|
|
167
|
+
def beat_at_maxmemory(conn, error, work_snapshot = {})
|
|
168
|
+
work = work_snapshot.flat_map { |tid, hash| [tid.to_s, Wurk.dump_json(hash)] }
|
|
169
|
+
conn.call('EVAL', OOM_BEAT, 3, Keys::PROCESSES, @identity, "#{@identity}:work",
|
|
170
|
+
Wurk.dump_json(info_hash), Time.now.to_f.to_s, TTL_SECONDS, *work)
|
|
171
|
+
refresh_holds_at_maxmemory(conn)
|
|
172
|
+
rescue RedisClient::ConnectionError
|
|
173
|
+
raise
|
|
174
|
+
rescue RedisClient::CommandError
|
|
175
|
+
raise error
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def refresh_holds_at_maxmemory(conn)
|
|
179
|
+
held = QueueSlot::HELD.snapshot
|
|
180
|
+
return if held.empty?
|
|
181
|
+
|
|
182
|
+
conn.call('EVAL', OOM_REFRESH_SLOTS, held.size, *held.values, QueueSlot::TTL_SECONDS, *held.keys)
|
|
183
|
+
end
|
|
184
|
+
|
|
139
185
|
def write_beat(pipe, work_snapshot, eval_method = :eval_cached)
|
|
140
186
|
pipe.call('SADD', Keys::PROCESSES, @identity)
|
|
141
187
|
pipe.call('HSET', @identity, *beat_hash_args(work_snapshot.size))
|
|
@@ -163,12 +209,18 @@ module Wurk
|
|
|
163
209
|
|
|
164
210
|
def write_work_hash(pipe, work_snapshot)
|
|
165
211
|
work_key = "#{@identity}:work"
|
|
166
|
-
pipe.call('UNLINK', work_key)
|
|
167
|
-
return if work_snapshot.empty?
|
|
212
|
+
return pipe.call('UNLINK', work_key) if work_snapshot.empty?
|
|
168
213
|
|
|
169
214
|
args = work_snapshot.flat_map { |tid, hash| [tid.to_s, Wurk.dump_json(hash)] }
|
|
215
|
+
# In a transaction because UNLINK is allowed at maxmemory and HSET is not: on
|
|
216
|
+
# its own that erased the mirror of jobs still running. On Redis and Valkey a
|
|
217
|
+
# refused queued command aborts the EXEC, so the old mirror stays. Dragonfly
|
|
218
|
+
# does not abort, so there the mirror is still erased, as it always was.
|
|
219
|
+
pipe.call('MULTI')
|
|
220
|
+
pipe.call('UNLINK', work_key)
|
|
170
221
|
pipe.call('HSET', work_key, *args)
|
|
171
222
|
pipe.call('EXPIRE', work_key, TTL_SECONDS)
|
|
223
|
+
pipe.call('EXEC')
|
|
172
224
|
end
|
|
173
225
|
|
|
174
226
|
# Its own checkout, and never an idempotent one: RPOP is destructive and
|
data/lib/wurk/job/options.rb
CHANGED
|
@@ -40,7 +40,7 @@ module Wurk
|
|
|
40
40
|
# table for the full set)
|
|
41
41
|
# @return [Hash] the merged, string-keyed options hash
|
|
42
42
|
def sidekiq_options(opts = {})
|
|
43
|
-
stringified = opts.transform_keys(&:to_s)
|
|
43
|
+
stringified = opts.to_h { |key, val| [key.to_s, val.is_a?(::Hash) ? val.transform_keys(&:to_s) : val] }
|
|
44
44
|
Wurk::JobUtil.validate_track!(stringified['track'], stringified) if stringified.key?('track')
|
|
45
45
|
Wurk::JobUtil.validate_bounds!(stringified)
|
|
46
46
|
merged = get_sidekiq_options.merge(stringified)
|
|
@@ -61,6 +61,8 @@ module Wurk
|
|
|
61
61
|
get_sidekiq_options
|
|
62
62
|
end
|
|
63
63
|
|
|
64
|
+
attr_writer :sidekiq_options_hash
|
|
65
|
+
|
|
64
66
|
attr_reader :sidekiq_retry_in_block, :sidekiq_retries_exhausted_block
|
|
65
67
|
|
|
66
68
|
def sidekiq_retry_in(&block)
|
|
@@ -96,4 +98,10 @@ module Wurk
|
|
|
96
98
|
end
|
|
97
99
|
end
|
|
98
100
|
end
|
|
101
|
+
|
|
102
|
+
# One module under both names, as upstream's Sidekiq::Worker and Sidekiq::Job
|
|
103
|
+
# are, so a prepend, `method_defined?` or constant lookup on either sees the
|
|
104
|
+
# job methods. Assigned here because worker.rb and worker/setter.rb reopen it
|
|
105
|
+
# as `module Worker`, and everything that loads them loads this file first.
|
|
106
|
+
Worker = Job
|
|
99
107
|
end
|
data/lib/wurk/job.rb
CHANGED
|
@@ -4,13 +4,8 @@ require_relative 'worker'
|
|
|
4
4
|
require_relative 'worker/setter'
|
|
5
5
|
|
|
6
6
|
module Wurk
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
# Instance-level `jid`, `_context`, `interrupted?`, and `logger` come
|
|
11
|
-
# from Wurk::Worker. Class-level DSL (`sidekiq_options`, `perform_*`,
|
|
12
|
-
# `set`, retry blocks) does too — Job is a pure alias module that
|
|
13
|
-
# re-exposes Worker under the modern name.
|
|
7
|
+
# The job mixin's control-flow errors. Wurk::Worker and Wurk::Job are one
|
|
8
|
+
# module (see job/options.rb), whose DSL and instance methods are in worker.rb.
|
|
14
9
|
module Job
|
|
15
10
|
# Raised mid-iteration when the run loop must yield — either a swarm
|
|
16
11
|
# shutdown signal or a cooperative cancellation. The interrupt-handler
|
|
@@ -39,22 +34,5 @@ module Wurk
|
|
|
39
34
|
# same terminal state as one whose deadline had already passed before it
|
|
40
35
|
# started. A job may still rescue it to clean up after itself.
|
|
41
36
|
class DeadlineExceeded < RuntimeError; end
|
|
42
|
-
|
|
43
|
-
# Per-call option carrier returned by `set(...)`. Sidekiq 7+ documents it
|
|
44
|
-
# under the modern mixin name `Sidekiq::Job::Setter`; since
|
|
45
|
-
# `Sidekiq::Job = Wurk::Job`, this rebind is what makes that constant
|
|
46
|
-
# resolve (without it `Sidekiq::Job::Setter` raises NameError). Same class
|
|
47
|
-
# as `Sidekiq::Worker::Setter`. Spec: docs/target/sidekiq-free.md §6.3.
|
|
48
|
-
Setter = Wurk::Worker::Setter
|
|
49
|
-
|
|
50
|
-
def self.included(base)
|
|
51
|
-
base.include(Wurk::Worker)
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
# Mirror the module-level test helpers so `Sidekiq::Job.jobs /
|
|
55
|
-
# clear_all / drain_all` work the same as `Sidekiq::Worker.*`.
|
|
56
|
-
def self.jobs = Wurk::Worker.jobs
|
|
57
|
-
def self.clear_all = Wurk::Worker.clear_all
|
|
58
|
-
def self.drain_all = Wurk::Worker.drain_all
|
|
59
37
|
end
|
|
60
38
|
end
|
data/lib/wurk/job_record.rb
CHANGED
|
@@ -41,16 +41,13 @@ module Wurk
|
|
|
41
41
|
|
|
42
42
|
def queue = @queue || item['queue']
|
|
43
43
|
|
|
44
|
-
# Lazily parsed payload. Memoized; never re-parses.
|
|
45
|
-
# an empty hash whose raw bytes become `args`, as
|
|
46
|
-
# corrupt payload can't break a
|
|
44
|
+
# Lazily parsed payload. Memoized; never re-parses. Anything that is not a
|
|
45
|
+
# JSON object reads as an empty hash whose raw bytes become `args`, as
|
|
46
|
+
# upstream does for invalid JSON, so one corrupt payload can't break a
|
|
47
|
+
# dashboard page or a Queue#each walk. Valid JSON counts: `[1,2]` and `123`
|
|
48
|
+
# would otherwise raise from every reader that indexes it by field name.
|
|
47
49
|
def item
|
|
48
|
-
@item ||=
|
|
49
|
-
Wurk.load_json(@value)
|
|
50
|
-
rescue ::JSON::ParserError
|
|
51
|
-
@invalid_args = [@value]
|
|
52
|
-
{}
|
|
53
|
-
end
|
|
50
|
+
@item ||= parse_payload
|
|
54
51
|
end
|
|
55
52
|
|
|
56
53
|
# Lazily serialized payload. When constructed from a Hash, we
|
|
@@ -155,6 +152,17 @@ module Wurk
|
|
|
155
152
|
|
|
156
153
|
private
|
|
157
154
|
|
|
155
|
+
def parse_payload
|
|
156
|
+
parsed = Wurk.load_json(@value)
|
|
157
|
+
return parsed if parsed.is_a?(::Hash)
|
|
158
|
+
|
|
159
|
+
@invalid_args = [@value]
|
|
160
|
+
{}
|
|
161
|
+
rescue ::JSON::ParserError, ::EncodingError
|
|
162
|
+
@invalid_args = [@value]
|
|
163
|
+
{}
|
|
164
|
+
end
|
|
165
|
+
|
|
158
166
|
def active_job_wrapper?
|
|
159
167
|
klass.is_a?(String) && (ACTIVE_JOB_WRAPPER.match?(klass) || klass == 'Sidekiq::ActiveJob::Wrapper')
|
|
160
168
|
end
|