wurk 1.9.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +9 -9
- data/app/controllers/wurk/api_controller.rb +4 -1
- data/app/controllers/wurk/dashboard_controller.rb +11 -6
- data/exe/wurk +7 -2
- data/exe/wurkswarm +1 -2
- data/lib/generators/wurk/install/install_generator.rb +12 -0
- data/lib/sidekiq/web.rb +5 -0
- data/lib/wurk/boot_checks.rb +93 -0
- data/lib/wurk/capsule.rb +33 -8
- data/lib/wurk/cli.rb +204 -30
- data/lib/wurk/client/buffered.rb +240 -26
- data/lib/wurk/client.rb +16 -5
- data/lib/wurk/compat.rb +15 -0
- data/lib/wurk/configuration.rb +114 -16
- data/lib/wurk/cron.rb +141 -16
- data/lib/wurk/debounce.rb +2 -2
- data/lib/wurk/embedded.rb +8 -2
- data/lib/wurk/encryption.rb +20 -5
- data/lib/wurk/error_coalescer.rb +35 -0
- data/lib/wurk/fetcher/at_maxmemory.rb +79 -0
- data/lib/wurk/fetcher/capped.rb +4 -0
- data/lib/wurk/fetcher/many_queues.rb +17 -1
- data/lib/wurk/fetcher/oom_drain.rb +102 -0
- data/lib/wurk/fetcher/orphan_grace.rb +60 -44
- data/lib/wurk/fetcher/private_list_key.rb +1 -16
- data/lib/wurk/fetcher/reaper.rb +169 -127
- data/lib/wurk/fetcher/reliable.rb +26 -4
- data/lib/wurk/fetcher/self_reconcile.rb +175 -0
- data/lib/wurk/fetcher/unit_of_work.rb +32 -8
- data/lib/wurk/health.rb +23 -4
- data/lib/wurk/health_probe.rb +64 -0
- data/lib/wurk/heartbeat.rb +55 -3
- data/lib/wurk/job/options.rb +9 -1
- data/lib/wurk/job.rb +2 -24
- data/lib/wurk/job_record.rb +17 -9
- data/lib/wurk/job_retry.rb +38 -10
- data/lib/wurk/job_set.rb +13 -5
- data/lib/wurk/launcher.rb +134 -52
- data/lib/wurk/leader.rb +71 -11
- data/lib/wurk/limiter/base.rb +17 -1
- data/lib/wurk/lua/debounce.lua +11 -6
- data/lib/wurk/lua/loader.rb +13 -1
- data/lib/wurk/lua.rb +71 -8
- data/lib/wurk/manager.rb +1 -1
- data/lib/wurk/metrics/process_counters.rb +27 -0
- data/lib/wurk/metrics/prometheus.rb +35 -14
- data/lib/wurk/middleware/current_attributes.rb +26 -10
- data/lib/wurk/middleware/poison_pill.rb +1 -2
- data/lib/wurk/preflight/checks.rb +203 -0
- data/lib/wurk/preflight/command.rb +135 -0
- data/lib/wurk/preflight/keyspace.rb +109 -0
- data/lib/wurk/preflight/owners.rb +74 -0
- data/lib/wurk/preflight/payload_sample.rb +108 -0
- data/lib/wurk/preflight/reader.rb +60 -0
- data/lib/wurk/preflight/report.rb +82 -0
- data/lib/wurk/preflight/verify.rb +161 -0
- data/lib/wurk/preflight.rb +40 -0
- data/lib/wurk/process_set.rb +3 -1
- data/lib/wurk/processor.rb +82 -18
- data/lib/wurk/queue_consumers.rb +134 -0
- data/lib/wurk/rails_boot.rb +40 -2
- data/lib/wurk/rails_reloader.rb +51 -0
- data/lib/wurk/railtie.rb +20 -1
- data/lib/wurk/redact.rb +12 -0
- data/lib/wurk/redis_pool.rb +34 -9
- data/lib/wurk/scheduled.rb +142 -67
- data/lib/wurk/sentry/death_handler.rb +29 -0
- data/lib/wurk/sentry/error_handler.rb +52 -10
- data/lib/wurk/sentry/middleware.rb +8 -1
- data/lib/wurk/sentry/retry_policy.rb +12 -3
- data/lib/wurk/sentry.rb +42 -5
- data/lib/wurk/sorted_entry.rb +38 -13
- data/lib/wurk/swarm/child_boot.rb +44 -31
- data/lib/wurk/swarm/crash_watch.rb +65 -0
- data/lib/wurk/swarm/test_mode_guard.rb +32 -0
- data/lib/wurk/swarm.rb +79 -11
- data/lib/wurk/testing.rb +74 -0
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/config.rb +8 -4
- data/lib/wurk/web/extension.rb +41 -62
- data/lib/wurk/web/helpers.rb +133 -0
- data/lib/wurk/web/paginator.rb +49 -0
- data/lib/wurk/web/rack_app.rb +30 -6
- data/lib/wurk/work_set.rb +6 -0
- data/lib/wurk/worker/setter.rb +11 -6
- data/lib/wurk/worker.rb +17 -1
- data/lib/wurk.rb +1 -1
- data/vendor/assets/dashboard/assets/{ArgsValue-DDqX6XWH.js → ArgsValue-CmaXArv8.js} +1 -1
- data/vendor/assets/dashboard/assets/{BatchDetail-lIytcn2C.js → BatchDetail-nc5yePZW.js} +1 -1
- data/vendor/assets/dashboard/assets/{Batches-BEumt-1O.js → Batches-DgEcyOZ_.js} +1 -1
- data/vendor/assets/dashboard/assets/{Busy-Vk9HmZcN.js → Busy-CUZncvCy.js} +1 -1
- data/vendor/assets/dashboard/assets/{Cron-CWcM-Cfr.js → Cron-CIZbaaZs.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dashboard-Bgv1kusv.js → Dashboard-BWmfAook.js} +1 -1
- data/vendor/assets/dashboard/assets/{Dead-Di5Rct2H.js → Dead-CGKsTZHq.js} +1 -1
- data/vendor/assets/dashboard/assets/{Extension-LZY5-imc.js → Extension-Doy4bJoA.js} +1 -1
- data/vendor/assets/dashboard/assets/{FilterBox-BDo-7IAy.js → FilterBox-C4bO26_d.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowDetail-CehwRLTt.js → FlowDetail-Dh3SJdFh.js} +1 -1
- data/vendor/assets/dashboard/assets/{FlowState-glK2Tnf8.js → FlowState-yb7virqR.js} +1 -1
- data/vendor/assets/dashboard/assets/{Flows-cgy-Fsbp.js → Flows-C5SMRuhq.js} +1 -1
- data/vendor/assets/dashboard/assets/{JobDetailModal-Cuk3jruU.js → JobDetailModal-Cg6sAsJB.js} +1 -1
- data/vendor/assets/dashboard/assets/{Limiters-Ci8G2yZG.js → Limiters-BIxwDOJM.js} +1 -1
- data/vendor/assets/dashboard/assets/{Metrics-CQFUeZTR.js → Metrics-Qp9q0nF8.js} +1 -1
- data/vendor/assets/dashboard/assets/{NotFound-CYjhuDsI.js → NotFound-DwjPxFjU.js} +1 -1
- data/vendor/assets/dashboard/assets/{PageHeader-DjIAFTyX.js → PageHeader-vWeBbSPF.js} +1 -1
- data/vendor/assets/dashboard/assets/{Profiles-DTowqIRg.js → Profiles-7dabYVe3.js} +1 -1
- data/vendor/assets/dashboard/assets/Queues-CMg-gyP8.js +1 -0
- data/vendor/assets/dashboard/assets/{Retries-SJ3BaKC0.js → Retries-CNX8ZtTH.js} +1 -1
- data/vendor/assets/dashboard/assets/{Scheduled-DxSRpmFZ.js → Scheduled-ClVBj0nc.js} +1 -1
- data/vendor/assets/dashboard/assets/{Search-bBbi5JNK.js → Search-BAsVE2qw.js} +1 -1
- data/vendor/assets/dashboard/assets/{charts-Bg2YBcaD.js → charts-DZs138BV.js} +1 -1
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +1 -0
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +1 -0
- data/vendor/assets/dashboard/assets/{index-BOTVSP9M.js → index-CMzPFxNE.js} +3 -3
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-BRPNyV2M.js → useResetPageOnEmpty-DxkLZCIm.js} +1 -1
- data/vendor/assets/dashboard/assets/{useSort-B-BLJik7.js → useSort-jBYVwXvg.js} +1 -1
- data/vendor/assets/dashboard/assets/{utils-CGyrKpTh.js → utils-B0kQFNE8.js} +1 -1
- data/vendor/assets/dashboard/index.html +3 -3
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +53 -30
- data/vendor/assets/dashboard/assets/Queues-DUGRPjME.js +0 -1
- data/vendor/assets/dashboard/assets/i18n-BZClP62Q.js +0 -1
- data/vendor/assets/dashboard/assets/index-Rewk4eSE.css +0 -1
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../keys'
|
|
4
|
+
require_relative '../lua'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
class Fetcher
|
|
8
|
+
# A live worker's check of its own private lists.
|
|
9
|
+
#
|
|
10
|
+
# The Reaper leaves a live owner's list alone, and a live owner only ever
|
|
11
|
+
# settles the jobs its threads know about. So a payload no thread knows about
|
|
12
|
+
# would wait there until the process exits. Two ways in:
|
|
13
|
+
# * a claim applied but its reply was lost (Redis restart, a stall past the
|
|
14
|
+
# read timeout, a dropped socket): the client re-sends and claims another
|
|
15
|
+
# job, so one job per thread per incident;
|
|
16
|
+
# * a job that failed while Redis was down, where the retry booking and the
|
|
17
|
+
# requeue both raised.
|
|
18
|
+
#
|
|
19
|
+
# A payload counts as stranded only if it is in the list and held by nothing:
|
|
20
|
+
# no thread's claim (Fetcher::Reliable::CLAIM_KEY, which covers the whole of
|
|
21
|
+
# a running job however long it runs) and no pending ACK. The list is read
|
|
22
|
+
# first and the holders second, so a claim landing between the two reads as
|
|
23
|
+
# held. What the reads cannot see is a unit in transit: claimed but not yet
|
|
24
|
+
# stamped, or an ACK on the wire. Those last microseconds to seconds, which
|
|
25
|
+
# is why one sighting proves nothing: it has to be seen stranded again at
|
|
26
|
+
# least `confirm` seconds later before it moves.
|
|
27
|
+
#
|
|
28
|
+
# Counted rather than compared as sets, because a client re-send can leave two
|
|
29
|
+
# byte-identical payloads, and a running job must not vouch for its twin.
|
|
30
|
+
class SelfReconcile
|
|
31
|
+
# Above anything a unit can spend in transit: the longest a held ACK sits on
|
|
32
|
+
# the wire is the pool's blip backoff plus its socket timeouts, ~18s.
|
|
33
|
+
DEFAULT_CONFIRM = 30
|
|
34
|
+
|
|
35
|
+
Suspect = Struct.new(:copies, :since)
|
|
36
|
+
|
|
37
|
+
# `on_moved` receives [public_queue, payload] for each payload put back, so
|
|
38
|
+
# the Reaper can run it through the same poison accounting as a reclaim.
|
|
39
|
+
def initialize(config, confirm: DEFAULT_CONFIRM, stop: -> { false }, on_moved: nil)
|
|
40
|
+
@config = config
|
|
41
|
+
@confirm = confirm
|
|
42
|
+
@stop = stop
|
|
43
|
+
@on_moved = on_moved
|
|
44
|
+
@suspects = {}
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# One pass. Returns how many payloads went back to their public queues.
|
|
48
|
+
def sweep
|
|
49
|
+
lists = own_lists
|
|
50
|
+
return 0 if lists.empty?
|
|
51
|
+
|
|
52
|
+
crowded = crowded_lists(lists)
|
|
53
|
+
return forget_all if crowded.empty?
|
|
54
|
+
|
|
55
|
+
contents = read(crowded.keys)
|
|
56
|
+
settle(crowded, contents, held_by_list)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
# { private_list => public_queue } for every queue a Reliable fetcher here
|
|
62
|
+
# serves. Two capsules on one queue share a list, hence the Hash. A custom
|
|
63
|
+
# `config[:fetch_class]` keeps its own books, so its queues are left alone.
|
|
64
|
+
def own_lists
|
|
65
|
+
@config.capsules.each_value.with_object({}) do |capsule, lists|
|
|
66
|
+
next unless capsule.fetcher.is_a?(Reliable)
|
|
67
|
+
|
|
68
|
+
capsule.queues.uniq.each do |name|
|
|
69
|
+
lists[Reliable.private_queue_name(Keys.queue(name))] = Keys.queue(name)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# A list no longer than what its holders account for has nothing stranded
|
|
75
|
+
# in it, which is every list on every healthy sweep, and LLEN is what keeps
|
|
76
|
+
# a healthy sweep from reading every in-flight payload back over the wire.
|
|
77
|
+
def crowded_lists(lists)
|
|
78
|
+
lengths = read_lengths(lists.keys)
|
|
79
|
+
held = held_by_list
|
|
80
|
+
lists.select { |list, _| lengths[list] > held_count(held, list) }
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def read_lengths(names)
|
|
84
|
+
replies = @config.redis(idempotent: true) do |conn|
|
|
85
|
+
conn.pipelined { |pipe| names.each { |list| pipe.call('LLEN', list) } }
|
|
86
|
+
end
|
|
87
|
+
names.zip(replies).to_h
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def held_count(held, list)
|
|
91
|
+
held.fetch(list, {}).each_value.sum
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def read(names)
|
|
95
|
+
replies = @config.redis(idempotent: true) do |conn|
|
|
96
|
+
conn.pipelined { |pipe| names.each { |list| pipe.call('LRANGE', list, 0, -1) } }
|
|
97
|
+
end
|
|
98
|
+
names.zip(replies).to_h
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# { private_list => { payload => count } } across every thread and fetcher
|
|
102
|
+
# in this process. Thread.list, not a registry: nothing is written per job,
|
|
103
|
+
# and a thread that died mid-job stops vouching for its payload by itself.
|
|
104
|
+
def held_by_list
|
|
105
|
+
held_units.each_with_object({}) do |unit, held|
|
|
106
|
+
tally = (held[unit.private_queue] ||= Hash.new(0))
|
|
107
|
+
tally[unit.job.b] += 1
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def held_units
|
|
112
|
+
claims = ::Thread.list.filter_map { |thread| thread.thread_variable_get(Reliable::CLAIM_KEY) }
|
|
113
|
+
fetchers = @config.capsules.each_value.map(&:fetcher).grep(Reliable)
|
|
114
|
+
claims + fetchers.flat_map(&:held_acks)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def settle(crowded, contents, held)
|
|
118
|
+
now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
119
|
+
following = {}
|
|
120
|
+
moved = contents.sum do |list, payloads|
|
|
121
|
+
stranded(payloads, held[list]).sum do |payload, count|
|
|
122
|
+
judge(list, crowded[list], payload, count, now, following)
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
@suspects = following
|
|
126
|
+
moved
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Moves what has been seen stranded for long enough, remembers the rest.
|
|
130
|
+
def judge(list, public_q, payload, count, now, following) # rubocop:disable Metrics/ParameterLists
|
|
131
|
+
suspect = @suspects.dig(list, payload)
|
|
132
|
+
confirmed = suspect && now - suspect.since >= @confirm
|
|
133
|
+
return put_back(list, public_q, payload, [count, suspect.copies].min) if confirmed
|
|
134
|
+
|
|
135
|
+
(following[list] ||= {})[payload] = Suspect.new(count, suspect&.since || now)
|
|
136
|
+
0
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def stranded(payloads, held)
|
|
140
|
+
held ||= {}
|
|
141
|
+
payloads.map(&:b).tally.filter_map do |payload, count|
|
|
142
|
+
excess = count - held.fetch(payload, 0)
|
|
143
|
+
[payload, excess] if excess.positive?
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def put_back(list, public_q, payload, count)
|
|
148
|
+
moved = 0
|
|
149
|
+
count.times do
|
|
150
|
+
break if @stop.call
|
|
151
|
+
|
|
152
|
+
moved += requeue(list, public_q, payload)
|
|
153
|
+
end
|
|
154
|
+
moved
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# LPOS-guarded, so a replay after a lost reply moves nothing the second
|
|
158
|
+
# time, and a payload settled since the last read stays settled.
|
|
159
|
+
def requeue(list, public_q, payload)
|
|
160
|
+
applied = @config.redis(idempotent: true) do |conn|
|
|
161
|
+
Wurk::Lua::Loader.eval_cached(conn, :reliable_requeue, keys: [list, public_q], argv: [payload])
|
|
162
|
+
rescue RedisClient::OutOfMemoryError => e
|
|
163
|
+
OomDrain.requeue(conn, e, list, public_q, payload)
|
|
164
|
+
end.to_i
|
|
165
|
+
@on_moved&.call(public_q, payload) if applied == 1
|
|
166
|
+
applied
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def forget_all
|
|
170
|
+
@suspects = {}
|
|
171
|
+
0
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
end
|
|
@@ -41,6 +41,19 @@ module Wurk
|
|
|
41
41
|
|
|
42
42
|
UnitOfWork = Struct.new(:queue, :queue_name, :private_queue, :job, :config, :jid, :fetcher,
|
|
43
43
|
:slot_key, :slot_token, keyword_init: true) do
|
|
44
|
+
# The positional form is upstream's documented `UnitOfWork.new(queue,
|
|
45
|
+
# job, config)`, which gem test suites build directly. No fetcher means
|
|
46
|
+
# no private list, so #acknowledge has nothing to do. Anonymous `**` is
|
|
47
|
+
# the cheap spelling for the fetch path, which always passes keywords:
|
|
48
|
+
# one allocation and ~230ns per job (a named **fields costs two).
|
|
49
|
+
def initialize(queue = nil, job = nil, config = nil, **)
|
|
50
|
+
if queue
|
|
51
|
+
super(queue: queue, queue_name: queue.delete_prefix('queue:'), job: job, config: config, **)
|
|
52
|
+
else
|
|
53
|
+
super(**)
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
44
57
|
# Deferred, never skipped: the LREM goes back to the fetcher, which
|
|
45
58
|
# pipelines it in front of the next fetch's LMOVE instead of spending a
|
|
46
59
|
# round trip of its own. Ordering against the job is unchanged — the
|
|
@@ -48,7 +61,7 @@ module Wurk
|
|
|
48
61
|
# so all that moves is the wall clock. Every path that stops fetching
|
|
49
62
|
# flushes first; see #flush_pending_acks.
|
|
50
63
|
def acknowledge
|
|
51
|
-
fetcher
|
|
64
|
+
fetcher&.defer_ack(self)
|
|
52
65
|
end
|
|
53
66
|
|
|
54
67
|
# Queue this unit's ACK into an already-open pipeline. The counter DEL
|
|
@@ -107,18 +120,29 @@ module Wurk
|
|
|
107
120
|
# both lists and runs twice — once from the public queue, once when the
|
|
108
121
|
# reaper reclaims this process's private list.
|
|
109
122
|
#
|
|
110
|
-
#
|
|
123
|
+
# RPUSH then LREM in one script, unconditionally: unlike bulk_requeue
|
|
111
124
|
# (which races a Processor ACKing the same UoW and so guards the push on
|
|
112
|
-
# the
|
|
113
|
-
# went out must still be re-queued rather than dropped.
|
|
125
|
+
# the private copy), the caller here owns the unit, and a job whose ACK
|
|
126
|
+
# already went out must still be re-queued rather than dropped. A script
|
|
127
|
+
# because a MULTI would apply the LREM after a refused RPUSH, losing the
|
|
128
|
+
# job. Unconditional is also why it goes by call_once: redis-client
|
|
129
|
+
# re-sends on a dropped socket, and the replay would push the job a second
|
|
130
|
+
# time. A raise leaves the outcome unknown, and both outcomes are safe:
|
|
131
|
+
# moved, or still in the private list, where Fetcher::SelfReconcile finds
|
|
132
|
+
# it.
|
|
114
133
|
def requeue
|
|
115
134
|
config.redis do |conn|
|
|
116
|
-
conn.
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
end
|
|
135
|
+
next conn.call_once('RPUSH', queue, job) unless private_queue
|
|
136
|
+
|
|
137
|
+
Wurk::Lua::Loader.eval_once(conn, :requeue_unit, keys: [private_queue, queue], argv: [job])
|
|
120
138
|
end
|
|
121
139
|
end
|
|
140
|
+
|
|
141
|
+
# The Processor has settled this unit, one way or the other: stop
|
|
142
|
+
# vouching for its payload. See Fetcher::SelfReconcile.
|
|
143
|
+
def release_claim
|
|
144
|
+
::Thread.current.thread_variable_set(CLAIM_KEY, nil)
|
|
145
|
+
end
|
|
122
146
|
end
|
|
123
147
|
end
|
|
124
148
|
end
|
data/lib/wurk/health.rb
CHANGED
|
@@ -6,14 +6,14 @@ require_relative 'metrics/prometheus'
|
|
|
6
6
|
|
|
7
7
|
module Wurk
|
|
8
8
|
# Thin HTTP listener for k8s liveness/readiness probes. Optional, off by
|
|
9
|
-
# default — opt in with `config.health_check(port: 7433)
|
|
10
|
-
# GET /metrics on that
|
|
9
|
+
# default — opt in with `config.health_check(port: 7433)` or
|
|
10
|
+
# WURK_HEALTH_PORT=7433, then scrape GET /metrics on that port.
|
|
11
11
|
# Mounting Wurk::Engine does not expose Prometheus metrics: this deliberate
|
|
12
12
|
# separation keeps internal metrics off the host application's HTTP surface
|
|
13
13
|
# so access can be restricted independently on the dedicated listener port.
|
|
14
14
|
#
|
|
15
15
|
# Endpoints:
|
|
16
|
-
# * GET /live → 200
|
|
16
|
+
# * GET /live → 200 until the Launcher is shutting down; quiet stays 200.
|
|
17
17
|
# * GET /ready → 200 only when Redis is reachable AND the heartbeat has
|
|
18
18
|
# fired within `ready_window` seconds AND, inside a swarm
|
|
19
19
|
# child, enough of the swarm's children have (see
|
|
@@ -43,6 +43,22 @@ module Wurk
|
|
|
43
43
|
# fleet of one and /ready judges them on their own heartbeat alone.
|
|
44
44
|
attr_accessor :fleet_size
|
|
45
45
|
|
|
46
|
+
# WURK_HEALTH_PORT turns the listener on from the environment, so whoever
|
|
47
|
+
# owns the deployment manifest can add a probe without an application
|
|
48
|
+
# change. `config.health_check` wins when both are set. A bad value raises:
|
|
49
|
+
# a probe silently not listening would restart the pod forever instead.
|
|
50
|
+
def options_from_env(env = ENV)
|
|
51
|
+
raw = env['WURK_HEALTH_PORT'].to_s.strip
|
|
52
|
+
return nil if raw.empty?
|
|
53
|
+
|
|
54
|
+
port = Integer(raw, 10)
|
|
55
|
+
raise ArgumentError unless port.between?(0, 65_535)
|
|
56
|
+
|
|
57
|
+
{ port: port }
|
|
58
|
+
rescue ArgumentError
|
|
59
|
+
raise ArgumentError, "WURK_HEALTH_PORT must be a port number (got #{raw.inspect})"
|
|
60
|
+
end
|
|
61
|
+
|
|
46
62
|
def json(status, **extra)
|
|
47
63
|
::JSON.generate({ status: status }.merge(extra))
|
|
48
64
|
end
|
|
@@ -278,8 +294,11 @@ module Wurk
|
|
|
278
294
|
@fleet ||= Fleet.new(@config, @ready_window)
|
|
279
295
|
end
|
|
280
296
|
|
|
297
|
+
# A quiet process is alive and is finishing its jobs. Failing here would have
|
|
298
|
+
# the orchestrator restart a pod an operator just quieted, and the
|
|
299
|
+
# replacement would fetch again. /ready is the probe that speaks to traffic.
|
|
281
300
|
def live_response
|
|
282
|
-
if @launcher.
|
|
301
|
+
if @launcher.shutting_down?
|
|
283
302
|
[json('down', check: 'live', reason: 'stopping'), 503]
|
|
284
303
|
else
|
|
285
304
|
[json('ok', check: 'live'), 200]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'component'
|
|
4
|
+
require_relative 'keys'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
# `wurk health`: an exec-style liveness probe, for a pod with no HTTP port. It
|
|
8
|
+
# boots no application, so it is fast and only needs to reach Redis (REDIS_URL
|
|
9
|
+
# or --redis-url). Passes when enough processes on this host have a recent
|
|
10
|
+
# heartbeat.
|
|
11
|
+
#
|
|
12
|
+
# A Redis it cannot reach is not this pod's fault, and restarting workers does
|
|
13
|
+
# not bring Redis back, so that passes too unless `--strict`.
|
|
14
|
+
class HealthProbe
|
|
15
|
+
HEALTHY = 0
|
|
16
|
+
UNHEALTHY = 1
|
|
17
|
+
DEFAULT_WINDOW = 60
|
|
18
|
+
|
|
19
|
+
# At most connect + two round trips: about 2s in all against a hung server.
|
|
20
|
+
CLIENT_LIMITS = { connect_timeout: 0.5, read_timeout: 0.6, write_timeout: 0.6, reconnect_attempts: 0 }.freeze
|
|
21
|
+
|
|
22
|
+
def initialize(config, window: DEFAULT_WINDOW, min: 1, strict: false, hostname: Component.hostname, out: $stdout)
|
|
23
|
+
@config = config
|
|
24
|
+
@window = window
|
|
25
|
+
@min = min
|
|
26
|
+
@strict = strict
|
|
27
|
+
@prefix = "#{hostname}:"
|
|
28
|
+
@out = out
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def call
|
|
32
|
+
fresh = fresh_processes
|
|
33
|
+
@out.puts "wurk health: #{fresh} process(es) on this host beat within #{@window}s (need #{@min})"
|
|
34
|
+
fresh >= @min ? HEALTHY : UNHEALTHY
|
|
35
|
+
rescue RedisClient::Error, ConnectionPool::TimeoutError => e
|
|
36
|
+
@out.puts "wurk health: cannot read Redis (#{e.class}: #{e.message})"
|
|
37
|
+
@strict ? UNHEALTHY : HEALTHY
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def fresh_processes
|
|
43
|
+
now = ::Time.now.to_f
|
|
44
|
+
read_local_beats.count { |beat| beat && now - beat.to_f < @window }
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def read_local_beats
|
|
48
|
+
client = build_client
|
|
49
|
+
ids = client.call('SMEMBERS', Keys::PROCESSES).select { |id| id.start_with?(@prefix) }
|
|
50
|
+
ids.empty? ? [] : client.pipelined { |pipe| ids.each { |id| pipe.call('HGET', id, 'beat') } }
|
|
51
|
+
ensure
|
|
52
|
+
client&.close
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Directly on redis-client rather than through RedisPool: the pool's retries
|
|
56
|
+
# and backoff are right for a worker and fatal here. They took 4s on a refused
|
|
57
|
+
# connection and 19s on a hung server, past a kubelet's probe timeout, so a
|
|
58
|
+
# store outage restarted every pod.
|
|
59
|
+
def build_client
|
|
60
|
+
opts = RedisOptions.normalize(@config.redis_config).except(*RedisOptions::POOL_KEYS).merge(CLIENT_LIMITS)
|
|
61
|
+
(RedisOptions.sentinel?(opts) ? RedisClient.sentinel(**opts) : RedisClient.config(**opts)).new_client
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
data/lib/wurk/heartbeat.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'etc'
|
|
4
4
|
|
|
5
5
|
require_relative 'component'
|
|
6
|
+
require_relative 'fetcher/oom_drain'
|
|
6
7
|
require_relative 'keys'
|
|
7
8
|
require_relative 'lua'
|
|
8
9
|
require_relative 'processor'
|
|
@@ -38,12 +39,30 @@ module Wurk
|
|
|
38
39
|
# network partition (rearmed when beat! rescues). `:beat` fires every tick.
|
|
39
40
|
#
|
|
40
41
|
# Spec: docs/target/sidekiq-free.md §12 (Launcher#❤️).
|
|
41
|
-
class Heartbeat
|
|
42
|
+
class Heartbeat # rubocop:disable Metrics/ClassLength
|
|
42
43
|
include Component
|
|
43
44
|
|
|
44
45
|
# Cadence in seconds. Key TTL is 60s — a process is dead after ~6 misses.
|
|
45
46
|
BEAT_PAUSE = 10
|
|
46
47
|
TTL_SECONDS = 60
|
|
48
|
+
|
|
49
|
+
OOM_REFRESH_SLOTS = "#{Fetcher::OomDrain::HEADER}#{Lua::SCRIPTS.fetch(:refresh_slots)}".freeze
|
|
50
|
+
|
|
51
|
+
# KEYS = [processes, identity, work], ARGV = [info_json, beat, ttl_seconds, tid, work_json, ...]
|
|
52
|
+
OOM_BEAT = <<~LUA.freeze
|
|
53
|
+
#{Fetcher::OomDrain::HEADER}if redis.call('EXISTS', KEYS[2]) == 0 then
|
|
54
|
+
redis.call('HSET', KEYS[2], 'info', ARGV[1])
|
|
55
|
+
redis.call('SADD', KEYS[1], KEYS[2])
|
|
56
|
+
end
|
|
57
|
+
redis.call('HSET', KEYS[2], 'beat', ARGV[2])
|
|
58
|
+
redis.call('EXPIRE', KEYS[2], ARGV[3])
|
|
59
|
+
redis.call('UNLINK', KEYS[3])
|
|
60
|
+
for i = 4, #ARGV, 2 do
|
|
61
|
+
redis.call('HSET', KEYS[3], ARGV[i], ARGV[i + 1])
|
|
62
|
+
end
|
|
63
|
+
redis.call('EXPIRE', KEYS[3], ARGV[3])
|
|
64
|
+
return 1
|
|
65
|
+
LUA
|
|
47
66
|
FALLBACK_PAGE_SIZE = 4096
|
|
48
67
|
|
|
49
68
|
# statm counts pages, and a page is not 4 KB everywhere: arm64 kernels
|
|
@@ -131,11 +150,38 @@ module Wurk
|
|
|
131
150
|
t0 = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
|
|
132
151
|
redis(idempotent: true) do |conn|
|
|
133
152
|
Wurk::Lua::Loader.pipelined_eval(conn) { |pipe, eval_method| write_beat(pipe, work_snapshot, eval_method) }
|
|
153
|
+
rescue RedisClient::OutOfMemoryError => e
|
|
154
|
+
beat_at_maxmemory(conn, e, work_snapshot)
|
|
134
155
|
end
|
|
135
156
|
rtt = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond) - t0
|
|
136
157
|
[drain_signals, rtt]
|
|
137
158
|
end
|
|
138
159
|
|
|
160
|
+
# At maxmemory the beat's HSET and SADD are refused. A worker with every thread
|
|
161
|
+
# in a long job claims nothing, so nothing else proves it alive: its `beat`
|
|
162
|
+
# would stand still and Swarm::Liveness would TERM it, just when the fleet has
|
|
163
|
+
# to stay up. Its global-concurrency holds would lapse the same way, and the
|
|
164
|
+
# cap then over-admit. Shebang scripts, so by EVAL for the reason
|
|
165
|
+
# Fetcher::OomDrain gives. A server refusing one raises the original OOM, the
|
|
166
|
+
# failure without this.
|
|
167
|
+
def beat_at_maxmemory(conn, error, work_snapshot = {})
|
|
168
|
+
work = work_snapshot.flat_map { |tid, hash| [tid.to_s, Wurk.dump_json(hash)] }
|
|
169
|
+
conn.call('EVAL', OOM_BEAT, 3, Keys::PROCESSES, @identity, "#{@identity}:work",
|
|
170
|
+
Wurk.dump_json(info_hash), Time.now.to_f.to_s, TTL_SECONDS, *work)
|
|
171
|
+
refresh_holds_at_maxmemory(conn)
|
|
172
|
+
rescue RedisClient::ConnectionError
|
|
173
|
+
raise
|
|
174
|
+
rescue RedisClient::CommandError
|
|
175
|
+
raise error
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def refresh_holds_at_maxmemory(conn)
|
|
179
|
+
held = QueueSlot::HELD.snapshot
|
|
180
|
+
return if held.empty?
|
|
181
|
+
|
|
182
|
+
conn.call('EVAL', OOM_REFRESH_SLOTS, held.size, *held.values, QueueSlot::TTL_SECONDS, *held.keys)
|
|
183
|
+
end
|
|
184
|
+
|
|
139
185
|
def write_beat(pipe, work_snapshot, eval_method = :eval_cached)
|
|
140
186
|
pipe.call('SADD', Keys::PROCESSES, @identity)
|
|
141
187
|
pipe.call('HSET', @identity, *beat_hash_args(work_snapshot.size))
|
|
@@ -163,12 +209,18 @@ module Wurk
|
|
|
163
209
|
|
|
164
210
|
def write_work_hash(pipe, work_snapshot)
|
|
165
211
|
work_key = "#{@identity}:work"
|
|
166
|
-
pipe.call('UNLINK', work_key)
|
|
167
|
-
return if work_snapshot.empty?
|
|
212
|
+
return pipe.call('UNLINK', work_key) if work_snapshot.empty?
|
|
168
213
|
|
|
169
214
|
args = work_snapshot.flat_map { |tid, hash| [tid.to_s, Wurk.dump_json(hash)] }
|
|
215
|
+
# In a transaction because UNLINK is allowed at maxmemory and HSET is not: on
|
|
216
|
+
# its own that erased the mirror of jobs still running. On Redis and Valkey a
|
|
217
|
+
# refused queued command aborts the EXEC, so the old mirror stays. Dragonfly
|
|
218
|
+
# does not abort, so there the mirror is still erased, as it always was.
|
|
219
|
+
pipe.call('MULTI')
|
|
220
|
+
pipe.call('UNLINK', work_key)
|
|
170
221
|
pipe.call('HSET', work_key, *args)
|
|
171
222
|
pipe.call('EXPIRE', work_key, TTL_SECONDS)
|
|
223
|
+
pipe.call('EXEC')
|
|
172
224
|
end
|
|
173
225
|
|
|
174
226
|
# Its own checkout, and never an idempotent one: RPOP is destructive and
|
data/lib/wurk/job/options.rb
CHANGED
|
@@ -40,7 +40,7 @@ module Wurk
|
|
|
40
40
|
# table for the full set)
|
|
41
41
|
# @return [Hash] the merged, string-keyed options hash
|
|
42
42
|
def sidekiq_options(opts = {})
|
|
43
|
-
stringified = opts.transform_keys(&:to_s)
|
|
43
|
+
stringified = opts.to_h { |key, val| [key.to_s, val.is_a?(::Hash) ? val.transform_keys(&:to_s) : val] }
|
|
44
44
|
Wurk::JobUtil.validate_track!(stringified['track'], stringified) if stringified.key?('track')
|
|
45
45
|
Wurk::JobUtil.validate_bounds!(stringified)
|
|
46
46
|
merged = get_sidekiq_options.merge(stringified)
|
|
@@ -61,6 +61,8 @@ module Wurk
|
|
|
61
61
|
get_sidekiq_options
|
|
62
62
|
end
|
|
63
63
|
|
|
64
|
+
attr_writer :sidekiq_options_hash
|
|
65
|
+
|
|
64
66
|
attr_reader :sidekiq_retry_in_block, :sidekiq_retries_exhausted_block
|
|
65
67
|
|
|
66
68
|
def sidekiq_retry_in(&block)
|
|
@@ -96,4 +98,10 @@ module Wurk
|
|
|
96
98
|
end
|
|
97
99
|
end
|
|
98
100
|
end
|
|
101
|
+
|
|
102
|
+
# One module under both names, as upstream's Sidekiq::Worker and Sidekiq::Job
|
|
103
|
+
# are, so a prepend, `method_defined?` or constant lookup on either sees the
|
|
104
|
+
# job methods. Assigned here because worker.rb and worker/setter.rb reopen it
|
|
105
|
+
# as `module Worker`, and everything that loads them loads this file first.
|
|
106
|
+
Worker = Job
|
|
99
107
|
end
|
data/lib/wurk/job.rb
CHANGED
|
@@ -4,13 +4,8 @@ require_relative 'worker'
|
|
|
4
4
|
require_relative 'worker/setter'
|
|
5
5
|
|
|
6
6
|
module Wurk
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
# Instance-level `jid`, `_context`, `interrupted?`, and `logger` come
|
|
11
|
-
# from Wurk::Worker. Class-level DSL (`sidekiq_options`, `perform_*`,
|
|
12
|
-
# `set`, retry blocks) does too — Job is a pure alias module that
|
|
13
|
-
# re-exposes Worker under the modern name.
|
|
7
|
+
# The job mixin's control-flow errors. Wurk::Worker and Wurk::Job are one
|
|
8
|
+
# module (see job/options.rb), whose DSL and instance methods are in worker.rb.
|
|
14
9
|
module Job
|
|
15
10
|
# Raised mid-iteration when the run loop must yield — either a swarm
|
|
16
11
|
# shutdown signal or a cooperative cancellation. The interrupt-handler
|
|
@@ -39,22 +34,5 @@ module Wurk
|
|
|
39
34
|
# same terminal state as one whose deadline had already passed before it
|
|
40
35
|
# started. A job may still rescue it to clean up after itself.
|
|
41
36
|
class DeadlineExceeded < RuntimeError; end
|
|
42
|
-
|
|
43
|
-
# Per-call option carrier returned by `set(...)`. Sidekiq 7+ documents it
|
|
44
|
-
# under the modern mixin name `Sidekiq::Job::Setter`; since
|
|
45
|
-
# `Sidekiq::Job = Wurk::Job`, this rebind is what makes that constant
|
|
46
|
-
# resolve (without it `Sidekiq::Job::Setter` raises NameError). Same class
|
|
47
|
-
# as `Sidekiq::Worker::Setter`. Spec: docs/target/sidekiq-free.md §6.3.
|
|
48
|
-
Setter = Wurk::Worker::Setter
|
|
49
|
-
|
|
50
|
-
def self.included(base)
|
|
51
|
-
base.include(Wurk::Worker)
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
# Mirror the module-level test helpers so `Sidekiq::Job.jobs /
|
|
55
|
-
# clear_all / drain_all` work the same as `Sidekiq::Worker.*`.
|
|
56
|
-
def self.jobs = Wurk::Worker.jobs
|
|
57
|
-
def self.clear_all = Wurk::Worker.clear_all
|
|
58
|
-
def self.drain_all = Wurk::Worker.drain_all
|
|
59
37
|
end
|
|
60
38
|
end
|
data/lib/wurk/job_record.rb
CHANGED
|
@@ -41,16 +41,13 @@ module Wurk
|
|
|
41
41
|
|
|
42
42
|
def queue = @queue || item['queue']
|
|
43
43
|
|
|
44
|
-
# Lazily parsed payload. Memoized; never re-parses.
|
|
45
|
-
# an empty hash whose raw bytes become `args`, as
|
|
46
|
-
# corrupt payload can't break a
|
|
44
|
+
# Lazily parsed payload. Memoized; never re-parses. Anything that is not a
|
|
45
|
+
# JSON object reads as an empty hash whose raw bytes become `args`, as
|
|
46
|
+
# upstream does for invalid JSON, so one corrupt payload can't break a
|
|
47
|
+
# dashboard page or a Queue#each walk. Valid JSON counts: `[1,2]` and `123`
|
|
48
|
+
# would otherwise raise from every reader that indexes it by field name.
|
|
47
49
|
def item
|
|
48
|
-
@item ||=
|
|
49
|
-
Wurk.load_json(@value)
|
|
50
|
-
rescue ::JSON::ParserError
|
|
51
|
-
@invalid_args = [@value]
|
|
52
|
-
{}
|
|
53
|
-
end
|
|
50
|
+
@item ||= parse_payload
|
|
54
51
|
end
|
|
55
52
|
|
|
56
53
|
# Lazily serialized payload. When constructed from a Hash, we
|
|
@@ -155,6 +152,17 @@ module Wurk
|
|
|
155
152
|
|
|
156
153
|
private
|
|
157
154
|
|
|
155
|
+
def parse_payload
|
|
156
|
+
parsed = Wurk.load_json(@value)
|
|
157
|
+
return parsed if parsed.is_a?(::Hash)
|
|
158
|
+
|
|
159
|
+
@invalid_args = [@value]
|
|
160
|
+
{}
|
|
161
|
+
rescue ::JSON::ParserError, ::EncodingError
|
|
162
|
+
@invalid_args = [@value]
|
|
163
|
+
{}
|
|
164
|
+
end
|
|
165
|
+
|
|
158
166
|
def active_job_wrapper?
|
|
159
167
|
klass.is_a?(String) && (ACTIVE_JOB_WRAPPER.match?(klass) || klass == 'Sidekiq::ActiveJob::Wrapper')
|
|
160
168
|
end
|
data/lib/wurk/job_retry.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'zlib'
|
|
4
4
|
require_relative 'component'
|
|
5
5
|
require_relative 'dead_set'
|
|
6
|
+
require_relative 'fetcher/oom_drain'
|
|
6
7
|
|
|
7
8
|
module Wurk
|
|
8
9
|
# Owns the retry pipeline. When perform raises, JobRetry decides whether to
|
|
@@ -88,8 +89,13 @@ module Wurk
|
|
|
88
89
|
# Component's `handle_exception` delegates to `config.handle_exception`.
|
|
89
90
|
# When initialized with a Capsule, that's not defined directly; route
|
|
90
91
|
# through the underlying Configuration.
|
|
92
|
+
#
|
|
93
|
+
# Reporting a hook's failure from inside the failure path, through a logger or
|
|
94
|
+
# handler that is itself host code: it can't be allowed to raise from here.
|
|
91
95
|
def handle_exception(ex, ctx = {})
|
|
92
96
|
inner_config.handle_exception(ex, ctx)
|
|
97
|
+
rescue ::StandardError => e
|
|
98
|
+
warn "wurk-retry: error reporter raised #{e.class}: #{e.message}"
|
|
93
99
|
end
|
|
94
100
|
|
|
95
101
|
private
|
|
@@ -167,11 +173,21 @@ module Wurk
|
|
|
167
173
|
return unless msg['backtrace']
|
|
168
174
|
return if exception.backtrace.nil?
|
|
169
175
|
|
|
170
|
-
cleaned = (
|
|
176
|
+
cleaned = clean_backtrace(exception.backtrace)
|
|
171
177
|
lines = msg['backtrace'] == true ? cleaned : cleaned[0...msg['backtrace'].to_i]
|
|
172
178
|
msg['error_backtrace'] = compress_backtrace(lines)
|
|
173
179
|
end
|
|
174
180
|
|
|
181
|
+
# Host code, like the other hooks here: a raise out of this path is a job
|
|
182
|
+
# the Processor could not book, and it kills those.
|
|
183
|
+
def clean_backtrace(backtrace)
|
|
184
|
+
cleaned = @backtrace_cleaner ? @backtrace_cleaner.call(backtrace) : backtrace
|
|
185
|
+
cleaned.is_a?(::Array) ? cleaned : backtrace
|
|
186
|
+
rescue ::Exception => e # rubocop:disable Lint/RescueException
|
|
187
|
+
handle_exception(e, context: 'Error calling backtrace_cleaner')
|
|
188
|
+
backtrace
|
|
189
|
+
end
|
|
190
|
+
|
|
175
191
|
def retry_for_exceeded?(failed_at, retry_for)
|
|
176
192
|
return false unless failed_at
|
|
177
193
|
|
|
@@ -182,9 +198,7 @@ module Wurk
|
|
|
182
198
|
jitter = rand(10 * (count + 1))
|
|
183
199
|
retry_at = ::Time.now.to_f + delay + jitter
|
|
184
200
|
payload = Wurk.dump_json(msg)
|
|
185
|
-
|
|
186
|
-
conn.call('ZADD', Keys::RETRY, retry_at.to_s, payload)
|
|
187
|
-
end
|
|
201
|
+
zadd_past_maxmemory(Keys::RETRY, retry_at.to_s, payload)
|
|
188
202
|
Wurk::Metrics::Statsd.increment(
|
|
189
203
|
'jobs.retried',
|
|
190
204
|
tags: ["worker:#{msg['class']}", "queue:#{msg['queue']}"]
|
|
@@ -196,6 +210,17 @@ module Wurk
|
|
|
196
210
|
Wurk::Middleware::Status.retrying(msg)
|
|
197
211
|
end
|
|
198
212
|
|
|
213
|
+
# A failing job has to be booked even at maxmemory: the fetcher can still
|
|
214
|
+
# drain a full Redis, and a failure it could not book would sit in the private
|
|
215
|
+
# list until the next reconcile, then run and fail again.
|
|
216
|
+
def zadd_past_maxmemory(key, score, member)
|
|
217
|
+
redis do |conn|
|
|
218
|
+
conn.call('ZADD', key, score, member)
|
|
219
|
+
rescue RedisClient::OutOfMemoryError => e
|
|
220
|
+
Wurk::Fetcher::OomDrain.zadd(conn, e, key, score, member)
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
|
|
199
224
|
def time_for(item)
|
|
200
225
|
if item.is_a?(::Float)
|
|
201
226
|
::Time.at(item)
|
|
@@ -208,9 +233,6 @@ module Wurk
|
|
|
208
233
|
# Caller branches on strategy; seconds is meaningful only for :default.
|
|
209
234
|
def delay_for(jobinst, count, exception, msg)
|
|
210
235
|
rv = run_retry_in_block(jobinst, count, exception, msg)
|
|
211
|
-
# `10.minutes * count` (a Duration), `"60"` and `30.0` all mean seconds;
|
|
212
|
-
# Symbols have no #to_i, so `:kill` / `:discard` pass through intact.
|
|
213
|
-
rv = rv.to_i if rv.respond_to?(:to_i)
|
|
214
236
|
default_delay = (count**4) + 15
|
|
215
237
|
|
|
216
238
|
case rv
|
|
@@ -227,7 +249,10 @@ module Wurk
|
|
|
227
249
|
|
|
228
250
|
def run_retry_in_block(jobinst, count, exception, msg)
|
|
229
251
|
block = msg['wrapped'] ? wrapped_block(msg, :sidekiq_retry_in_block) : jobinst&.class&.sidekiq_retry_in_block
|
|
230
|
-
block&.call(count, exception, msg)
|
|
252
|
+
rv = block&.call(count, exception, msg)
|
|
253
|
+
# `10.minutes * count` (a Duration), `"60"` and `30.0` all mean seconds;
|
|
254
|
+
# Symbols have no #to_i, so `:kill` / `:discard` pass through intact.
|
|
255
|
+
rv.respond_to?(:to_i) ? rv.to_i : rv
|
|
231
256
|
rescue ::Exception => e # rubocop:disable Lint/RescueException
|
|
232
257
|
handle_exception(e, context: "Failure scheduling retry via `sidekiq_retry_in` on #{jobinst&.class&.name}")
|
|
233
258
|
nil
|
|
@@ -274,13 +299,16 @@ module Wurk
|
|
|
274
299
|
|
|
275
300
|
def send_to_morgue(msg)
|
|
276
301
|
logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
|
|
277
|
-
|
|
302
|
+
payload = Wurk.dump_json(msg)
|
|
303
|
+
DeadSet.new.kill_raw(payload)
|
|
304
|
+
rescue RedisClient::OutOfMemoryError
|
|
305
|
+
zadd_past_maxmemory(Keys::DEAD, ::Process.clock_gettime(::Process::CLOCK_REALTIME).to_s, payload)
|
|
278
306
|
end
|
|
279
307
|
|
|
280
308
|
def run_death_handlers(job, exception)
|
|
281
309
|
inner_config.death_handlers.each do |handler|
|
|
282
310
|
handler.call(job, exception)
|
|
283
|
-
rescue ::
|
|
311
|
+
rescue ::Exception => e # rubocop:disable Lint/RescueException
|
|
284
312
|
handle_exception(e, context: 'Error calling death handler', job: job)
|
|
285
313
|
end
|
|
286
314
|
end
|