wurk 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -1
- data/app/controllers/concerns/wurk/stream_concurrency_guard.rb +22 -5
- data/lib/wurk/capsule.rb +40 -12
- data/lib/wurk/client/buffered.rb +7 -0
- data/lib/wurk/client.rb +102 -32
- data/lib/wurk/component.rb +34 -2
- data/lib/wurk/configuration.rb +45 -6
- data/lib/wurk/context.rb +1 -1
- data/lib/wurk/engine.rb +41 -1
- data/lib/wurk/errors.rb +15 -0
- data/lib/wurk/fetcher/reaper.rb +12 -2
- data/lib/wurk/fetcher/reliable.rb +322 -60
- data/lib/wurk/fetcher.rb +6 -0
- data/lib/wurk/heartbeat.rb +2 -2
- data/lib/wurk/job_logger.rb +16 -7
- data/lib/wurk/job_set.rb +3 -2
- data/lib/wurk/job_util.rb +44 -24
- data/lib/wurk/launcher.rb +116 -72
- data/lib/wurk/leader.rb +36 -6
- data/lib/wurk/logger.rb +1 -1
- data/lib/wurk/lua/loader.rb +9 -3
- data/lib/wurk/metrics/accumulator.rb +95 -0
- data/lib/wurk/metrics/flusher.rb +70 -0
- data/lib/wurk/metrics/history.rb +98 -29
- data/lib/wurk/metrics/statsd.rb +32 -18
- data/lib/wurk/middleware/chain.rb +31 -14
- data/lib/wurk/middleware/poison_pill.rb +23 -2
- data/lib/wurk/pool_checkout.rb +10 -0
- data/lib/wurk/processor.rb +79 -11
- data/lib/wurk/profiler.rb +6 -2
- data/lib/wurk/queue.rb +8 -0
- data/lib/wurk/redis_client_adapter.rb +1 -1
- data/lib/wurk/redis_pool.rb +2 -1
- data/lib/wurk/shutdown_gate.rb +79 -0
- data/lib/wurk/stats.rb +5 -1
- data/lib/wurk/swarm/child_boot.rb +24 -4
- data/lib/wurk/swarm.rb +88 -14
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/config.rb +11 -7
- data/lib/wurk.rb +2 -8
- data/vendor/assets/dashboard/assets/{ArgsValue-D74zX0MI.js → ArgsValue-CcR2ya6e.js} +1 -1
- data/vendor/assets/dashboard/assets/{BatchDetail-OmC5NPgw.js → BatchDetail-CUXJUQ3Q.js} +1 -1
- data/vendor/assets/dashboard/assets/{Batches-CIpai7St.js → Batches-Cxan6Ngw.js} +1 -1
- data/vendor/assets/dashboard/assets/{Busy-A_kwSR6Q.js → Busy-DC5EGM0g.js} +1 -1
- data/vendor/assets/dashboard/assets/{Cron-BG7HTqlp.js → Cron-Dlt8tXJA.js} +1 -1
- data/vendor/assets/dashboard/assets/Dashboard-DNLu_WCg.js +1 -0
- data/vendor/assets/dashboard/assets/{Dead-8J21jMyK.js → Dead-dZ7VGlKS.js} +1 -1
- data/vendor/assets/dashboard/assets/{Extension-B4Q9FIQu.js → Extension-DaFpEIJf.js} +1 -1
- data/vendor/assets/dashboard/assets/{FilterBox-Fh_Ae7UW.js → FilterBox-CO3aYWIq.js} +1 -1
- data/vendor/assets/dashboard/assets/{JobDetailModal-Ceng0PMB.js → JobDetailModal-DSWbT6G0.js} +1 -1
- data/vendor/assets/dashboard/assets/{Limiters-CruDWvNZ.js → Limiters-Cb4PKXNR.js} +1 -1
- data/vendor/assets/dashboard/assets/Metrics-CCGzgCsT.js +1 -0
- data/vendor/assets/dashboard/assets/{Modal-CN3rdKA_.js → Modal-B86q6ruL.js} +1 -1
- data/vendor/assets/dashboard/assets/{PageHeader-C44KNMGm.js → PageHeader-fPrCcp_-.js} +1 -1
- data/vendor/assets/dashboard/assets/{Profiles-xEVTyS2N.js → Profiles-BnS82nR_.js} +1 -1
- data/vendor/assets/dashboard/assets/{Queues-D86FYohJ.js → Queues-CIyPevOy.js} +1 -1
- data/vendor/assets/dashboard/assets/{Retries-Bz1O1D-i.js → Retries-DopwXkXl.js} +1 -1
- data/vendor/assets/dashboard/assets/{Scheduled-B6h2akTu.js → Scheduled-1-Z7i1zE.js} +1 -1
- data/vendor/assets/dashboard/assets/{Search-OOu22e5s.js → Search-ByA6eTma.js} +1 -1
- data/vendor/assets/dashboard/assets/{Skeleton-DzR7XNxz.js → Skeleton-bC7HfQ9r.js} +1 -1
- data/vendor/assets/dashboard/assets/{charts-BVHHGof7.js → charts-CLLzJ7vK.js} +1 -1
- data/vendor/assets/dashboard/assets/index-B1N8hQUh.js +141 -0
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-dVPGEWzn.js → useResetPageOnEmpty-DpBjkf6_.js} +1 -1
- data/vendor/assets/dashboard/assets/{useSort-BeYbztkN.js → useSort-DvpwuNQE.js} +1 -1
- data/vendor/assets/dashboard/index.html +2 -2
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +30 -26
- data/vendor/assets/dashboard/assets/Dashboard-A_ToqHoo.js +0 -1
- data/vendor/assets/dashboard/assets/Metrics-CIT7VCoN.js +0 -1
- data/vendor/assets/dashboard/assets/index-D_lSDwKw.js +0 -141
data/lib/wurk/launcher.rb
CHANGED
|
@@ -11,9 +11,11 @@ require_relative 'leader'
|
|
|
11
11
|
require_relative 'cron'
|
|
12
12
|
require_relative 'metrics/rollup'
|
|
13
13
|
require_relative 'metrics/queue_rollup'
|
|
14
|
+
require_relative 'metrics/flusher'
|
|
14
15
|
require_relative 'history'
|
|
15
16
|
require_relative 'fetcher/reaper'
|
|
16
17
|
require_relative 'timer_loop'
|
|
18
|
+
require_relative 'shutdown_gate'
|
|
17
19
|
|
|
18
20
|
module Wurk
|
|
19
21
|
# Top-level supervisor inside each worker process. Owns the Manager pool
|
|
@@ -49,7 +51,7 @@ module Wurk
|
|
|
49
51
|
# shutdown lands right after boot; teardown must not hang on it.
|
|
50
52
|
BOOT_RECLAIM_JOIN_TIMEOUT = 5
|
|
51
53
|
|
|
52
|
-
attr_accessor :managers, :poller, :cron_poller, :metrics_rollup, :queue_rollup, :history
|
|
54
|
+
attr_accessor :managers, :poller, :cron_poller, :metrics_rollup, :queue_rollup, :metrics_flusher, :history
|
|
53
55
|
|
|
54
56
|
def initialize(config, embedded: false)
|
|
55
57
|
@config = config
|
|
@@ -59,31 +61,37 @@ module Wurk
|
|
|
59
61
|
# beating would never publish quiet=true and would expire out of the live
|
|
60
62
|
# set (#236). Only #stop ends the beat, by terminating @beat_timer.
|
|
61
63
|
@done = false
|
|
64
|
+
@shutdown_gate = ShutdownGate.new
|
|
62
65
|
@beat_timer = TimerLoop.new(BEAT_PAUSE)
|
|
63
66
|
@managers = build_managers
|
|
64
|
-
|
|
65
|
-
@cron_poller = build_cron_poller
|
|
66
|
-
@metrics_rollup = build_metrics_rollup
|
|
67
|
-
@queue_rollup = build_queue_rollup
|
|
68
|
-
@history = build_history
|
|
67
|
+
build_loops
|
|
69
68
|
@leader = build_leader
|
|
70
69
|
@reaper = build_reaper
|
|
71
|
-
@started_at = nil
|
|
72
|
-
@heartbeat = nil
|
|
73
|
-
@heartbeat_thread = nil
|
|
74
|
-
@boot_reclaim_thread = nil
|
|
75
70
|
@health_server = build_health_server
|
|
71
|
+
reset_thread_state
|
|
76
72
|
end
|
|
77
73
|
|
|
78
74
|
# Boot order matters:
|
|
79
75
|
# 1. freeze! the config so mutations after fork are visible mistakes.
|
|
80
|
-
#
|
|
81
|
-
#
|
|
82
|
-
#
|
|
83
|
-
#
|
|
84
|
-
#
|
|
85
|
-
#
|
|
86
|
-
#
|
|
76
|
+
# A swarm child finds the slot-independent half already frozen by its
|
|
77
|
+
# parent (Configuration#prepare_for_fork!); what closes here is the
|
|
78
|
+
# capsules, which stayed open for this child's slot and pools.
|
|
79
|
+
# 2. start the managers FIRST. They are the only thing here on the
|
|
80
|
+
# time-to-first-job path (the number bench/vs_sidekiq.rb measures);
|
|
81
|
+
# everything below is periodic background work whose first tick is
|
|
82
|
+
# seconds away, so it has nothing to gain from going ahead of them.
|
|
83
|
+
# 3. spawn the heartbeat thread, so the dashboard sees the process
|
|
84
|
+
# right after it can pick up jobs.
|
|
85
|
+
# 4. start the reaper and the boot-time reclaim sweep — kill-9 recovery,
|
|
86
|
+
# not decoration: a SIGKILLed sibling's in-flight jobs wait on this,
|
|
87
|
+
# so it stays ahead of the periodic loops.
|
|
88
|
+
# 5. start the periodic loops. None of them ticks at zero — TimerLoop#run
|
|
89
|
+
# waits an interval before its first yield, Scheduled::Poller waits
|
|
90
|
+
# INITIAL_WAIT, and Leader waits DEFAULT_INITIAL_WAIT — so starting
|
|
91
|
+
# them last costs a few thread spawns and nothing else. Both leader-
|
|
92
|
+
# gated pollers are also safe to start before leadership settles: a
|
|
93
|
+
# non-leader tick just returns early.
|
|
94
|
+
# 6. start the health probe server LAST so the listener doesn't
|
|
87
95
|
# accept k8s probes until the rest of the launcher is up.
|
|
88
96
|
def run(async_beat: true)
|
|
89
97
|
@started_at = Time.now.to_f
|
|
@@ -92,13 +100,13 @@ module Wurk
|
|
|
92
100
|
# CLI, embedded) runs through here, so none boots with a nil fetcher.
|
|
93
101
|
@config.capsules.each_value(&:prepare!)
|
|
94
102
|
@config.freeze!
|
|
95
|
-
@heartbeat_thread = safe_thread('heartbeat', &method(:start_heartbeat)) if async_beat
|
|
96
|
-
[@poller, @leader, @cron_poller, @metrics_rollup, @queue_rollup, @history].compact.each(&:start)
|
|
97
103
|
@managers.each(&:start)
|
|
104
|
+
@heartbeat_thread = safe_thread('heartbeat', &method(:start_heartbeat)) if async_beat
|
|
98
105
|
@reaper.start
|
|
99
106
|
# Run on a background thread so /ready probe isn't delayed by a large
|
|
100
107
|
# orphan sweep (reaper.reclaim! is atomic, but can scan many entries).
|
|
101
108
|
@boot_reclaim_thread = safe_thread('boot-reclaim', &method(:boot_reclaim))
|
|
109
|
+
start_periodic_components
|
|
102
110
|
@health_server&.start
|
|
103
111
|
rescue StandardError
|
|
104
112
|
# Boot is not atomic: whatever raised (a health-check port already bound,
|
|
@@ -125,20 +133,35 @@ module Wurk
|
|
|
125
133
|
fire_event(:quiet, reverse: true)
|
|
126
134
|
end
|
|
127
135
|
|
|
128
|
-
# Graceful shutdown
|
|
129
|
-
#
|
|
130
|
-
#
|
|
131
|
-
#
|
|
132
|
-
#
|
|
136
|
+
# Graceful shutdown, single-shot: several requests can be in flight at once
|
|
137
|
+
# (a dashboard-queued TERM, the embedded host's own `Embedded#stop`, a
|
|
138
|
+
# Manager that can no longer hold its concurrency), so the first caller
|
|
139
|
+
# drains and the rest wait it out — see ShutdownGate. Deadline is monotonic
|
|
140
|
+
# so wall-clock skew can't extend it. Managers stop in parallel threads so a
|
|
141
|
+
# slow capsule doesn't block its siblings — and each drain is guarded,
|
|
142
|
+
# because a capsule that blows up mid-drain (Redis down during bulk_requeue)
|
|
143
|
+
# used to surface out of `join` and skip the whole teardown tail, leaving the
|
|
133
144
|
# leader lock held, the process listed as live, and its port open.
|
|
145
|
+
#
|
|
146
|
+
# The joins get one shared budget of `deadline + TimerLoop::JOIN_TIMEOUT`,
|
|
147
|
+
# not an open-ended wait. Manager#stop is meant to run a little past the
|
|
148
|
+
# deadline — on expiry it bulk_requeues the in-flight UnitsOfWork, kills the
|
|
149
|
+
# threads and gives them a ~3s window to run their `ensure` blocks — but
|
|
150
|
+
# past that it is wedged (a `fetcher.terminate` / `bulk_requeue` parked on a
|
|
151
|
+
# Redis that stopped answering), and every further second spent waiting
|
|
152
|
+
# comes out of the teardown tail below, which the swarm parent only allows
|
|
153
|
+
# `shutdown_timeout + Swarm::SHUTDOWN_GRACE` for before it SIGKILLs us
|
|
154
|
+
# mid-drain.
|
|
134
155
|
def stop
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
156
|
+
@shutdown_gate.run do
|
|
157
|
+
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + (@config[:timeout] || 25)
|
|
158
|
+
quiet
|
|
159
|
+
stoppers = @managers.map { |m| Thread.new { teardown_step('manager') { m.stop(deadline) } } }
|
|
160
|
+
fire_event(:shutdown, reverse: true)
|
|
161
|
+
@shutdown_gate.join_within(stoppers, deadline + TimerLoop::JOIN_TIMEOUT)
|
|
162
|
+
ensure
|
|
163
|
+
release_components
|
|
164
|
+
end
|
|
142
165
|
end
|
|
143
166
|
|
|
144
167
|
def stopping?
|
|
@@ -200,12 +223,14 @@ module Wurk
|
|
|
200
223
|
%i[stop_heartbeat clear_heartbeat].each { |step| teardown_step(step) { send(step) } }
|
|
201
224
|
# Unguarded: fire_event already reports and skips past a raising hook.
|
|
202
225
|
fire_event(:exit, reverse: true)
|
|
203
|
-
#
|
|
204
|
-
# #stop
|
|
205
|
-
#
|
|
206
|
-
#
|
|
207
|
-
#
|
|
208
|
-
|
|
226
|
+
# Unconditional. A swarm child or standalone process exits right after
|
|
227
|
+
# #stop, so for those it only closes sockets the kernel was about to close
|
|
228
|
+
# anyway — but every other driver survives us: an embedded host (Puma, a
|
|
229
|
+
# rake task) that boots a fresh Launcher later, and anything driving
|
|
230
|
+
# Launcher directly. Skipping those leaked one main + one fetch pool per
|
|
231
|
+
# capsule, live, for the life of the host. Idempotent: `reset_redis_pools!`
|
|
232
|
+
# drops the memo, so the next checkout rebuilds.
|
|
233
|
+
teardown_step('redis-pools') { @config.reset_redis_pools! }
|
|
209
234
|
ensure
|
|
210
235
|
# In an ensure of its own, not merely a guarded step: a leaked TCPServer
|
|
211
236
|
# FD survives even a non-StandardError unwind (a second TERM landing
|
|
@@ -214,11 +239,26 @@ module Wurk
|
|
|
214
239
|
teardown_step('health-server') { @health_server&.stop }
|
|
215
240
|
end
|
|
216
241
|
|
|
242
|
+
# Mirror of #stop_periodic_components: every loop this process runs on a
|
|
243
|
+
# timer, started in one place at the tail of #run. `compact` because the
|
|
244
|
+
# history snapshotter is opt-in and an embedded boot may have stripped the
|
|
245
|
+
# rest.
|
|
246
|
+
def start_periodic_components
|
|
247
|
+
[@poller, @leader, @cron_poller, @metrics_rollup,
|
|
248
|
+
@queue_rollup, @metrics_flusher, @history].compact.each(&:start)
|
|
249
|
+
end
|
|
250
|
+
|
|
217
251
|
# Split out of #release_components to keep it under the AbcSize/
|
|
218
|
-
# CyclomaticComplexity ceilings — these
|
|
219
|
-
#
|
|
252
|
+
# CyclomaticComplexity ceilings — these are the "periodic loop" releases,
|
|
253
|
+
# independently guarded like everything else in the tail.
|
|
254
|
+
#
|
|
255
|
+
# The metrics flusher goes first: its #terminate drains the last window of
|
|
256
|
+
# per-job counters into the `j|…` minute buckets, and Metrics::Rollup reads
|
|
257
|
+
# exactly those buckets — stopping the rollup first would give its final
|
|
258
|
+
# tick nothing to roll.
|
|
220
259
|
def stop_periodic_components
|
|
221
|
-
[@cron_poller, @metrics_rollup, @queue_rollup,
|
|
260
|
+
[@metrics_flusher, @cron_poller, @metrics_rollup, @queue_rollup,
|
|
261
|
+
@history].each { |t| teardown_step(t.class) { t&.terminate } }
|
|
222
262
|
teardown_step('reaper') { @reaper&.stop }
|
|
223
263
|
teardown_step('leader') { @leader&.stop }
|
|
224
264
|
end
|
|
@@ -335,8 +375,15 @@ module Wurk
|
|
|
335
375
|
# concurrency. Standalone hands off to the installed TERM trap; embedded
|
|
336
376
|
# owns no traps, so it drains in place — on its own thread, because callers
|
|
337
377
|
# may be a thread `stop` itself joins (the heartbeat) or kills (a Processor).
|
|
378
|
+
#
|
|
379
|
+
# That thread is spawned once and retained by the gate: every later request
|
|
380
|
+
# rides the same drain instead of leaving another unreferenced `stop`
|
|
381
|
+
# running behind it. Returned so a caller that can afford to wait (a test,
|
|
382
|
+
# a host teardown) has something to join.
|
|
338
383
|
def request_shutdown
|
|
339
|
-
|
|
384
|
+
return redeliver('TERM') unless @embedded
|
|
385
|
+
|
|
386
|
+
@shutdown_gate.runner { safe_thread('launcher-stop') { stop } }
|
|
340
387
|
end
|
|
341
388
|
|
|
342
389
|
# Separate method so tests can stub it — really sending TERM/TSTP would
|
|
@@ -352,49 +399,46 @@ module Wurk
|
|
|
352
399
|
@config.capsules.values.map { |cap| Manager.new(cap, shutdown: method(:request_shutdown)) }
|
|
353
400
|
end
|
|
354
401
|
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
#
|
|
360
|
-
#
|
|
361
|
-
#
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
#
|
|
367
|
-
#
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
Wurk::Metrics::Rollup.new(@config)
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
# process runs one but only the leader writes the `qm|…` size/latency
|
|
376
|
-
# buckets the Historical tab's per-queue charts read.
|
|
377
|
-
def build_queue_rollup
|
|
378
|
-
Wurk::Metrics::QueueRollup.new(@config)
|
|
402
|
+
# The periodic loops, built in one place because they are started together
|
|
403
|
+
# (#run) and released together (#stop_periodic_components). Every process
|
|
404
|
+
# runs every one of them; what differs is who does the work inside a tick:
|
|
405
|
+
#
|
|
406
|
+
# poller / reaper every process, work shared through Redis
|
|
407
|
+
# cron_poller leader only enqueues — that is the exactly-once guard
|
|
408
|
+
# metrics_rollup leader only writes the cluster-total chart buckets
|
|
409
|
+
# (cadence: `config.metrics_rollup_interval`)
|
|
410
|
+
# queue_rollup leader only writes the `qm|…` per-queue gauges
|
|
411
|
+
# metrics_flusher NOT leader-gated: the counters it drains exist only in
|
|
412
|
+
# this process's memory, so every process flushes its own
|
|
413
|
+
# history Ent §5 snapshotter, leader-gated, opt-in via
|
|
414
|
+
# `config.retain_history`
|
|
415
|
+
def build_loops
|
|
416
|
+
@poller = Wurk::Scheduled::Poller.new(@config)
|
|
417
|
+
@cron_poller = Wurk::Cron::Poller.new(@config)
|
|
418
|
+
@metrics_rollup = Wurk::Metrics::Rollup.new(@config)
|
|
419
|
+
@queue_rollup = Wurk::Metrics::QueueRollup.new(@config)
|
|
420
|
+
@metrics_flusher = Wurk::Metrics::Flusher.new(@config)
|
|
421
|
+
@history = Wurk::History.new(@config) if @config.history_enabled?
|
|
379
422
|
end
|
|
380
423
|
|
|
381
|
-
#
|
|
382
|
-
#
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
Wurk::History.new(@config) if @config.history_enabled?
|
|
424
|
+
# Filled in by #run. Declared up front so #initialize reads as the full
|
|
425
|
+
# inventory of what a Launcher owns.
|
|
426
|
+
def reset_thread_state
|
|
427
|
+
@started_at = @heartbeat = @heartbeat_thread = @boot_reclaim_thread = nil
|
|
386
428
|
end
|
|
387
429
|
|
|
388
430
|
# Every worker process campaigns for the single cluster lock (`dear-leader`);
|
|
389
431
|
# one wins and renews it, the rest follow and promote on its death. Cadence
|
|
390
432
|
# falls back to the spec defaults (TTL 30 / renew 15 / follower 60) unless
|
|
391
|
-
# the host tunes it
|
|
433
|
+
# the host tunes it, plus the short pre-campaign delay that keeps the first
|
|
434
|
+
# CAS off the boot path. `Leader#start` no-ops under `WURK_LEADER=false`.
|
|
392
435
|
def build_leader
|
|
393
436
|
Wurk::Leader.new(
|
|
394
437
|
config: @config,
|
|
395
438
|
ttl: @config[:leader_ttl] || Wurk::Leader::DEFAULT_TTL,
|
|
396
439
|
renew_interval: @config[:leader_renew_interval] || Wurk::Leader::DEFAULT_RENEW_INTERVAL,
|
|
397
|
-
follower_interval: @config[:leader_follower_interval] || Wurk::Leader::DEFAULT_FOLLOWER_INTERVAL
|
|
440
|
+
follower_interval: @config[:leader_follower_interval] || Wurk::Leader::DEFAULT_FOLLOWER_INTERVAL,
|
|
441
|
+
initial_wait: @config[:leader_initial_wait] || Wurk::Leader::DEFAULT_INITIAL_WAIT
|
|
398
442
|
)
|
|
399
443
|
end
|
|
400
444
|
|
data/lib/wurk/leader.rb
CHANGED
|
@@ -5,6 +5,7 @@ require 'socket'
|
|
|
5
5
|
require_relative 'component'
|
|
6
6
|
require_relative 'lua'
|
|
7
7
|
require_relative 'pool_checkout'
|
|
8
|
+
require_relative 'timer_loop'
|
|
8
9
|
|
|
9
10
|
module Wurk
|
|
10
11
|
# Cluster leader election via Redis `SET NX EX`. Single-leader-per-cluster
|
|
@@ -21,9 +22,12 @@ module Wurk
|
|
|
21
22
|
# too — it is never re-read on subsequent acquires, only on transitions.
|
|
22
23
|
#
|
|
23
24
|
# Cadence per spec: renew every 15s while leader, recheck every 60s as
|
|
24
|
-
# follower, lock TTL 30s.
|
|
25
|
-
#
|
|
26
|
-
#
|
|
25
|
+
# follower, lock TTL 30s. The campaign itself starts `initial_wait` seconds
|
|
26
|
+
# after `#start` — the spec fixes the renew/follower/TTL cadence but says
|
|
27
|
+
# nothing about when the first campaign runs, and holding it back keeps a
|
|
28
|
+
# booting process's opening Redis round trips on the job path. Opt out a
|
|
29
|
+
# process from campaigning entirely with `WURK_LEADER=false` (or its Sidekiq
|
|
30
|
+
# alias `SIDEKIQ_LEADER=false`), useful for hot-standby pools.
|
|
27
31
|
#
|
|
28
32
|
# Spec: docs/target/sidekiq-ent.md §6.
|
|
29
33
|
class Leader
|
|
@@ -32,6 +36,13 @@ module Wurk
|
|
|
32
36
|
DEFAULT_TTL = 30
|
|
33
37
|
DEFAULT_RENEW_INTERVAL = 15
|
|
34
38
|
DEFAULT_FOLLOWER_INTERVAL = 60
|
|
39
|
+
# Delay before the campaign's first `SET NX EX`, so a booting process
|
|
40
|
+
# spends its opening Redis round trips on fetching jobs rather than on a
|
|
41
|
+
# leadership CAS. Every leader-gated consumer (cron, both rollups,
|
|
42
|
+
# history) waits out a 30-60s interval before its own first tick, so
|
|
43
|
+
# nothing observes the gap. Tune with `config[:leader_initial_wait]`;
|
|
44
|
+
# 0 campaigns immediately.
|
|
45
|
+
DEFAULT_INITIAL_WAIT = 1
|
|
35
46
|
OPT_OUT_ENV = 'WURK_LEADER' # native opt-out env
|
|
36
47
|
SIDEKIQ_OPT_OUT_ENV = 'SIDEKIQ_LEADER' # Sidekiq Ent drop-in alias (§6.2/§7.2)
|
|
37
48
|
THREAD_NAME = 'wurk-leader'
|
|
@@ -48,12 +59,14 @@ module Wurk
|
|
|
48
59
|
def initialize(config: nil, key: DEFAULT_KEY, ttl: DEFAULT_TTL, # rubocop:disable Metrics/ParameterLists
|
|
49
60
|
renew_interval: DEFAULT_RENEW_INTERVAL,
|
|
50
61
|
follower_interval: DEFAULT_FOLLOWER_INTERVAL,
|
|
62
|
+
initial_wait: DEFAULT_INITIAL_WAIT,
|
|
51
63
|
pool: nil, owner: nil)
|
|
52
64
|
@config = config
|
|
53
65
|
@key = key
|
|
54
66
|
@ttl = ttl
|
|
55
67
|
@renew_interval = renew_interval
|
|
56
68
|
@follower_interval = follower_interval
|
|
69
|
+
@initial_wait = initial_wait.to_f
|
|
57
70
|
@pool = pool
|
|
58
71
|
@owner = owner || cluster_identity
|
|
59
72
|
@held = false
|
|
@@ -136,14 +149,20 @@ module Wurk
|
|
|
136
149
|
@thread
|
|
137
150
|
end
|
|
138
151
|
|
|
152
|
+
# Bounded at TimerLoop::JOIN_TIMEOUT like every other periodic component: a
|
|
153
|
+
# tick parked on a wedged Redis (the campaign is a `SET NX EX`, so it waits
|
|
154
|
+
# out the socket timeouts) must not hold the whole process's teardown open —
|
|
155
|
+
# the swarm parent SIGKILLs a child that overruns its shutdown grace. The
|
|
156
|
+
# CAS release below runs either way; the loop's own trailing release makes
|
|
157
|
+
# a straggler's second attempt a no-op. A straggler also stays referenced,
|
|
158
|
+
# so a restart after a timed-out stop can't campaign twice in parallel.
|
|
139
159
|
def stop
|
|
140
160
|
thread = @mutex.synchronize do
|
|
141
161
|
@done = true
|
|
142
162
|
@sleeper.signal
|
|
143
163
|
@thread
|
|
144
164
|
end
|
|
145
|
-
thread
|
|
146
|
-
@mutex.synchronize { @thread = nil }
|
|
165
|
+
@mutex.synchronize { @thread = nil } if thread.nil? || thread.join(TimerLoop::JOIN_TIMEOUT)
|
|
147
166
|
release
|
|
148
167
|
end
|
|
149
168
|
|
|
@@ -224,6 +243,7 @@ module Wurk
|
|
|
224
243
|
end
|
|
225
244
|
|
|
226
245
|
def run_loop
|
|
246
|
+
wait_initial
|
|
227
247
|
until done?
|
|
228
248
|
tick_once
|
|
229
249
|
wait_next
|
|
@@ -231,6 +251,13 @@ module Wurk
|
|
|
231
251
|
release
|
|
232
252
|
end
|
|
233
253
|
|
|
254
|
+
# Same condvar as #wait_next, so a `stop` landing inside the pre-campaign
|
|
255
|
+
# delay wakes the loop immediately instead of holding shutdown open for a
|
|
256
|
+
# whole initial_wait.
|
|
257
|
+
def wait_initial
|
|
258
|
+
wait_for(@initial_wait) if @initial_wait.positive?
|
|
259
|
+
end
|
|
260
|
+
|
|
234
261
|
def tick_once
|
|
235
262
|
acquire
|
|
236
263
|
rescue StandardError => e
|
|
@@ -238,7 +265,10 @@ module Wurk
|
|
|
238
265
|
end
|
|
239
266
|
|
|
240
267
|
def wait_next
|
|
241
|
-
|
|
268
|
+
wait_for(@held ? @renew_interval : @follower_interval)
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def wait_for(interval)
|
|
242
272
|
@mutex.synchronize { @sleeper.wait(@mutex, interval) unless @done }
|
|
243
273
|
end
|
|
244
274
|
|
data/lib/wurk/logger.rb
CHANGED
data/lib/wurk/lua/loader.rb
CHANGED
|
@@ -23,9 +23,15 @@ module Wurk
|
|
|
23
23
|
# connection errors are the pool wrapper's job (Wurk::RedisPool#with);
|
|
24
24
|
# this only ships the loads.
|
|
25
25
|
def script_load_all(redis)
|
|
26
|
-
redis.pipelined
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
redis.pipelined { |pipe| queue_script_loads(pipe) }
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# The same loads, queued onto a pipeline the caller already owns, for a
|
|
30
|
+
# caller with other work to batch with them: ChildBoot pairs them with
|
|
31
|
+
# its liveness PING so a child's whole Redis validation costs one round
|
|
32
|
+
# trip instead of two, on the boot-critical path.
|
|
33
|
+
def queue_script_loads(pipe)
|
|
34
|
+
SCRIPTS.each_value { |src| pipe.call('SCRIPT', 'LOAD', src) }
|
|
29
35
|
end
|
|
30
36
|
|
|
31
37
|
# @param redis [RedisClient] a single connection (not a pool)
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
module Metrics
|
|
5
|
+
# The in-memory half of Wurk::Metrics::History. Every execution folds into a
|
|
6
|
+
# `{pool => {minute => {class => [processed, failed, ms]}}}` tree under one
|
|
7
|
+
# mutex; Wurk::Metrics::Flusher drains it into Redis on a timer. `HINCRBY`
|
|
8
|
+
# is additive, so N folded executions leave Redis in the same state as the N
|
|
9
|
+
# individual pipelines the hot path used to send.
|
|
10
|
+
#
|
|
11
|
+
# Knows nothing about Redis — History owns the write, this owns the counts.
|
|
12
|
+
#
|
|
13
|
+
# The minute is the bucket the job *ran* in, decided by the caller at record
|
|
14
|
+
# time. That is what keeps the batching a visibility lag instead of a
|
|
15
|
+
# misattribution: a job that runs at 12:03:59 and flushes at 12:04:02 still
|
|
16
|
+
# counts toward 12:03. Signed off in
|
|
17
|
+
# docs/plans/2026/08/06/101-faster-than-sidekiq/00-semantics-signoff.md §2.
|
|
18
|
+
#
|
|
19
|
+
# Keyed by the pool the recording middleware handed us (nil = the process
|
|
20
|
+
# default), so a multi-capsule process flushes each capsule's counts through
|
|
21
|
+
# the pool that capsule's jobs ran on.
|
|
22
|
+
class Accumulator
|
|
23
|
+
# Ceiling on minute buckets retained per pool. Only reachable through
|
|
24
|
+
# #merge_back: a flush that keeps failing puts its counts back every tick,
|
|
25
|
+
# and a Redis outage that outlives the process would otherwise grow a
|
|
26
|
+
# structure fed by the job hot path without bound. The newest buckets win
|
|
27
|
+
# — they are the ones the dashboard is still drawing — and the oldest are
|
|
28
|
+
# dropped, which these counters have always been allowed to do (a Redis
|
|
29
|
+
# failure during a metrics write has never changed a job's outcome).
|
|
30
|
+
MAX_RETAINED_MINUTES = 60
|
|
31
|
+
|
|
32
|
+
def initialize
|
|
33
|
+
@lock = ::Mutex.new
|
|
34
|
+
@pools = {}
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Hot path. Three hash lookups under the mutex; allocates only the first
|
|
38
|
+
# time a (pool, minute, class) triple is seen.
|
|
39
|
+
def add(pool, klass, minute, ms, success)
|
|
40
|
+
@lock.synchronize do
|
|
41
|
+
counts = (((@pools[pool] ||= {})[minute] ||= {})[klass] ||= [0, 0, 0])
|
|
42
|
+
counts[success ? 0 : 1] += 1
|
|
43
|
+
counts[2] += ms
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Hands the whole tree over and starts a fresh one, so recording never
|
|
48
|
+
# blocks behind the flush's Redis round trip.
|
|
49
|
+
def drain
|
|
50
|
+
@lock.synchronize do
|
|
51
|
+
drained = @pools
|
|
52
|
+
@pools = {}
|
|
53
|
+
drained
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Puts one pool's counts back after its write failed, so the next tick
|
|
58
|
+
# retries them instead of dropping the window. Merged rather than
|
|
59
|
+
# assigned: jobs kept recording into the fresh tree while the flush was in
|
|
60
|
+
# flight, and those counts have not been written yet either.
|
|
61
|
+
def merge_back(pool, minutes)
|
|
62
|
+
@lock.synchronize do
|
|
63
|
+
into = (@pools[pool] ||= {})
|
|
64
|
+
minutes.each { |minute, classes| merge_minute(into, minute, classes) }
|
|
65
|
+
trim(into)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def empty?
|
|
70
|
+
@lock.synchronize { @pools.empty? }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
def merge_minute(into, minute, classes)
|
|
76
|
+
target = (into[minute] ||= {})
|
|
77
|
+
classes.each do |klass, counts|
|
|
78
|
+
existing = target[klass]
|
|
79
|
+
if existing
|
|
80
|
+
counts.each_with_index { |count, i| existing[i] += count }
|
|
81
|
+
else
|
|
82
|
+
target[klass] = counts
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def trim(minutes)
|
|
88
|
+
excess = minutes.size - MAX_RETAINED_MINUTES
|
|
89
|
+
return if excess <= 0
|
|
90
|
+
|
|
91
|
+
minutes.keys.min(excess).each { |minute| minutes.delete(minute) }
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../component'
|
|
4
|
+
require_relative '../timer_loop'
|
|
5
|
+
require_relative 'history'
|
|
6
|
+
|
|
7
|
+
module Wurk
|
|
8
|
+
module Metrics
|
|
9
|
+
# Drains Wurk::Metrics::History's in-process accumulator into Redis every
|
|
10
|
+
# `History::FLUSH_INTERVAL` seconds.
|
|
11
|
+
#
|
|
12
|
+
# Deliberately NOT leader-gated, unlike every other periodic component here
|
|
13
|
+
# (Metrics::Rollup, Metrics::QueueRollup, Wurk::History). Those publish
|
|
14
|
+
# *cluster* state that one process should own; this publishes counters that
|
|
15
|
+
# only exist inside this process's memory and that no other process can
|
|
16
|
+
# write. A leader gate would strand every follower's metrics until it died.
|
|
17
|
+
#
|
|
18
|
+
# One tick per process, not per capsule: the accumulator is process-wide and
|
|
19
|
+
# already keyed by the pool each capsule's jobs recorded through.
|
|
20
|
+
class Flusher
|
|
21
|
+
include Component
|
|
22
|
+
|
|
23
|
+
# `accumulator` is a collaborator, not an option: every real boot flushes
|
|
24
|
+
# the process-wide one. Tests inject their own so a broken pool stays
|
|
25
|
+
# inside the test that built it.
|
|
26
|
+
def initialize(config, accumulator: History::ACCUMULATOR)
|
|
27
|
+
@config = config
|
|
28
|
+
@accumulator = accumulator
|
|
29
|
+
@timer = TimerLoop.new(History::FLUSH_INTERVAL)
|
|
30
|
+
@thread = nil
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def start
|
|
34
|
+
return @thread if @thread
|
|
35
|
+
|
|
36
|
+
@timer.reset
|
|
37
|
+
@thread = safe_thread('metrics-flush') { @timer.run { tick } }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Flushes once more in an `ensure`: this stops the only thread that would
|
|
41
|
+
# have written the window accumulated since the last tick, so without it a
|
|
42
|
+
# graceful shutdown would drop up to FLUSH_INTERVAL of counters — the cost
|
|
43
|
+
# the sign-off accepts for a *hard* kill only.
|
|
44
|
+
#
|
|
45
|
+
# Launcher#stop joins the manager drains before the teardown tail reaches
|
|
46
|
+
# us, so by the time this runs no job is still recording.
|
|
47
|
+
#
|
|
48
|
+
# Cleared only on a confirmed join (Thread#join returns nil on timeout): a
|
|
49
|
+
# wedged thread must stay tracked so #start's guard returns it instead of
|
|
50
|
+
# calling @timer.reset, which would un-terminate the loop it is still
|
|
51
|
+
# inside and leave two threads draining the same accumulator.
|
|
52
|
+
def terminate
|
|
53
|
+
@timer.terminate
|
|
54
|
+
@thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
|
|
55
|
+
ensure
|
|
56
|
+
tick
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Never raises: this runs on a `safe_thread`, whose watchdog re-raises
|
|
60
|
+
# after reporting, and a reported-then-dead flusher would silently stop
|
|
61
|
+
# every counter in the process for a blip Redis recovers from. The
|
|
62
|
+
# accumulator has already merged the failed window back for the next tick.
|
|
63
|
+
def tick
|
|
64
|
+
History.flush(@accumulator)
|
|
65
|
+
rescue StandardError => e
|
|
66
|
+
handle_exception(e, { context: 'metrics-flush' })
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|