wurk 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +3 -1
  3. data/app/controllers/concerns/wurk/stream_concurrency_guard.rb +22 -5
  4. data/lib/wurk/capsule.rb +40 -12
  5. data/lib/wurk/client/buffered.rb +7 -0
  6. data/lib/wurk/client.rb +102 -32
  7. data/lib/wurk/component.rb +34 -2
  8. data/lib/wurk/configuration.rb +45 -6
  9. data/lib/wurk/context.rb +1 -1
  10. data/lib/wurk/engine.rb +41 -1
  11. data/lib/wurk/errors.rb +15 -0
  12. data/lib/wurk/fetcher/reaper.rb +12 -2
  13. data/lib/wurk/fetcher/reliable.rb +322 -60
  14. data/lib/wurk/fetcher.rb +6 -0
  15. data/lib/wurk/heartbeat.rb +2 -2
  16. data/lib/wurk/job_logger.rb +16 -7
  17. data/lib/wurk/job_set.rb +3 -2
  18. data/lib/wurk/job_util.rb +44 -24
  19. data/lib/wurk/launcher.rb +116 -72
  20. data/lib/wurk/leader.rb +36 -6
  21. data/lib/wurk/logger.rb +1 -1
  22. data/lib/wurk/lua/loader.rb +9 -3
  23. data/lib/wurk/metrics/accumulator.rb +95 -0
  24. data/lib/wurk/metrics/flusher.rb +70 -0
  25. data/lib/wurk/metrics/history.rb +98 -29
  26. data/lib/wurk/metrics/statsd.rb +32 -18
  27. data/lib/wurk/middleware/chain.rb +31 -14
  28. data/lib/wurk/middleware/poison_pill.rb +23 -2
  29. data/lib/wurk/pool_checkout.rb +10 -0
  30. data/lib/wurk/processor.rb +79 -11
  31. data/lib/wurk/profiler.rb +6 -2
  32. data/lib/wurk/queue.rb +8 -0
  33. data/lib/wurk/redis_client_adapter.rb +1 -1
  34. data/lib/wurk/redis_pool.rb +2 -1
  35. data/lib/wurk/shutdown_gate.rb +79 -0
  36. data/lib/wurk/stats.rb +5 -1
  37. data/lib/wurk/swarm/child_boot.rb +24 -4
  38. data/lib/wurk/swarm.rb +88 -14
  39. data/lib/wurk/version.rb +1 -1
  40. data/lib/wurk/web/config.rb +11 -7
  41. data/lib/wurk.rb +2 -8
  42. data/vendor/assets/dashboard/assets/{ArgsValue-D74zX0MI.js → ArgsValue-CcR2ya6e.js} +1 -1
  43. data/vendor/assets/dashboard/assets/{BatchDetail-OmC5NPgw.js → BatchDetail-CUXJUQ3Q.js} +1 -1
  44. data/vendor/assets/dashboard/assets/{Batches-CIpai7St.js → Batches-Cxan6Ngw.js} +1 -1
  45. data/vendor/assets/dashboard/assets/{Busy-A_kwSR6Q.js → Busy-DC5EGM0g.js} +1 -1
  46. data/vendor/assets/dashboard/assets/{Cron-BG7HTqlp.js → Cron-Dlt8tXJA.js} +1 -1
  47. data/vendor/assets/dashboard/assets/Dashboard-DNLu_WCg.js +1 -0
  48. data/vendor/assets/dashboard/assets/{Dead-8J21jMyK.js → Dead-dZ7VGlKS.js} +1 -1
  49. data/vendor/assets/dashboard/assets/{Extension-B4Q9FIQu.js → Extension-DaFpEIJf.js} +1 -1
  50. data/vendor/assets/dashboard/assets/{FilterBox-Fh_Ae7UW.js → FilterBox-CO3aYWIq.js} +1 -1
  51. data/vendor/assets/dashboard/assets/{JobDetailModal-Ceng0PMB.js → JobDetailModal-DSWbT6G0.js} +1 -1
  52. data/vendor/assets/dashboard/assets/{Limiters-CruDWvNZ.js → Limiters-Cb4PKXNR.js} +1 -1
  53. data/vendor/assets/dashboard/assets/Metrics-CCGzgCsT.js +1 -0
  54. data/vendor/assets/dashboard/assets/{Modal-CN3rdKA_.js → Modal-B86q6ruL.js} +1 -1
  55. data/vendor/assets/dashboard/assets/{PageHeader-C44KNMGm.js → PageHeader-fPrCcp_-.js} +1 -1
  56. data/vendor/assets/dashboard/assets/{Profiles-xEVTyS2N.js → Profiles-BnS82nR_.js} +1 -1
  57. data/vendor/assets/dashboard/assets/{Queues-D86FYohJ.js → Queues-CIyPevOy.js} +1 -1
  58. data/vendor/assets/dashboard/assets/{Retries-Bz1O1D-i.js → Retries-DopwXkXl.js} +1 -1
  59. data/vendor/assets/dashboard/assets/{Scheduled-B6h2akTu.js → Scheduled-1-Z7i1zE.js} +1 -1
  60. data/vendor/assets/dashboard/assets/{Search-OOu22e5s.js → Search-ByA6eTma.js} +1 -1
  61. data/vendor/assets/dashboard/assets/{Skeleton-DzR7XNxz.js → Skeleton-bC7HfQ9r.js} +1 -1
  62. data/vendor/assets/dashboard/assets/{charts-BVHHGof7.js → charts-CLLzJ7vK.js} +1 -1
  63. data/vendor/assets/dashboard/assets/index-B1N8hQUh.js +141 -0
  64. data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-dVPGEWzn.js → useResetPageOnEmpty-DpBjkf6_.js} +1 -1
  65. data/vendor/assets/dashboard/assets/{useSort-BeYbztkN.js → useSort-DvpwuNQE.js} +1 -1
  66. data/vendor/assets/dashboard/index.html +2 -2
  67. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  68. metadata +30 -26
  69. data/vendor/assets/dashboard/assets/Dashboard-A_ToqHoo.js +0 -1
  70. data/vendor/assets/dashboard/assets/Metrics-CIT7VCoN.js +0 -1
  71. data/vendor/assets/dashboard/assets/index-D_lSDwKw.js +0 -141
data/lib/wurk/launcher.rb CHANGED
@@ -11,9 +11,11 @@ require_relative 'leader'
11
11
  require_relative 'cron'
12
12
  require_relative 'metrics/rollup'
13
13
  require_relative 'metrics/queue_rollup'
14
+ require_relative 'metrics/flusher'
14
15
  require_relative 'history'
15
16
  require_relative 'fetcher/reaper'
16
17
  require_relative 'timer_loop'
18
+ require_relative 'shutdown_gate'
17
19
 
18
20
  module Wurk
19
21
  # Top-level supervisor inside each worker process. Owns the Manager pool
@@ -49,7 +51,7 @@ module Wurk
49
51
  # shutdown lands right after boot; teardown must not hang on it.
50
52
  BOOT_RECLAIM_JOIN_TIMEOUT = 5
51
53
 
52
- attr_accessor :managers, :poller, :cron_poller, :metrics_rollup, :queue_rollup, :history
54
+ attr_accessor :managers, :poller, :cron_poller, :metrics_rollup, :queue_rollup, :metrics_flusher, :history
53
55
 
54
56
  def initialize(config, embedded: false)
55
57
  @config = config
@@ -59,31 +61,37 @@ module Wurk
59
61
  # beating would never publish quiet=true and would expire out of the live
60
62
  # set (#236). Only #stop ends the beat, by terminating @beat_timer.
61
63
  @done = false
64
+ @shutdown_gate = ShutdownGate.new
62
65
  @beat_timer = TimerLoop.new(BEAT_PAUSE)
63
66
  @managers = build_managers
64
- @poller = build_poller
65
- @cron_poller = build_cron_poller
66
- @metrics_rollup = build_metrics_rollup
67
- @queue_rollup = build_queue_rollup
68
- @history = build_history
67
+ build_loops
69
68
  @leader = build_leader
70
69
  @reaper = build_reaper
71
- @started_at = nil
72
- @heartbeat = nil
73
- @heartbeat_thread = nil
74
- @boot_reclaim_thread = nil
75
70
  @health_server = build_health_server
71
+ reset_thread_state
76
72
  end
77
73
 
78
74
  # Boot order matters:
79
75
  # 1. freeze! the config so mutations after fork are visible mistakes.
80
- # 2. spawn the heartbeat thread BEFORE the managers so the dashboard
81
- # sees the process the moment it can pick up jobs.
82
- # 3. start the scheduler poller + the cron poller (both leader-gated for
83
- # what they enqueue; safe to start before leadership is settled since
84
- # a non-leader tick just returns early).
85
- # 4. start the managers (which start their processors).
86
- # 5. start the health probe server LAST so the listener doesn't
76
+ # A swarm child finds the slot-independent half already frozen by its
77
+ # parent (Configuration#prepare_for_fork!); what closes here is the
78
+ # capsules, which stayed open for this child's slot and pools.
79
+ # 2. start the managers FIRST. They are the only thing here on the
80
+ # time-to-first-job path (the number bench/vs_sidekiq.rb measures);
81
+ # everything below is periodic background work whose first tick is
82
+ # seconds away, so it has nothing to gain from going ahead of them.
83
+ # 3. spawn the heartbeat thread, so the dashboard sees the process
84
+ # right after it can pick up jobs.
85
+ # 4. start the reaper and the boot-time reclaim sweep — kill-9 recovery,
86
+ # not decoration: a SIGKILLed sibling's in-flight jobs wait on this,
87
+ # so it stays ahead of the periodic loops.
88
+ # 5. start the periodic loops. None of them ticks at zero — TimerLoop#run
89
+ # waits an interval before its first yield, Scheduled::Poller waits
90
+ # INITIAL_WAIT, and Leader waits DEFAULT_INITIAL_WAIT — so starting
91
+ # them last costs a few thread spawns and nothing else. Both leader-
92
+ # gated pollers are also safe to start before leadership settles: a
93
+ # non-leader tick just returns early.
94
+ # 6. start the health probe server LAST so the listener doesn't
87
95
  # accept k8s probes until the rest of the launcher is up.
88
96
  def run(async_beat: true)
89
97
  @started_at = Time.now.to_f
@@ -92,13 +100,13 @@ module Wurk
92
100
  # CLI, embedded) runs through here, so none boots with a nil fetcher.
93
101
  @config.capsules.each_value(&:prepare!)
94
102
  @config.freeze!
95
- @heartbeat_thread = safe_thread('heartbeat', &method(:start_heartbeat)) if async_beat
96
- [@poller, @leader, @cron_poller, @metrics_rollup, @queue_rollup, @history].compact.each(&:start)
97
103
  @managers.each(&:start)
104
+ @heartbeat_thread = safe_thread('heartbeat', &method(:start_heartbeat)) if async_beat
98
105
  @reaper.start
99
106
  # Run on a background thread so /ready probe isn't delayed by a large
100
107
  # orphan sweep (reaper.reclaim! is atomic, but can scan many entries).
101
108
  @boot_reclaim_thread = safe_thread('boot-reclaim', &method(:boot_reclaim))
109
+ start_periodic_components
102
110
  @health_server&.start
103
111
  rescue StandardError
104
112
  # Boot is not atomic: whatever raised (a health-check port already bound,
@@ -125,20 +133,35 @@ module Wurk
125
133
  fire_event(:quiet, reverse: true)
126
134
  end
127
135
 
128
- # Graceful shutdown. Deadline is monotonic so wall-clock skew can't
129
- # extend it. Managers stop in parallel threads so a slow capsule
130
- # doesn't block its siblings — and each drain is guarded, because a
131
- # capsule that blows up mid-drain (Redis down during bulk_requeue) used
132
- # to surface out of `join` and skip the whole teardown tail, leaving the
136
+ # Graceful shutdown, single-shot: several requests can be in flight at once
137
+ # (a dashboard-queued TERM, the embedded host's own `Embedded#stop`, a
138
+ # Manager that can no longer hold its concurrency), so the first caller
139
+ # drains and the rest wait it out — see ShutdownGate. Deadline is monotonic
140
+ # so wall-clock skew can't extend it. Managers stop in parallel threads so a
141
+ # slow capsule doesn't block its siblings — and each drain is guarded,
142
+ # because a capsule that blows up mid-drain (Redis down during bulk_requeue)
143
+ # used to surface out of `join` and skip the whole teardown tail, leaving the
133
144
  # leader lock held, the process listed as live, and its port open.
145
+ #
146
+ # The joins get one shared budget of `deadline + TimerLoop::JOIN_TIMEOUT`,
147
+ # not an open-ended wait. Manager#stop is meant to run a little past the
148
+ # deadline — on expiry it bulk_requeues the in-flight UnitsOfWork, kills the
149
+ # threads and gives them a ~3s window to run their `ensure` blocks — but
150
+ # past that it is wedged (a `fetcher.terminate` / `bulk_requeue` parked on a
151
+ # Redis that stopped answering), and every further second spent waiting
152
+ # comes out of the teardown tail below, which the swarm parent only allows
153
+ # `shutdown_timeout + Swarm::SHUTDOWN_GRACE` for before it SIGKILLs us
154
+ # mid-drain.
134
155
  def stop
135
- deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + (@config[:timeout] || 25)
136
- quiet
137
- stoppers = @managers.map { |m| Thread.new { teardown_step('manager') { m.stop(deadline) } } }
138
- fire_event(:shutdown, reverse: true)
139
- stoppers.each(&:join)
140
- ensure
141
- release_components
156
+ @shutdown_gate.run do
157
+ deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + (@config[:timeout] || 25)
158
+ quiet
159
+ stoppers = @managers.map { |m| Thread.new { teardown_step('manager') { m.stop(deadline) } } }
160
+ fire_event(:shutdown, reverse: true)
161
+ @shutdown_gate.join_within(stoppers, deadline + TimerLoop::JOIN_TIMEOUT)
162
+ ensure
163
+ release_components
164
+ end
142
165
  end
143
166
 
144
167
  def stopping?
@@ -200,12 +223,14 @@ module Wurk
200
223
  %i[stop_heartbeat clear_heartbeat].each { |step| teardown_step(step) { send(step) } }
201
224
  # Unguarded: fire_event already reports and skips past a raising hook.
202
225
  fire_event(:exit, reverse: true)
203
- # Embedded only: a swarm child or standalone process exits right after
204
- # #stop anyway, so disconnecting here would just make every unit test
205
- # that inspects Redis post-stop rebuild a pool for no reason. Embedded
206
- # hosts (Puma, a rake task) keep running and can `run` again later —
207
- # without this a stop-then-run cycle doubles the live socket set.
208
- teardown_step('redis-pools') { @config.reset_redis_pools! } if @embedded
226
+ # Unconditional. A swarm child or standalone process exits right after
227
+ # #stop, so for those it only closes sockets the kernel was about to close
228
+ # anyway — but every other driver survives us: an embedded host (Puma, a
229
+ # rake task) that boots a fresh Launcher later, and anything driving
230
+ # Launcher directly. Skipping those leaked one main + one fetch pool per
231
+ # capsule, live, for the life of the host. Idempotent: `reset_redis_pools!`
232
+ # drops the memo, so the next checkout rebuilds.
233
+ teardown_step('redis-pools') { @config.reset_redis_pools! }
209
234
  ensure
210
235
  # In an ensure of its own, not merely a guarded step: a leaked TCPServer
211
236
  # FD survives even a non-StandardError unwind (a second TERM landing
@@ -214,11 +239,26 @@ module Wurk
214
239
  teardown_step('health-server') { @health_server&.stop }
215
240
  end
216
241
 
242
+ # Mirror of #stop_periodic_components: every loop this process runs on a
243
+ # timer, started in one place at the tail of #run. `compact` because the
244
+ # history snapshotter is opt-in and an embedded boot may have stripped the
245
+ # rest.
246
+ def start_periodic_components
247
+ [@poller, @leader, @cron_poller, @metrics_rollup,
248
+ @queue_rollup, @metrics_flusher, @history].compact.each(&:start)
249
+ end
250
+
217
251
  # Split out of #release_components to keep it under the AbcSize/
218
- # CyclomaticComplexity ceilings — these three are the "periodic loop"
219
- # releases, independently guarded like everything else in the tail.
252
+ # CyclomaticComplexity ceilings — these are the "periodic loop" releases,
253
+ # independently guarded like everything else in the tail.
254
+ #
255
+ # The metrics flusher goes first: its #terminate drains the last window of
256
+ # per-job counters into the `j|…` minute buckets, and Metrics::Rollup reads
257
+ # exactly those buckets — stopping the rollup first would give its final
258
+ # tick nothing to roll.
220
259
  def stop_periodic_components
221
- [@cron_poller, @metrics_rollup, @queue_rollup, @history].each { |t| teardown_step(t.class) { t&.terminate } }
260
+ [@metrics_flusher, @cron_poller, @metrics_rollup, @queue_rollup,
261
+ @history].each { |t| teardown_step(t.class) { t&.terminate } }
222
262
  teardown_step('reaper') { @reaper&.stop }
223
263
  teardown_step('leader') { @leader&.stop }
224
264
  end
@@ -335,8 +375,15 @@ module Wurk
335
375
  # concurrency. Standalone hands off to the installed TERM trap; embedded
336
376
  # owns no traps, so it drains in place — on its own thread, because callers
337
377
  # may be a thread `stop` itself joins (the heartbeat) or kills (a Processor).
378
+ #
379
+ # That thread is spawned once and retained by the gate: every later request
380
+ # rides the same drain instead of leaving another unreferenced `stop`
381
+ # running behind it. Returned so a caller that can afford to wait (a test,
382
+ # a host teardown) has something to join.
338
383
  def request_shutdown
339
- @embedded ? Thread.new { stop } : redeliver('TERM')
384
+ return redeliver('TERM') unless @embedded
385
+
386
+ @shutdown_gate.runner { safe_thread('launcher-stop') { stop } }
340
387
  end
341
388
 
342
389
  # Separate method so tests can stub it — really sending TERM/TSTP would
@@ -352,49 +399,46 @@ module Wurk
352
399
  @config.capsules.values.map { |cap| Manager.new(cap, shutdown: method(:request_shutdown)) }
353
400
  end
354
401
 
355
- def build_poller
356
- Wurk::Scheduled::Poller.new(@config)
357
- end
358
-
359
- # Periodic (cron) tick loop. Like the scheduler poller, every process runs
360
- # one, but only the elected leader enqueues — the single-leader invariant is
361
- # what guarantees exactly one enqueue per (loop, tick) across the cluster.
362
- def build_cron_poller
363
- Wurk::Cron::Poller.new(@config)
364
- end
365
-
366
- # Leader-only metrics rollup. Every process runs one, but only the elected
367
- # leader writes the cluster-total time-series buckets the dashboard charts
368
- # read — a non-leader tick returns early. Tune the cadence (tests shrink it)
369
- # with `config.metrics_rollup_interval`.
370
- def build_metrics_rollup
371
- Wurk::Metrics::Rollup.new(@config)
372
- end
373
-
374
- # Leader-only per-queue gauge sampler. Like the metrics rollup, every
375
- # process runs one but only the leader writes the `qm|…` size/latency
376
- # buckets the Historical tab's per-queue charts read.
377
- def build_queue_rollup
378
- Wurk::Metrics::QueueRollup.new(@config)
402
+ # The periodic loops, built in one place because they are started together
403
+ # (#run) and released together (#stop_periodic_components). Every process
404
+ # runs every one of them; what differs is who does the work inside a tick:
405
+ #
406
+ # poller / reaper every process, work shared through Redis
407
+ # cron_poller leader only enqueues — that is the exactly-once guard
408
+ # metrics_rollup leader only writes the cluster-total chart buckets
409
+ # (cadence: `config.metrics_rollup_interval`)
410
+ # queue_rollup leader only writes the `qm|…` per-queue gauges
411
+ # metrics_flusher NOT leader-gated: the counters it drains exist only in
412
+ # this process's memory, so every process flushes its own
413
+ # history Ent §5 snapshotter, leader-gated, opt-in via
414
+ # `config.retain_history`
415
+ def build_loops
416
+ @poller = Wurk::Scheduled::Poller.new(@config)
417
+ @cron_poller = Wurk::Cron::Poller.new(@config)
418
+ @metrics_rollup = Wurk::Metrics::Rollup.new(@config)
419
+ @queue_rollup = Wurk::Metrics::QueueRollup.new(@config)
420
+ @metrics_flusher = Wurk::Metrics::Flusher.new(@config)
421
+ @history = Wurk::History.new(@config) if @config.history_enabled?
379
422
  end
380
423
 
381
- # Ent §5 Historical Metrics snapshotter — only when the host opted in via
382
- # `config.retain_history`. Leader-gated like the rollups, so just one
383
- # process emits the cluster-wide snapshot per interval.
384
- def build_history
385
- Wurk::History.new(@config) if @config.history_enabled?
424
+ # Filled in by #run. Declared up front so #initialize reads as the full
425
+ # inventory of what a Launcher owns.
426
+ def reset_thread_state
427
+ @started_at = @heartbeat = @heartbeat_thread = @boot_reclaim_thread = nil
386
428
  end
387
429
 
388
430
  # Every worker process campaigns for the single cluster lock (`dear-leader`);
389
431
  # one wins and renews it, the rest follow and promote on its death. Cadence
390
432
  # falls back to the spec defaults (TTL 30 / renew 15 / follower 60) unless
391
- # the host tunes it. `Leader#start` no-ops under `WURK_LEADER=false`.
433
+ # the host tunes it, plus the short pre-campaign delay that keeps the first
434
+ # CAS off the boot path. `Leader#start` no-ops under `WURK_LEADER=false`.
392
435
  def build_leader
393
436
  Wurk::Leader.new(
394
437
  config: @config,
395
438
  ttl: @config[:leader_ttl] || Wurk::Leader::DEFAULT_TTL,
396
439
  renew_interval: @config[:leader_renew_interval] || Wurk::Leader::DEFAULT_RENEW_INTERVAL,
397
- follower_interval: @config[:leader_follower_interval] || Wurk::Leader::DEFAULT_FOLLOWER_INTERVAL
440
+ follower_interval: @config[:leader_follower_interval] || Wurk::Leader::DEFAULT_FOLLOWER_INTERVAL,
441
+ initial_wait: @config[:leader_initial_wait] || Wurk::Leader::DEFAULT_INITIAL_WAIT
398
442
  )
399
443
  end
400
444
 
data/lib/wurk/leader.rb CHANGED
@@ -5,6 +5,7 @@ require 'socket'
5
5
  require_relative 'component'
6
6
  require_relative 'lua'
7
7
  require_relative 'pool_checkout'
8
+ require_relative 'timer_loop'
8
9
 
9
10
  module Wurk
10
11
  # Cluster leader election via Redis `SET NX EX`. Single-leader-per-cluster
@@ -21,9 +22,12 @@ module Wurk
21
22
  # too — it is never re-read on subsequent acquires, only on transitions.
22
23
  #
23
24
  # Cadence per spec: renew every 15s while leader, recheck every 60s as
24
- # follower, lock TTL 30s. Opt out a process from campaigning entirely
25
- # with `WURK_LEADER=false` (or its Sidekiq alias `SIDEKIQ_LEADER=false`),
26
- # useful for hot-standby pools.
25
+ # follower, lock TTL 30s. The campaign itself starts `initial_wait` seconds
26
+ # after `#start` — the spec fixes the renew/follower/TTL cadence but says
27
+ # nothing about when the first campaign runs, and holding it back keeps a
28
+ # booting process's opening Redis round trips on the job path. Opt out a
29
+ # process from campaigning entirely with `WURK_LEADER=false` (or its Sidekiq
30
+ # alias `SIDEKIQ_LEADER=false`), useful for hot-standby pools.
27
31
  #
28
32
  # Spec: docs/target/sidekiq-ent.md §6.
29
33
  class Leader
@@ -32,6 +36,13 @@ module Wurk
32
36
  DEFAULT_TTL = 30
33
37
  DEFAULT_RENEW_INTERVAL = 15
34
38
  DEFAULT_FOLLOWER_INTERVAL = 60
39
+ # Delay before the campaign's first `SET NX EX`, so a booting process
40
+ # spends its opening Redis round trips on fetching jobs rather than on a
41
+ # leadership CAS. Every leader-gated consumer (cron, both rollups,
42
+ # history) waits out a 30-60s interval before its own first tick, so
43
+ # nothing observes the gap. Tune with `config[:leader_initial_wait]`;
44
+ # 0 campaigns immediately.
45
+ DEFAULT_INITIAL_WAIT = 1
35
46
  OPT_OUT_ENV = 'WURK_LEADER' # native opt-out env
36
47
  SIDEKIQ_OPT_OUT_ENV = 'SIDEKIQ_LEADER' # Sidekiq Ent drop-in alias (§6.2/§7.2)
37
48
  THREAD_NAME = 'wurk-leader'
@@ -48,12 +59,14 @@ module Wurk
48
59
  def initialize(config: nil, key: DEFAULT_KEY, ttl: DEFAULT_TTL, # rubocop:disable Metrics/ParameterLists
49
60
  renew_interval: DEFAULT_RENEW_INTERVAL,
50
61
  follower_interval: DEFAULT_FOLLOWER_INTERVAL,
62
+ initial_wait: DEFAULT_INITIAL_WAIT,
51
63
  pool: nil, owner: nil)
52
64
  @config = config
53
65
  @key = key
54
66
  @ttl = ttl
55
67
  @renew_interval = renew_interval
56
68
  @follower_interval = follower_interval
69
+ @initial_wait = initial_wait.to_f
57
70
  @pool = pool
58
71
  @owner = owner || cluster_identity
59
72
  @held = false
@@ -136,14 +149,20 @@ module Wurk
136
149
  @thread
137
150
  end
138
151
 
152
+ # Bounded at TimerLoop::JOIN_TIMEOUT like every other periodic component: a
153
+ # tick parked on a wedged Redis (the campaign is a `SET NX EX`, so it waits
154
+ # out the socket timeouts) must not hold the whole process's teardown open —
155
+ # the swarm parent SIGKILLs a child that overruns its shutdown grace. The
156
+ # CAS release below runs either way; the loop's own trailing release makes
157
+ # a straggler's second attempt a no-op. A straggler also stays referenced,
158
+ # so a restart after a timed-out stop can't campaign twice in parallel.
139
159
  def stop
140
160
  thread = @mutex.synchronize do
141
161
  @done = true
142
162
  @sleeper.signal
143
163
  @thread
144
164
  end
145
- thread&.join
146
- @mutex.synchronize { @thread = nil }
165
+ @mutex.synchronize { @thread = nil } if thread.nil? || thread.join(TimerLoop::JOIN_TIMEOUT)
147
166
  release
148
167
  end
149
168
 
@@ -224,6 +243,7 @@ module Wurk
224
243
  end
225
244
 
226
245
  def run_loop
246
+ wait_initial
227
247
  until done?
228
248
  tick_once
229
249
  wait_next
@@ -231,6 +251,13 @@ module Wurk
231
251
  release
232
252
  end
233
253
 
254
+ # Same condvar as #wait_next, so a `stop` landing inside the pre-campaign
255
+ # delay wakes the loop immediately instead of holding shutdown open for a
256
+ # whole initial_wait.
257
+ def wait_initial
258
+ wait_for(@initial_wait) if @initial_wait.positive?
259
+ end
260
+
234
261
  def tick_once
235
262
  acquire
236
263
  rescue StandardError => e
@@ -238,7 +265,10 @@ module Wurk
238
265
  end
239
266
 
240
267
  def wait_next
241
- interval = @held ? @renew_interval : @follower_interval
268
+ wait_for(@held ? @renew_interval : @follower_interval)
269
+ end
270
+
271
+ def wait_for(interval)
242
272
  @mutex.synchronize { @sleeper.wait(@mutex, interval) unless @done }
243
273
  end
244
274
 
data/lib/wurk/logger.rb CHANGED
@@ -28,7 +28,7 @@ module Wurk
28
28
  }.freeze
29
29
 
30
30
  def tid
31
- Thread.current[:wurk_tid] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
31
+ Wurk::Component.tid
32
32
  end
33
33
 
34
34
  def format_context(ctxt = Wurk::Context.current)
@@ -23,9 +23,15 @@ module Wurk
23
23
  # connection errors are the pool wrapper's job (Wurk::RedisPool#with);
24
24
  # this only ships the loads.
25
25
  def script_load_all(redis)
26
- redis.pipelined do |pipe|
27
- SCRIPTS.each_value { |src| pipe.call('SCRIPT', 'LOAD', src) }
28
- end
26
+ redis.pipelined { |pipe| queue_script_loads(pipe) }
27
+ end
28
+
29
+ # The same loads, queued onto a pipeline the caller already owns, for a
30
+ # caller with other work to batch with them: ChildBoot pairs them with
31
+ # its liveness PING so a child's whole Redis validation costs one round
32
+ # trip instead of two, on the boot-critical path.
33
+ def queue_script_loads(pipe)
34
+ SCRIPTS.each_value { |src| pipe.call('SCRIPT', 'LOAD', src) }
29
35
  end
30
36
 
31
37
  # @param redis [RedisClient] a single connection (not a pool)
@@ -0,0 +1,95 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Wurk
4
+ module Metrics
5
+ # The in-memory half of Wurk::Metrics::History. Every execution folds into a
6
+ # `{pool => {minute => {class => [processed, failed, ms]}}}` tree under one
7
+ # mutex; Wurk::Metrics::Flusher drains it into Redis on a timer. `HINCRBY`
8
+ # is additive, so N folded executions leave Redis in the same state as the N
9
+ # individual pipelines the hot path used to send.
10
+ #
11
+ # Knows nothing about Redis — History owns the write, this owns the counts.
12
+ #
13
+ # The minute is the bucket the job *ran* in, decided by the caller at record
14
+ # time. That is what keeps the batching a visibility lag instead of a
15
+ # misattribution: a job that runs at 12:03:59 and flushes at 12:04:02 still
16
+ # counts toward 12:03. Signed off in
17
+ # docs/plans/2026/08/06/101-faster-than-sidekiq/00-semantics-signoff.md §2.
18
+ #
19
+ # Keyed by the pool the recording middleware handed us (nil = the process
20
+ # default), so a multi-capsule process flushes each capsule's counts through
21
+ # the pool that capsule's jobs ran on.
22
+ class Accumulator
23
+ # Ceiling on minute buckets retained per pool. Only reachable through
24
+ # #merge_back: a flush that keeps failing puts its counts back every tick,
25
+ # and a Redis outage that outlives the process would otherwise grow a
26
+ # structure fed by the job hot path without bound. The newest buckets win
27
+ # — they are the ones the dashboard is still drawing — and the oldest are
28
+ # dropped, which these counters have always been allowed to do (a Redis
29
+ # failure during a metrics write has never changed a job's outcome).
30
+ MAX_RETAINED_MINUTES = 60
31
+
32
+ def initialize
33
+ @lock = ::Mutex.new
34
+ @pools = {}
35
+ end
36
+
37
+ # Hot path. Three hash lookups under the mutex; allocates only the first
38
+ # time a (pool, minute, class) triple is seen.
39
+ def add(pool, klass, minute, ms, success)
40
+ @lock.synchronize do
41
+ counts = (((@pools[pool] ||= {})[minute] ||= {})[klass] ||= [0, 0, 0])
42
+ counts[success ? 0 : 1] += 1
43
+ counts[2] += ms
44
+ end
45
+ end
46
+
47
+ # Hands the whole tree over and starts a fresh one, so recording never
48
+ # blocks behind the flush's Redis round trip.
49
+ def drain
50
+ @lock.synchronize do
51
+ drained = @pools
52
+ @pools = {}
53
+ drained
54
+ end
55
+ end
56
+
57
+ # Puts one pool's counts back after its write failed, so the next tick
58
+ # retries them instead of dropping the window. Merged rather than
59
+ # assigned: jobs kept recording into the fresh tree while the flush was in
60
+ # flight, and those counts have not been written yet either.
61
+ def merge_back(pool, minutes)
62
+ @lock.synchronize do
63
+ into = (@pools[pool] ||= {})
64
+ minutes.each { |minute, classes| merge_minute(into, minute, classes) }
65
+ trim(into)
66
+ end
67
+ end
68
+
69
+ def empty?
70
+ @lock.synchronize { @pools.empty? }
71
+ end
72
+
73
+ private
74
+
75
+ def merge_minute(into, minute, classes)
76
+ target = (into[minute] ||= {})
77
+ classes.each do |klass, counts|
78
+ existing = target[klass]
79
+ if existing
80
+ counts.each_with_index { |count, i| existing[i] += count }
81
+ else
82
+ target[klass] = counts
83
+ end
84
+ end
85
+ end
86
+
87
+ def trim(minutes)
88
+ excess = minutes.size - MAX_RETAINED_MINUTES
89
+ return if excess <= 0
90
+
91
+ minutes.keys.min(excess).each { |minute| minutes.delete(minute) }
92
+ end
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,70 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../component'
4
+ require_relative '../timer_loop'
5
+ require_relative 'history'
6
+
7
+ module Wurk
8
+ module Metrics
9
+ # Drains Wurk::Metrics::History's in-process accumulator into Redis every
10
+ # `History::FLUSH_INTERVAL` seconds.
11
+ #
12
+ # Deliberately NOT leader-gated, unlike every other periodic component here
13
+ # (Metrics::Rollup, Metrics::QueueRollup, Wurk::History). Those publish
14
+ # *cluster* state that one process should own; this publishes counters that
15
+ # only exist inside this process's memory and that no other process can
16
+ # write. A leader gate would strand every follower's metrics until it died.
17
+ #
18
+ # One tick per process, not per capsule: the accumulator is process-wide and
19
+ # already keyed by the pool each capsule's jobs recorded through.
20
+ class Flusher
21
+ include Component
22
+
23
+ # `accumulator` is a collaborator, not an option: every real boot flushes
24
+ # the process-wide one. Tests inject their own so a broken pool stays
25
+ # inside the test that built it.
26
+ def initialize(config, accumulator: History::ACCUMULATOR)
27
+ @config = config
28
+ @accumulator = accumulator
29
+ @timer = TimerLoop.new(History::FLUSH_INTERVAL)
30
+ @thread = nil
31
+ end
32
+
33
+ def start
34
+ return @thread if @thread
35
+
36
+ @timer.reset
37
+ @thread = safe_thread('metrics-flush') { @timer.run { tick } }
38
+ end
39
+
40
+ # Flushes once more in an `ensure`: this stops the only thread that would
41
+ # have written the window accumulated since the last tick, so without it a
42
+ # graceful shutdown would drop up to FLUSH_INTERVAL of counters — the cost
43
+ # the sign-off accepts for a *hard* kill only.
44
+ #
45
+ # Launcher#stop joins the manager drains before the teardown tail reaches
46
+ # us, so by the time this runs no job is still recording.
47
+ #
48
+ # Cleared only on a confirmed join (Thread#join returns nil on timeout): a
49
+ # wedged thread must stay tracked so #start's guard returns it instead of
50
+ # calling @timer.reset, which would un-terminate the loop it is still
51
+ # inside and leave two threads draining the same accumulator.
52
+ def terminate
53
+ @timer.terminate
54
+ @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
55
+ ensure
56
+ tick
57
+ end
58
+
59
+ # Never raises: this runs on a `safe_thread`, whose watchdog re-raises
60
+ # after reporting, and a reported-then-dead flusher would silently stop
61
+ # every counter in the process for a blip Redis recovers from. The
62
+ # accumulator has already merged the failed window back for the next tick.
63
+ def tick
64
+ History.flush(@accumulator)
65
+ rescue StandardError => e
66
+ handle_exception(e, { context: 'metrics-flush' })
67
+ end
68
+ end
69
+ end
70
+ end