wurk 1.3.1 → 1.5.0

data/lib/wurk/job_logger.rb

@@ -41,13 +41,7 @@ module Wurk
     # under a per-job log level. ActiveJob-wrapped jobs expose the real
     # class via the "wrapped" key — log that, not the wrapper.
     def prepare(job_hash, &block)
-      h = {
-        jid: job_hash['jid'],
-        class: job_hash['wrapped'] || job_hash['class']
-      }
-      @config[:logged_job_attributes].each do |attr|
-        h[attr.to_sym] = job_hash[attr] if job_hash.key?(attr)
-      end
+      h = context_hash(job_hash)
 
       level = job_hash['log_level']
       Wurk::Context.with(h) do
@@ -64,5 +58,20 @@ module Wurk
     def elapsed(start)
       (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
     end
+
+    # The attribute list is a host setting (defaults to %w[bid tags]), so the
+    # walk cannot be short-circuited on the default keys: a host that adds
+    # `tenant_id` logs jobs that carry neither `bid` nor `tags`.
+    def context_hash(job_hash)
+      h = {
+        jid: job_hash['jid'],
+        class: job_hash['wrapped'] || job_hash['class']
+      }
+
+      @config[:logged_job_attributes].each do |attr|
+        h[attr.to_sym] = job_hash[attr] if job_hash.key?(attr)
+      end
+      h
+    end
   end
 end

data/lib/wurk/job_set.rb

@@ -65,8 +65,9 @@ module Wurk
   #
   # Spec: docs/target/sidekiq-free.md §19.5.
   class JobSet < SortedSet
-    # ZADD with NX so re-scheduling the same payload doesn't reset its score.
-    # Mirrors Sidekiq::JobSet#schedule exactly.
+    # Plain ZADD, no NX: re-scheduling an identical payload *moves* it to the
+    # new timestamp rather than keeping the first one. Mirrors
+    # Sidekiq::JobSet#schedule exactly — the last schedule call wins.
     def schedule(timestamp, message)
       Wurk.redis { |conn| conn.call('ZADD', @name, timestamp.to_f.to_s, Wurk.dump_json(message)) }
     end

data/lib/wurk/job_util.rb

@@ -140,32 +140,52 @@ module Wurk
       end
     end
 
+    JSON_NATIVE = ->(_val) {}
+
+    # Dispatch table for the args walk, ported from Sidekiq (job_util.rb:80-111).
+    # Keyed by `val.class` under `compare_by_identity`, so every node costs one
+    # identity lookup + one call instead of the ladder of `Module#===` checks a
+    # `case/when` compiles to — the entry node is always the args Array, which
+    # sat behind six failing `===` probes.
+    #
+    # Identity lookup means a *subclass* of a JSON-native type misses the table
+    # and falls to the default lambda, i.e. is reported unsafe: an
+    # ActiveSupport::SafeBuffer arg raises where a bare String does not. That is
+    # Sidekiq's behavior and the drop-in contract, not an oversight. Hash keys
+    # are the one exception — they are checked with `is_a?`, so a String
+    # subclass key is accepted, again matching Sidekiq.
+    RECURSIVE_JSON_UNSAFE = { # rubocop:disable Style/MutableConstant -- frozen below, after the two setup calls
+      Integer => JSON_NATIVE,
+      Float => JSON_NATIVE,
+      TrueClass => JSON_NATIVE,
+      FalseClass => JSON_NATIVE,
+      NilClass => JSON_NATIVE,
+      String => JSON_NATIVE,
+      Array => lambda { |val|
+        val.each do |e|
+          bad = RECURSIVE_JSON_UNSAFE[e.class].call(e)
+          return bad unless bad.nil?
+        end
+        nil
+      },
+      Hash => lambda { |val|
+        val.each do |k, v|
+          return k unless k.is_a?(String)
+
+          bad = RECURSIVE_JSON_UNSAFE[v.class].call(v)
+          return bad unless bad.nil?
+        end
+        nil
+      }
+    }
+    RECURSIVE_JSON_UNSAFE.default = ->(val) { val }
+    RECURSIVE_JSON_UNSAFE.compare_by_identity
+    RECURSIVE_JSON_UNSAFE.freeze
+    private_constant :JSON_NATIVE, :RECURSIVE_JSON_UNSAFE
+
     # Returns the first offending value, or nil when the tree is JSON-native.
     def json_unsafe(obj)
-      case obj
-      when String, Integer, Float, TrueClass, FalseClass, NilClass then nil
-      when Array then json_unsafe_array(obj)
-      when Hash then json_unsafe_hash(obj)
-      else obj
-      end
-    end
-
-    def json_unsafe_array(arr)
-      arr.each do |v|
-        bad = json_unsafe(v)
-        return bad unless bad.nil?
-      end
-      nil
-    end
-
-    def json_unsafe_hash(hash)
-      hash.each do |k, v|
-        return k unless k.is_a?(String)
-
-        bad = json_unsafe(v)
-        return bad unless bad.nil?
-      end
-      nil
+      RECURSIVE_JSON_UNSAFE[obj.class].call(obj)
     end
   end
 end

data/lib/wurk/launcher.rb

@@ -11,8 +11,11 @@ require_relative 'leader'
 require_relative 'cron'
 require_relative 'metrics/rollup'
 require_relative 'metrics/queue_rollup'
+require_relative 'metrics/flusher'
 require_relative 'history'
 require_relative 'fetcher/reaper'
+require_relative 'timer_loop'
+require_relative 'shutdown_gate'
 
 module Wurk
   # Top-level supervisor inside each worker process. Owns the Manager pool
@@ -43,41 +46,52 @@ module Wurk
     # (Sidekiq's drop-in surface). The single source of truth is Heartbeat.
     BEAT_PAUSE = Heartbeat::BEAT_PAUSE
 
-    attr_accessor :managers, :poller, :cron_poller, :metrics_rollup, :queue_rollup, :history
+    # Bound on how long #stop waits for the boot-time reclaim sweep before
+    # moving on — it can still be scanning a large keyspace when a fast
+    # shutdown lands right after boot; teardown must not hang on it.
+    BOOT_RECLAIM_JOIN_TIMEOUT = 5
+
+    attr_accessor :managers, :poller, :cron_poller, :metrics_rollup, :queue_rollup, :metrics_flusher, :history
 
     def initialize(config, embedded: false)
       @config = config
       @embedded = embedded
-      # Two separate flags, deliberately. @done = "quieted" (stop fetching, stay
-      # alive, report quiet=true). @stopped = "shutting down" (terminate the
-      # heartbeat loop). Quiet must NOT stop the heartbeat — otherwise a quieted
-      # process never publishes quiet=true and expires out of the live set (#236).
+      # @done is "quieted": stop fetching, stay alive, report quiet=true. It
+      # deliberately does NOT stop the heartbeat — a quieted process that stopped
+      # beating would never publish quiet=true and would expire out of the live
+      # set (#236). Only #stop ends the beat, by terminating @beat_timer.
       @done = false
-      @stopped = false
-      @managers = config.capsules.values.map { |cap| Manager.new(cap) }
-      @poller = build_poller
-      @cron_poller = build_cron_poller
-      @metrics_rollup = build_metrics_rollup
-      @queue_rollup = build_queue_rollup
-      @history = build_history
+      @shutdown_gate = ShutdownGate.new
+      @beat_timer = TimerLoop.new(BEAT_PAUSE)
+      @managers = build_managers
+      build_loops
       @leader = build_leader
       @reaper = build_reaper
-      @started_at = nil
-      @heartbeat = nil
-      @heartbeat_thread = nil
-      @boot_reclaim_thread = nil
       @health_server = build_health_server
+      reset_thread_state
     end
 
     # Boot order matters:
     #   1. freeze! the config so mutations after fork are visible mistakes.
-    #   2. spawn the heartbeat thread BEFORE the managers so the dashboard
-    #      sees the process the moment it can pick up jobs.
-    #   3. start the scheduler poller + the cron poller (both leader-gated for
-    #      what they enqueue; safe to start before leadership is settled since
-    #      a non-leader tick just returns early).
-    #   4. start the managers (which start their processors).
-    #   5. start the health probe server LAST so the listener doesn't
+    #      A swarm child finds the slot-independent half already frozen by its
+    #      parent (Configuration#prepare_for_fork!); what closes here is the
+    #      capsules, which stayed open for this child's slot and pools.
+    #   2. start the managers FIRST. They are the only thing here on the
+    #      time-to-first-job path (the number bench/vs_sidekiq.rb measures);
+    #      everything below is periodic background work whose first tick is
+    #      seconds away, so it has nothing to gain from going ahead of them.
+    #   3. spawn the heartbeat thread, so the dashboard sees the process
+    #      right after it can pick up jobs.
+    #   4. start the reaper and the boot-time reclaim sweep — kill-9 recovery,
+    #      not decoration: a SIGKILLed sibling's in-flight jobs wait on this,
+    #      so it stays ahead of the periodic loops.
+    #   5. start the periodic loops. None of them ticks at zero — TimerLoop#run
+    #      waits an interval before its first yield, Scheduled::Poller waits
+    #      INITIAL_WAIT, and Leader waits DEFAULT_INITIAL_WAIT — so starting
+    #      them last costs a few thread spawns and nothing else. Both leader-
+    #      gated pollers are also safe to start before leadership settles: a
+    #      non-leader tick just returns early.
+    #   6. start the health probe server LAST so the listener doesn't
     #      accept k8s probes until the rest of the launcher is up.
     def run(async_beat: true)
       @started_at = Time.now.to_f
@@ -86,19 +100,22 @@ module Wurk
       # CLI, embedded) runs through here, so none boots with a nil fetcher.
       @config.capsules.each_value(&:prepare!)
       @config.freeze!
-      @heartbeat_thread = safe_thread('heartbeat', &method(:start_heartbeat)) if async_beat
-      @poller&.start
-      @leader&.start
-      @cron_poller&.start
-      @metrics_rollup&.start
-      @queue_rollup&.start
-      @history&.start
       @managers.each(&:start)
+      @heartbeat_thread = safe_thread('heartbeat', &method(:start_heartbeat)) if async_beat
       @reaper.start
       # Run on a background thread so /ready probe isn't delayed by a large
       # orphan sweep (reaper.reclaim! is atomic, but can scan many entries).
       @boot_reclaim_thread = safe_thread('boot-reclaim', &method(:boot_reclaim))
+      start_periodic_components
       @health_server&.start
+    rescue StandardError
+      # Boot is not atomic: whatever raised (a health-check port already bound,
+      # ThreadError at the OS thread limit) leaves the steps before it holding
+      # threads, sockets and a leader campaign — and the caller is about to drop
+      # its only reference to us, so nothing else can ever release them. Guarded,
+      # because the caller must see the boot failure, not a rollback failure.
+      teardown_step('boot-rollback') { stop }
+      raise
     end
 
     # Idempotent. Flips `stopping?` true, halts fetching across every
@@ -116,28 +133,35 @@ module Wurk
       fire_event(:quiet, reverse: true)
     end
 
-    # Graceful shutdown. Deadline is monotonic so wall-clock skew can't
-    # extend it. Managers stop in parallel threads so a slow capsule
-    # doesn't block its siblings.
+    # Graceful shutdown, single-shot: several requests can be in flight at once
+    # (a dashboard-queued TERM, the embedded host's own `Embedded#stop`, a
+    # Manager that can no longer hold its concurrency), so the first caller
+    # drains and the rest wait it out — see ShutdownGate. Deadline is monotonic
+    # so wall-clock skew can't extend it. Managers stop in parallel threads so a
+    # slow capsule doesn't block its siblings — and each drain is guarded,
+    # because a capsule that blows up mid-drain (Redis down during bulk_requeue)
+    # used to surface out of `join` and skip the whole teardown tail, leaving the
+    # leader lock held, the process listed as live, and its port open.
+    #
+    # The joins get one shared budget of `deadline + TimerLoop::JOIN_TIMEOUT`,
+    # not an open-ended wait. Manager#stop is meant to run a little past the
+    # deadline — on expiry it bulk_requeues the in-flight UnitsOfWork, kills the
+    # threads and gives them a ~3s window to run their `ensure` blocks — but
+    # past that it is wedged (a `fetcher.terminate` / `bulk_requeue` parked on a
+    # Redis that stopped answering), and every further second spent waiting
+    # comes out of the teardown tail below, which the swarm parent only allows
+    # `shutdown_timeout + Swarm::SHUTDOWN_GRACE` for before it SIGKILLs us
+    # mid-drain.
     def stop
-      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + (@config[:timeout] || 25)
-      quiet
-      stoppers = @managers.map { |m| Thread.new { m.stop(deadline) } }
-      fire_event(:shutdown, reverse: true)
-      stoppers.each(&:join)
-      # Full shutdown stops periodic firing (it survived #quiet); do this before
-      # releasing the lock so no tick races a follower's promotion.
-      @cron_poller&.terminate
-      @metrics_rollup&.terminate
-      @queue_rollup&.terminate
-      @history&.terminate
-      @reaper&.stop
-      # CAS-release the cluster lock now (planned shutdown) so a follower can
-      # take over immediately instead of waiting out the TTL.
-      @leader&.stop
-      stop_heartbeat
-      clear_heartbeat
-      fire_event(:exit, reverse: true)
+      @shutdown_gate.run do
+        deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + (@config[:timeout] || 25)
+        quiet
+        stoppers = @managers.map { |m| Thread.new { teardown_step('manager') { m.stop(deadline) } } }
+        fire_event(:shutdown, reverse: true)
+        @shutdown_gate.join_within(stoppers, deadline + TimerLoop::JOIN_TIMEOUT)
+      ensure
+        release_components
+      end
     end
 
     def stopping?
@@ -162,9 +186,12 @@ module Wurk
 
       write_stats(processed, failed, expired)
     rescue StandardError => e
-      # Replay-safety: counters were reset above, so a Redis blip would
-      # otherwise drop stats. We log and accept — the per-job at-least-once
-      # semantics don't apply to *counters*, and the next beat resets again.
+      # The counters were reset above, so a raise here drops this window's stats
+      # for good: #write_stats cannot claim apply-safety and the pool no longer
+      # replays it through a blip. Accepted deliberately — adding them back
+      # would double-count the case where the INCRBYs did land and only the
+      # reply was lost, and the per-job at-least-once semantics don't apply to
+      # *counters*. The next beat resets again.
       handle_exception(e, { context: 'flush_stats' })
     end
 
@@ -174,6 +201,77 @@ module Wurk
 
     private
 
+    # Teardown tail, driven from #stop's ensure. Every release is guarded on
+    # its own because they are independent: a Redis blip in one (leader CAS
+    # release, heartbeat clear) must not strand the ones after it — the process
+    # is exiting either way, so the only thing worse than a failed release is a
+    # skipped one. Order matters twice over: full shutdown stops the periodic
+    # loops (they survived #quiet) before the cluster lock is CAS-released, so
+    # no tick races a follower's promotion, and the release itself happens on a
+    # planned shutdown rather than waiting out the lock TTL.
+    def release_components
+      # Joined first, before anything below tears down the Redis pool it's
+      # still reading from: it started as a fire-and-forget thread in #run
+      # and a shutdown landing right after boot could otherwise race a
+      # pool disconnect mid-scan. Bounded: a full-keyspace scan can outlast
+      # the timeout, in which case the thread is left running rather than
+      # blocking shutdown on it — #boot_reclaim already rescues and logs on
+      # its own, so a straggler racing #reset_redis_pools! below just means
+      # a noisy log line, not a hang.
+      teardown_step('boot-reclaim-join') { @boot_reclaim_thread&.join(BOOT_RECLAIM_JOIN_TIMEOUT) }
+      stop_periodic_components
+      %i[stop_heartbeat clear_heartbeat].each { |step| teardown_step(step) { send(step) } }
+      # Unguarded: fire_event already reports and skips past a raising hook.
+      fire_event(:exit, reverse: true)
+      # Unconditional. A swarm child or standalone process exits right after
+      # #stop, so for those it only closes sockets the kernel was about to close
+      # anyway — but every other driver survives us: an embedded host (Puma, a
+      # rake task) that boots a fresh Launcher later, and anything driving
+      # Launcher directly. Skipping those leaked one main + one fetch pool per
+      # capsule, live, for the life of the host. Idempotent: `reset_redis_pools!`
+      # drops the memo, so the next checkout rebuilds.
+      teardown_step('redis-pools') { @config.reset_redis_pools! }
+    ensure
+      # In an ensure of its own, not merely a guarded step: a leaked TCPServer
+      # FD survives even a non-StandardError unwind (a second TERM landing
+      # mid-teardown), and kubelet would keep getting 200s from a process that
+      # is already gone.
+      teardown_step('health-server') { @health_server&.stop }
+    end
+
+    # Mirror of #stop_periodic_components: every loop this process runs on a
+    # timer, started in one place at the tail of #run. `compact` because the
+    # history snapshotter is opt-in and an embedded boot may have stripped the
+    # rest.
+    def start_periodic_components
+      [@poller, @leader, @cron_poller, @metrics_rollup,
+       @queue_rollup, @metrics_flusher, @history].compact.each(&:start)
+    end
+
+    # Split out of #release_components to keep it under the AbcSize/
+    # CyclomaticComplexity ceilings — these are the "periodic loop" releases,
+    # independently guarded like everything else in the tail.
+    #
+    # The metrics flusher goes first: its #terminate drains the last window of
+    # per-job counters into the `j|…` minute buckets, and Metrics::Rollup reads
+    # exactly those buckets — stopping the rollup first would give its final
+    # tick nothing to roll.
+    def stop_periodic_components
+      [@metrics_flusher, @cron_poller, @metrics_rollup, @queue_rollup,
+       @history].each { |t| teardown_step(t.class) { t&.terminate } }
+      teardown_step('reaper') { @reaper&.stop }
+      teardown_step('leader') { @leader&.stop }
+    end
+
+    def teardown_step(label)
+      yield
+    rescue StandardError => e
+      handle_exception(e, { context: "launcher-stop-#{label}" })
+    end
+
+    # INCRBY is additive, so a pipeline replayed after a lost reply counts this
+    # window twice on every `stat:` key it touches — no apply-safety claim. (The
+    # EXPIREs riding along are harmless to repeat; the INCRBYs pin the block.)
     def write_stats(processed, failed, expired)
       day = Time.now.utc.strftime('%F')
       @config.redis do |conn|
@@ -217,117 +315,130 @@ module Wurk
 
     # Erase the live-process footprint. flush_stats first so we don't drop
     # the final batch of counters; then Heartbeat#stop! removes us from the
-    # `processes` SET and UNLINK-s the identity + work hashes. The probe
-    # server is closed alongside so kubelet stops getting 200s after the
-    # process is no longer healthy.
+    # `processes` SET and UNLINK-s the identity + work hashes.
     def clear_heartbeat
       flush_stats
       @heartbeat&.stop!
-      @health_server&.stop
     end
 
     # Terminate the heartbeat loop and wait for it to exit before clear_heartbeat
     # removes us from the `processes` SET — otherwise a final in-flight beat could
-    # SADD us back right after the SREM. Wakes the thread out of its BEAT_PAUSE
-    # sleep so shutdown isn't delayed up to a full interval.
+    # SADD us back right after the SREM and the identity would linger for a full
+    # 60s TTL.
+    #
+    # The join is unbounded on purpose. This used to be `wakeup` + `join(BEAT_PAUSE)`,
+    # which lost the race whenever the beat was mid-Redis-call: `wakeup` does nothing
+    # to a thread that isn't sleeping, so the loop then slept a full interval and the
+    # bounded join returned with the thread still live — exactly the resurrect-after-
+    # SREM this ordering exists to prevent. A condvar can't be missed (terminate flips
+    # the flag inside the critical section the loop re-checks it in), so all that is
+    # left to wait out is one in-flight beat, whose Redis calls are timeout-bounded.
     def stop_heartbeat
-      @stopped = true
+      @beat_timer.terminate
       thread = @heartbeat_thread
       return unless thread
-      # Embedded dashboard-TERM runs `stop` from the beat itself; a self-join
-      # raises ThreadError. @stopped is set, so the loop exits after this beat.
+      # Embedded dashboard-TERM can run `stop` from the beat itself; a self-join
+      # raises ThreadError. The timer is terminated, so the loop exits after this beat.
       return if thread == Thread.current
 
-      begin
-        thread.wakeup
-      rescue ThreadError
-        nil
-      end
-      thread.join(BEAT_PAUSE)
+      thread.join
     end
 
-    # Heartbeat thread loop. `safe_thread` already wraps exceptions. Loops on
-    # `@stopped` — NOT `@done` — so a *quieted* process keeps beating and publishes
-    # `quiet=true` instead of vanishing from the live set (#236). Only `#stop`
-    # flips `@stopped`; its `Thread#wakeup` breaks the sleep so the loop re-checks
-    # `@stopped` and exits without waiting out the interval.
+    # Heartbeat thread loop. `safe_thread` already wraps exceptions. Beats once up
+    # front — TimerLoop#run waits before its first yield, and the dashboard has to
+    # see this process the moment it can pick up jobs — then ticks until #stop
+    # terminates the timer. Note it does NOT stop on `@done`: a *quieted* process
+    # keeps beating so it publishes `quiet=true` instead of vanishing (#236).
     def start_heartbeat
-      until @stopped
-        heartbeat
-        sleep BEAT_PAUSE
-      end
+      heartbeat
+      @beat_timer.run { heartbeat }
       logger.info('Heartbeat stopping...')
     end
 
     # Dashboard-queued signals must behave exactly like OS signals, so a
-    # standalone process re-delivers to itself and lets the installed trap
-    # run — that wakes the main thread (CLI self-pipe / child dispatcher)
-    # so the process actually exits instead of stopping its managers and
-    # then parking forever. Embedded mode owns no traps (and self-TERM
-    # would kill the host app), so it calls quiet/stop directly — stop on
-    # its own thread because `stop` joins the heartbeat thread we're on.
+    # standalone process re-delivers to itself and lets the installed trap run —
+    # that wakes the main thread (CLI self-pipe / child dispatcher) so the
+    # process actually exits instead of stopping its managers and then parking
+    # forever. Embedded mode owns no traps (and self-TERM would kill the host
+    # app), so quiet applies directly.
     def dispatch_signal(sig)
       case sig
-      when 'TSTP', 'TERM'
-        if @embedded
-          sig == 'TSTP' ? quiet : Thread.new { stop }
-        else
-          redeliver(sig)
-        end
+      when 'TSTP' then @embedded ? quiet : redeliver(sig)
+      when 'TERM' then request_shutdown
       else
         logger.warn { "Unknown signal in #{identity}-signals: #{sig.inspect}" }
       end
     end
 
+    # The one way anything inside this process asks it to shut down gracefully:
+    # dashboard-queued TERM, or a Manager that can no longer hold its
+    # concurrency. Standalone hands off to the installed TERM trap; embedded
+    # owns no traps, so it drains in place — on its own thread, because callers
+    # may be a thread `stop` itself joins (the heartbeat) or kills (a Processor).
+    #
+    # That thread is spawned once and retained by the gate: every later request
+    # rides the same drain instead of leaving another unreferenced `stop`
+    # running behind it. Returned so a caller that can afford to wait (a test,
+    # a host teardown) has something to join.
+    def request_shutdown
+      return redeliver('TERM') unless @embedded
+
+      @shutdown_gate.runner { safe_thread('launcher-stop') { stop } }
+    end
+
     # Separate method so tests can stub it — really sending TERM/TSTP would
     # kill or suspend the test process.
     def redeliver(sig)
       ::Process.kill(sig, ::Process.pid)
     end
 
-    def build_poller
-      Wurk::Scheduled::Poller.new(@config)
-    end
-
-    # Periodic (cron) tick loop. Like the scheduler poller, every process runs
-    # one, but only the elected leader enqueues — the single-leader invariant is
-    # what guarantees exactly one enqueue per (loop, tick) across the cluster.
-    def build_cron_poller
-      Wurk::Cron::Poller.new(@config)
-    end
-
-    # Leader-only metrics rollup. Every process runs one, but only the elected
-    # leader writes the cluster-total time-series buckets the dashboard charts
-    # read — a non-leader tick returns early. Tune the cadence (tests shrink it)
-    # with `config.metrics_rollup_interval`.
-    def build_metrics_rollup
-      Wurk::Metrics::Rollup.new(@config)
+    # One Manager per capsule, each holding our shutdown request: a Manager
+    # that can no longer replace a dead Processor has to take the process
+    # down, and only the Launcher knows how this process exits.
+    def build_managers
+      @config.capsules.values.map { |cap| Manager.new(cap, shutdown: method(:request_shutdown)) }
     end
 
-    # Leader-only per-queue gauge sampler. Like the metrics rollup, every
-    # process runs one but only the leader writes the `qm|…` size/latency
-    # buckets the Historical tab's per-queue charts read.
-    def build_queue_rollup
-      Wurk::Metrics::QueueRollup.new(@config)
+    # The periodic loops, built in one place because they are started together
+    # (#run) and released together (#stop_periodic_components). Every process
+    # runs every one of them; what differs is who does the work inside a tick:
+    #
+    #   poller / reaper   every process, work shared through Redis
+    #   cron_poller       leader only enqueues — that is the exactly-once guard
+    #   metrics_rollup    leader only writes the cluster-total chart buckets
+    #                     (cadence: `config.metrics_rollup_interval`)
+    #   queue_rollup      leader only writes the `qm|…` per-queue gauges
+    #   metrics_flusher   NOT leader-gated: the counters it drains exist only in
+    #                     this process's memory, so every process flushes its own
+    #   history           Ent §5 snapshotter, leader-gated, opt-in via
+    #                     `config.retain_history`
+    def build_loops
+      @poller = Wurk::Scheduled::Poller.new(@config)
+      @cron_poller = Wurk::Cron::Poller.new(@config)
+      @metrics_rollup = Wurk::Metrics::Rollup.new(@config)
+      @queue_rollup = Wurk::Metrics::QueueRollup.new(@config)
+      @metrics_flusher = Wurk::Metrics::Flusher.new(@config)
+      @history = Wurk::History.new(@config) if @config.history_enabled?
     end
 
-    # Ent §5 Historical Metrics snapshotter — only when the host opted in via
-    # `config.retain_history`. Leader-gated like the rollups, so just one
-    # process emits the cluster-wide snapshot per interval.
-    def build_history
-      Wurk::History.new(@config) if @config.history_enabled?
+    # Filled in by #run. Declared up front so #initialize reads as the full
+    # inventory of what a Launcher owns.
+    def reset_thread_state
+      @started_at = @heartbeat = @heartbeat_thread = @boot_reclaim_thread = nil
     end
 
     # Every worker process campaigns for the single cluster lock (`dear-leader`);
     # one wins and renews it, the rest follow and promote on its death. Cadence
     # falls back to the spec defaults (TTL 30 / renew 15 / follower 60) unless
-    # the host tunes it. `Leader#start` no-ops under `WURK_LEADER=false`.
+    # the host tunes it, plus the short pre-campaign delay that keeps the first
+    # CAS off the boot path. `Leader#start` no-ops under `WURK_LEADER=false`.
     def build_leader
       Wurk::Leader.new(
         config: @config,
         ttl: @config[:leader_ttl] || Wurk::Leader::DEFAULT_TTL,
         renew_interval: @config[:leader_renew_interval] || Wurk::Leader::DEFAULT_RENEW_INTERVAL,
-        follower_interval: @config[:leader_follower_interval] || Wurk::Leader::DEFAULT_FOLLOWER_INTERVAL
+        follower_interval: @config[:leader_follower_interval] || Wurk::Leader::DEFAULT_FOLLOWER_INTERVAL,
+        initial_wait: @config[:leader_initial_wait] || Wurk::Leader::DEFAULT_INITIAL_WAIT
       )
     end