wurk 1.3.1 → 1.5.0

data/lib/wurk/middleware/poison_pill.rb

@@ -19,15 +19,22 @@ module Wurk
     # Counter key TTL is wire-compat with Sidekiq Pro — third-party tooling
     # that watches `super_fetch:recovered:*` expects 72h.
     #
-    # No server-middleware registration: callers are reaper / bulk_requeue
-    # paths that drive the lifecycle directly via `track!(payload, queue:)`.
-    # When that integration lands, it just calls `PoisonPill.track!` on each
-    # orphan it about to RPUSH back to the public queue.
+    # No server-middleware registration: the producer is Reaper#drain, which
+    # drives the lifecycle directly via `track!(payload, queue:)` on each
+    # orphan it moves back to the public queue. The consumer is the ACK —
+    # `Fetcher::Reliable::UnitOfWork#acknowledge` pipelines {clear_in} next to
+    # its LREM, so an attempt that finished starts the next one at zero
+    # without costing a round trip of its own.
     module PoisonPill
       RECOVERY_THRESHOLD = 3
       RECOVERY_TTL = 72 * 60 * 60
       KEY_PREFIX = 'super_fetch:recovered:'
-      DEAD_RECORD_LIMIT = 100
+
+      # Handed to death handlers (and therefore to `:death` batch callbacks)
+      # as the cause when a recovered job is killed as a poison pill. Never
+      # raised: nothing is on a stack here — the reaper kills the job from the
+      # outside, on behalf of the workers it took down.
+      class Poisoned < ::StandardError; end
 
       # The `pill` handed to a Pro `super_fetch! { |jobstr, pill| }` recovery
       # callback on the kill path. Responds to .jid/.klass/.count/.queue so a
@@ -62,7 +69,7 @@ module Wurk
 
         jid = job['jid']
         klass = job['class']
-        emit_recovered_fetch(klass, queue)
+        emit('jobs.recovered.fetch', klass, queue)
 
         count = bump_counter(jid) if jid && !jid.empty?
         if count && count >= RECOVERY_THRESHOLD
@@ -80,25 +87,74 @@ module Wurk
       def recovery_count(jid)
         return 0 if jid.nil? || jid.to_s.empty?
 
-        Wurk.redis { |conn| conn.call('GET', "#{KEY_PREFIX}#{jid}") }.to_i
+        Wurk.redis { |conn| conn.call('GET', counter_key(jid)) }.to_i
       end
 
-      # Resets the counter for a jid — call after a successful perform so a
-      # job that recovered twice and then completed doesn't accumulate state.
+      # Resets the counter for a jid. The hot path uses {clear_in} instead —
+      # this is the standalone form for the API/dashboard and for callers with
+      # no pipeline of their own.
       def clear!(jid)
         return if jid.nil? || jid.to_s.empty?
 
-        Wurk.redis { |conn| conn.call('DEL', "#{KEY_PREFIX}#{jid}") }
+        Wurk.redis { |conn| conn.call('DEL', counter_key(jid)) }
+      end
+
+      # Queue the counter reset onto a pipeline the caller already has open.
+      #
+      # The ACK is the reset point. An attempt that got as far as acking —
+      # returned, or raised and booked its retry — is proof the job did not
+      # take its worker down, and that is the only thing this counter
+      # measures. Without the reset, three reclaims caused by three unrelated
+      # crashes inside 72h dead-set a job that has been completing all along
+      # (jids do come back: a UI/API retry re-pushes the same one, as does any
+      # client that supplies its own). Riding a round trip the ACK already
+      # makes is what keeps that free for the jobs — nearly all of them — that
+      # were never reclaimed at all.
+      # Guards the blank jid for the same reason {clear!} and {recovery_count}
+      # do, rather than relying on the one caller to do it: `counter_key(nil)`
+      # is the bare KEY_PREFIX, so an unguarded DEL here would delete a key
+      # that is shared rather than per-job. The caller's own check stays --
+      # it also skips queueing the command at all -- but the invariant belongs
+      # on the method that builds the key.
+      def clear_in(pipe, jid)
+        return if jid.nil? || jid.to_s.empty?
+
+        pipe.call('DEL', counter_key(jid))
+      end
+
+      def counter_key(jid)
+        "#{KEY_PREFIX}#{jid}"
       end
 
       # Register a callback fired when a poison pill is detected. Callbacks
       # receive a single Hash `{jid:, klass:, count:, queue:}` — matches
       # Sidekiq Pro's documented shape so consumers can drop in unchanged.
+      #
+      # Dedups by callable identity — an initializer that re-runs (Rails
+      # code reloading, a re-required file) and hands back the *same* stored
+      # Proc each time won't accumulate duplicate closures (and the bindings
+      # they retain). A fresh block literal per call still registers
+      # separately, same as before. Returns a {Registration} the caller can
+      # `#remove!` to deregister explicitly.
       def on_poison(&block)
         raise ArgumentError, 'block required' unless block
 
-        callbacks << block
-        block
+        callbacks << block unless callbacks.any? { |cb| cb.equal?(block) }
+        Registration.new(callbacks, block)
+      end
+
+      # Handle returned by {on_poison}. Removing it is idempotent and safe
+      # even after a {reset!} discarded the underlying list.
+      class Registration
+        def initialize(list, callback)
+          @list = list
+          @callback = callback
+        end
+
+        def remove!
+          @list.delete(@callback)
+          nil
+        end
       end
 
       def callbacks
@@ -113,19 +169,21 @@ module Wurk
       # ---- internals --------------------------------------------------
 
       def parse(payload)
-        case payload
-        when Hash then payload
-        when String
-          begin
-            Wurk.load_json(payload)
-          rescue ::JSON::ParserError
-            nil
-          end
-        end
+        return payload if payload.is_a?(::Hash)
+        return nil unless payload.is_a?(::String)
+
+        Wurk.load_json(payload)
+      rescue ::JSON::ParserError
+        nil
       end
 
+      # No apply-safety claim: INCR is additive, so a block replayed after a
+      # lost reply bumps twice and can carry a healthy job past
+      # RECOVERY_THRESHOLD into the dead set. Raising instead only leaves the
+      # count short — Reaper#drain rescues, and the job is already back on its
+      # public queue, so it just misses one poison check.
       def bump_counter(jid)
-        key = "#{KEY_PREFIX}#{jid}"
+        key = counter_key(jid)
         Wurk.redis do |conn|
           count = conn.call('INCR', key).to_i
           conn.call('EXPIRE', key, RECOVERY_TTL)
@@ -133,25 +191,29 @@ module Wurk
         end
       end
 
-      def emit_recovered_fetch(klass, queue)
+      def emit(metric, klass, queue)
         tags = []
         tags << "class:#{klass}" if klass
         tags << "queue:#{queue}" if queue
-        Wurk::Metrics::Statsd.increment('jobs.recovered.fetch', tags: tags.empty? ? nil : tags)
+        Wurk::Metrics::Statsd.increment(metric, tags: tags.empty? ? nil : tags)
       end
 
+      # Death handlers fire (`notify_failure` defaults to true). A poison kill
+      # is a death like any other exhaustion: suppressing it strands every
+      # batch that owns the job — Batch::DeathHandler is a death handler, so a
+      # silent kill leaves the batch's pending count stuck forever and neither
+      # `:death` nor `:complete` ever runs. Pro's spec is silent here
+      # (docs/target/sidekiq-pro.md §12); recorded in docs/idea/parity-divergences.md.
       def mark_poison(payload, job, queue:, count:)
-        emit_poison(job['class'], queue)
+        emit('jobs.poison', job['class'], queue)
         json = payload.is_a?(String) ? payload : Wurk.dump_json(job)
-        Wurk::DeadSet.new.kill(json, notify_failure: false)
+        Wurk::DeadSet.new.kill(json, ex: poisoned_error(job['class'], count))
         fire_callbacks(jid: job['jid'], klass: job['class'], count: count, queue: queue)
       end
 
-      def emit_poison(klass, queue)
-        tags = []
-        tags << "class:#{klass}" if klass
-        tags << "queue:#{queue}" if queue
-        Wurk::Metrics::Statsd.increment('jobs.poison', tags: tags.empty? ? nil : tags)
+      def poisoned_error(klass, count)
+        message = "#{klass || 'job'} was recovered #{count} times without completing"
+        Poisoned.new(message).tap { |e| e.set_backtrace(caller) }
       end
 
       def fire_callbacks(pill)

data/lib/wurk/middleware.rb

@@ -24,8 +24,8 @@ module Wurk
         config.logger
       end
 
-      def redis(&)
-        config.redis(&)
+      def redis(idempotent: false, &)
+        config.redis(idempotent:, &)
       end
     end
 

data/lib/wurk/pool_checkout.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative 'redis_pool'
+
+module Wurk
+  # The seam every `#redis`-style wrapper checks out through.
+  #
+  # `idempotent:` is a {Wurk::RedisPool} concept — it re-enables block replay
+  # after a command may already have applied server-side (see RedisPool#with).
+  # The object behind a wrapper is not always ours, though: the Sidekiq surface
+  # Wurk drops into lets a host supply any ConnectionPool-shaped object
+  # (`Sidekiq::Client#redis_pool=`, `Sidekiq::Web.redis_pool=`, the `pool:`
+  # kwarg on Stats::History / Deploy / Leader / Profiler / Metrics::History),
+  # and a bare `def with; yield conn; end` decorator — the shape people reach
+  # for to count or trace round trips — is a legal implementation of it. Handing
+  # that an unknown keyword is an ArgumentError, so the claim goes only to a pool
+  # that understands it.
+  #
+  # Dropping the claim is always sound: the pool falls back to the conservative
+  # no-replay default, which is exactly what a foreign pool did before the
+  # keyword existed. Apply-safety is an optimization, never a correctness input.
+  module PoolCheckout
+    def self.with(pool, idempotent, &)
+      return pool.with(&) unless idempotent && pool.is_a?(RedisPool)
+
+      pool.with(idempotent: true, &)
+    end
+
+    # Same apply-safety forwarding as .with, minus the is_a? probe. Only for
+    # call sites whose pool is always this process's own {RedisPool} — never a
+    # host-supplied `pool:` kwarg — so the polymorphic check has nothing to
+    # decide.
+    def self.trusted(pool, idempotent, &)
+      return pool.with(&) unless idempotent
+
+      pool.with(idempotent: true, &)
+    end
+  end
+end

data/lib/wurk/process_set.rb

@@ -32,7 +32,7 @@ module Wurk
     # absence (process never registered) and expiry (heartbeat lapsed,
     # info field gone) both return nil.
     def self.[](identity)
-      exists, fields = Wurk.redis do |conn|
+      exists, fields = Wurk.redis(idempotent: true) do |conn|
         conn.pipelined do |pipe|
           pipe.call('SISMEMBER', Keys::PROCESSES, identity)
           pipe.call('HMGET', identity, *LOOKUP_FIELDS)
@@ -48,6 +48,11 @@ module Wurk
     # don't dogpile the prune. Returns the number of identities removed
     # (or 0 when the lock was held by someone else).
     #
+    # No apply-safety claim, unlike the reads around it: a replay re-decides
+    # which identities are dead, so it could prune one that registered in
+    # between and hasn't written `info` yet. A blip here just skips one prune —
+    # the next caller a minute later does it.
+    #
     # Spec: docs/target/sidekiq-free.md §31.17.
     def cleanup
       return 0 unless acquired_cleanup_lock?
@@ -84,7 +89,7 @@ module Wurk
     # SCARD over `processes`. Not pruned — may include identities whose
     # heartbeat has lapsed. Use `each` for the accurate count.
     def size
-      Wurk.redis { |conn| conn.call('SCARD', Keys::PROCESSES) }
+      Wurk.redis(idempotent: true) { |conn| conn.call('SCARD', Keys::PROCESSES) }
     end
 
     # Sum of `concurrency` across live processes. Iterates `each` so dead
@@ -104,7 +109,7 @@ module Wurk
     # `||=` with empty-string fallback distinguishes "leader is unset" from
     # "memoization not yet computed".
     def leader
-      @leader ||= Wurk.redis { |c| c.call('GET', 'dear-leader') } || ''
+      @leader ||= Wurk.redis(idempotent: true) { |c| c.call('GET', 'dear-leader') } || ''
     end
 
     class << self
@@ -134,7 +139,7 @@ module Wurk
     private
 
     def fetch_each_rows
-      Wurk.redis do |conn|
+      Wurk.redis(idempotent: true) do |conn|
         procs = conn.call('SMEMBERS', Keys::PROCESSES).sort
         next [] if procs.empty?
 
@@ -223,7 +228,7 @@ module Wurk
     # Compares identity against the `dear-leader` STRING. Ent-only;
     # always false in OSS/free.
     def leader?
-      Wurk.redis { |c| c.call('GET', 'dear-leader') == identity }
+      Wurk.redis(idempotent: true) { |c| c.call('GET', 'dear-leader') == identity }
     end
 
     private

data/lib/wurk/processor.rb

@@ -10,8 +10,11 @@ require_relative 'profiler'
 module Wurk
   # Inside each Manager, N Processors run in parallel. Each owns one thread,
   # pulls a UnitOfWork from the capsule's fetcher, parses the payload, walks
-  # the server middleware chain, invokes `perform`, then ACKs (removes the
-  # payload from the per-process private list).
+  # the server middleware chain, invokes `perform`, then ACKs (retires the
+  # payload from the per-process private list). The ACK is handed to the
+  # fetcher, which pipelines it with the next fetch rather than spending a
+  # round trip on it — #flush_acks covers the case where there is no next
+  # fetch.
   #
   # Shutdown is two-stage:
   #   * `terminate` flips a flag; the run loop exits between jobs.
@@ -23,6 +26,26 @@ module Wurk
   class Processor
     include Component
 
+    # Interrupt masks for the two `Thread.handle_interrupt` scopes every job
+    # runs inside. Frozen constants rather than the inline literals they
+    # replace: the mask never varies, and a literal allocated two Hashes per
+    # job. Sidekiq hoists the same pair (processor.rb:161-164).
+    IGNORE_SHUTDOWN_INTERRUPTS = { Wurk::Shutdown => :never }.freeze
+    private_constant :IGNORE_SHUTDOWN_INTERRUPTS
+    ALLOW_SHUTDOWN_INTERRUPTS = { Wurk::Shutdown => :immediate }.freeze
+    private_constant :ALLOW_SHUTDOWN_INTERRUPTS
+
+    # Stand-in for the default reloader, `proc { |&b| b.call }`. That default
+    # is an identity wrapper, but a block param (`|&b|`) forces MRI to reify
+    # the dispatch block into a Proc on every job just to call it straight
+    # back; `yield` does not. Same contract, one less allocation per job.
+    module IdentityReloader
+      def self.call
+        yield
+      end
+    end
+    private_constant :IdentityReloader
+
     attr_reader :thread, :job, :capsule
 
     def initialize(capsule, &callback)
@@ -32,7 +55,7 @@ module Wurk
       @done = false
       @job = nil
       @thread = nil
-      @reloader = capsule.config[:reloader] || proc { |&b| b.call }
+      @reloader = resolve_reloader(capsule.config[:reloader])
       @job_logger = (capsule.config[:job_logger] || JobLogger).new(capsule.config)
       @retrier = JobRetry.new(capsule)
     end
@@ -116,6 +139,19 @@ module Wurk
         @lock.synchronize { @work.delete(tid) }
       end
 
+      # RAII: publish for the duration of the block, always retract. The write
+      # lives *inside* this method's ensure frame on purpose — with the `set`
+      # one line above a `begin`, an async raise landing in that gap (a host
+      # timeout's `Thread#raise`, say) escapes the ensure and strands the
+      # entry for the life of the process: the payload String stays reachable
+      # and every heartbeat reports the thread as busy.
+      def track(tid, hash)
+        set(tid, hash)
+        yield
+      ensure
+        delete(tid)
+      end
+
       def dup
         @lock.synchronize { @work.dup }
       end
@@ -136,8 +172,23 @@ module Wurk
 
     private
 
+    # Only the untouched framework default is swapped out — a host-supplied
+    # reloader (Rails wraps every job in `Rails.application.reloader`) is used
+    # exactly as given. `equal?` against DEFAULTS is sound because
+    # Configuration's deep-dup copies Hashes/Arrays only, so an unset
+    # `:reloader` is still the very Proc object DEFAULTS holds.
+    def resolve_reloader(configured)
+      return IdentityReloader if configured.nil? || configured.equal?(Configuration::DEFAULTS[:reloader])
+
+      configured
+    end
+
     def run
-      process_one until @done
+      begin
+        process_one until @done
+      ensure
+        flush_acks
+      end
       @callback&.call(self)
     rescue Wurk::Shutdown
       @callback&.call(self)
@@ -147,6 +198,24 @@ module Wurk
       raise
     end
 
+    # The fetcher holds each finished job's LREM until a fetch can pipeline it
+    # (Fetcher::Reliable#defer_ack). This thread has stopped fetching, so
+    # nothing else will send the one it may still be holding — and a finished
+    # job left in the private list is invisible to Manager#hard_shutdown's
+    # in-flight list, so the next boot's reaper would run it a second time.
+    #
+    # Inside the loop's own ensure rather than the method's: the callback below
+    # drops this Processor from the Manager's pool, which is what lets
+    # Manager#stop return and close the capsule's Redis pool out from under us.
+    # `respond_to?` because a config[:fetch_class] fetcher need not defer, and
+    # the capsule has no fetcher at all if the launcher died before prepare!.
+    def flush_acks
+      fetcher = @capsule.fetcher
+      fetcher.flush_pending_acks if fetcher.respond_to?(:flush_pending_acks)
+    rescue StandardError => e
+      handle_exception(e, { context: 'Error flushing pending acks' })
+    end
+
     def fetch
       @capsule.fetcher.retrieve_work
     rescue Wurk::Shutdown
@@ -164,11 +233,17 @@ module Wurk
       job_hash = parse_or_kill(jobstr, uow)
       return if job_hash.nil?
 
+      # The fetcher never parses, so hand it the jid we just read: the ACK
+      # retires this job's poison-pill recovery counter inside the round trip
+      # it rides. A fetcher plugged in via `config[:fetch_class]` has no jid
+      # slot and simply ACKs — the counter then ages out on its 72h TTL.
+      uow.jid = job_hash['jid'] if uow.respond_to?(:jid=)
+
       ack = false
       begin
-        Thread.handle_interrupt(Wurk::Shutdown => :never) do
+        Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
           dispatch(job_hash, queue, jobstr) do |instance|
-            Thread.handle_interrupt(Wurk::Shutdown => :immediate) do
+            Thread.handle_interrupt(ALLOW_SHUTDOWN_INTERRUPTS) do
               execute_job(instance, job_hash, queue)
             end
           end
@@ -241,15 +316,14 @@ module Wurk
     # Heartbeat can mirror it into Redis (`<identity>:work`), and increments
     # PROCESSED/FAILURE counters around the inner block.
     def stats(jobstr, queue)
-      WORK_STATE.set(tid, queue: queue, payload: jobstr, run_at: ::Time.now.to_i)
-      begin
+      run_at = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :second)
+      WORK_STATE.track(tid, queue: queue, payload: jobstr, run_at: run_at) do
         yield
       rescue Exception # rubocop:disable Lint/RescueException
         FAILURE.incr
         raise
       ensure
         PROCESSED.incr
-        WORK_STATE.delete(tid)
       end
     end
   end

data/lib/wurk/profiler.rb

@@ -5,6 +5,7 @@ require 'zlib'
 require 'stringio'
 require 'tempfile'
 require_relative 'keys'
+require_relative 'pool_checkout'
 
 module Wurk
   # Job profiling (Sidekiq 8.0+, OSS). When a job is pushed with a `profile`
@@ -33,11 +34,15 @@ module Wurk
       # from the interrupt/expiry middleware) must propagate untouched so the
       # retry/skip flow works. Only the storage step is made failure-safe
       # (see #safe_store).
-      def call(job_hash, &)
+      # No `&block` parameter: every job passes through here, and declaring one
+      # would reify the caller's block into a Proc even on the `yield`-straight-
+      # through path that opted-out jobs take. The capture path pays for its own
+      # block instead.
+      def call(job_hash)
         label = job_hash['profile']
         return yield unless label && defined?(::Vernier)
 
-        capture(job_hash, label, &)
+        capture(job_hash, label) { yield } # rubocop:disable Style/ExplicitBlockArgument
       end
 
       # Persists a profile. Extracted from capture so it is unit-testable
@@ -113,8 +118,8 @@ module Wurk
         end
       end
 
-      def with_pool(pool, &)
-        pool ? pool.with(&) : Wurk.redis(&)
+      def with_pool(pool, idempotent: false, &)
+        pool ? PoolCheckout.with(pool, idempotent, &) : Wurk.redis(idempotent:, &)
       end
 
       def now

data/lib/wurk/queue.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'fetcher/reliable'
 require_relative 'job_record'
 
 module Wurk
@@ -12,6 +13,11 @@ module Wurk
   # (see `Wurk::Fetcher::Reliable#queues_cmd`). In-flight jobs continue to
   # completion — pausing only stops new fetches.
   #
+  # Fetchers answer from a `PAUSED_TTL` cache rather than re-reading the SET
+  # every pass, so `pause!`/`unpause!` expire this process's copy inline and
+  # other processes pick the change up within the TTL. `paused?` is a reporting
+  # call and always reads Redis.
+  #
   # Spec: docs/target/sidekiq-free.md §19.2, sidekiq-pro.md §6.
   class Queue
     include Enumerable
@@ -25,7 +31,7 @@ module Wurk
 
     # @return [Array<Queue>] one per known queue, sorted by name.
     def self.all
-      names = Wurk.redis { |conn| conn.call('SMEMBERS', Keys::QUEUES_SET) }
+      names = Wurk.redis(idempotent: true) { |conn| conn.call('SMEMBERS', Keys::QUEUES_SET) }
       names.sort.map { |n| new(n) }
     end
 
@@ -35,12 +41,12 @@ module Wurk
     end
 
     def size
-      Wurk.redis { |conn| conn.call('LLEN', @rname) }
+      Wurk.redis(idempotent: true) { |conn| conn.call('LLEN', @rname) }
     end
 
     # Seconds since the oldest job (tail of LIST) was enqueued. 0.0 when empty.
     def latency
-      payload = Wurk.redis { |conn| conn.call('LRANGE', @rname, -1, -1).first }
+      payload = Wurk.redis(idempotent: true) { |conn| conn.call('LRANGE', @rname, -1, -1).first }
       return 0.0 if payload.nil?
 
       JobRecord.latency_from(Wurk.load_json(payload)['enqueued_at'])
@@ -51,19 +57,21 @@ module Wurk
     # True iff this queue's name is a member of the `paused` SET. Wurk
     # implements the Pro contract for free; fetchers consult the same set.
     def paused?
-      Wurk.redis { |conn| conn.call('SISMEMBER', Keys::PAUSED_SET, @name) } == 1
+      Wurk.redis(idempotent: true) { |conn| conn.call('SISMEMBER', Keys::PAUSED_SET, @name) } == 1
     end
 
     # Pause new fetches against this queue. Idempotent — `SADD` returns
     # 0 when the name was already present. In-flight jobs are untouched.
     def pause! # rubocop:disable Naming/PredicateMethod
-      Wurk.redis { |conn| conn.call('SADD', Keys::PAUSED_SET, @name) }
+      Wurk.redis(idempotent: true) { |conn| conn.call('SADD', Keys::PAUSED_SET, @name) }
+      Fetcher::Reliable.invalidate_paused_cache!
       true
     end
 
     # Resume fetches. Idempotent.
     def unpause! # rubocop:disable Naming/PredicateMethod
-      Wurk.redis { |conn| conn.call('SREM', Keys::PAUSED_SET, @name) }
+      Wurk.redis(idempotent: true) { |conn| conn.call('SREM', Keys::PAUSED_SET, @name) }
+      Fetcher::Reliable.invalidate_paused_cache!
       true
     end
 
@@ -74,7 +82,7 @@ module Wurk
       loop do
         start  = page * PAGE_SIZE
         stop   = start + PAGE_SIZE - 1
-        slice  = Wurk.redis { |conn| conn.call('LRANGE', @rname, start, stop) }
+        slice  = Wurk.redis(idempotent: true) { |conn| conn.call('LRANGE', @rname, start, stop) }
         slice.each { |value| yield JobRecord.new(value, @name) }
         break if slice.size < PAGE_SIZE
 
@@ -91,6 +99,9 @@ module Wurk
     # UNLINK the list + drop the queue from the `queues` set. Pipelined
     # so a partial failure leaves at most one of the two ops applied.
     # Method name is Sidekiq wire-compat — `clear?` would break the alias.
+    #
+    # Unlike pause!/unpause!, this one can't claim apply-safety: a replay after
+    # a lost reply would UNLINK whatever a producer enqueued in between.
     def clear # rubocop:disable Naming/PredicateMethod
       Wurk.redis do |conn|
         conn.pipelined do |pipe|

data/lib/wurk/rails_boot.rb

@@ -8,6 +8,11 @@ module Wurk
   # boot policy stays pure and unit-testable without the Railtie DSL.
   # See docs/idea/03-process-model.md for the exact ordering.
   module RailsBoot
+    # What `at_exit` waits for the supervise thread on top of the swarm's own
+    # drain budget: one supervise tick to notice the request, plus slack for
+    # the final reap.
+    DRAIN_JOIN_SLACK = 1
+
     module_function
 
     # Invoked from the `wurk.server_mode` initializer, before config/initializers
@@ -103,36 +108,62 @@ module Wurk
     end
 
     def boot_swarm
-      swarm = Wurk::Swarm.new(topology: Wurk.configuration.topology,
-                              shutdown_timeout: Wurk.configuration[:timeout] || Swarm::DEFAULT_SHUTDOWN_TIMEOUT)
+      timeout = Wurk.configuration[:timeout] || Swarm::DEFAULT_SHUTDOWN_TIMEOUT
+      swarm = Wurk::Swarm.new(topology: Wurk.configuration.topology, shutdown_timeout: timeout)
+      supervisor = nil
+      # Registered BEFORE boot, which forks the children one slot at a time: a
+      # fork that raises partway through would otherwise leave the children it
+      # already spawned with nothing to drain them on host exit. Children
+      # inherit the hook — stop_swarm no-ops off the process that forked them.
+      at_exit { stop_swarm(swarm, supervisor, timeout + Swarm::SHUTDOWN_GRACE + DRAIN_JOIN_SLACK) }
       # Co-hosted in the web process (e.g. Puma single mode): the host owns the
       # process-wide TERM/INT traps. Installing the swarm's own would hijack
       # them — a deploy TERM would drain the swarm but never stop the HTTP
       # server. Let the host keep signal ownership and drain the swarm on its
       # graceful exit (same contract as boot_embedded).
       swarm.boot(install_signals: false)
-      at_exit { swarm.shutdown }
       # supervise must still run somewhere or crashed children never respawn and
       # memory checks never fire. A background thread keeps the host's main
       # thread free to serve HTTP.
-      Thread.new do
+      supervisor = Thread.new do
         swarm.supervise
       rescue StandardError => e
         logger.error { "wurk supervisor thread died: #{e.class}: #{e.message}" }
       end
     end
 
+    # at_exit fires on the host's main thread, but the supervise thread owns the
+    # child table and two threads inside `shutdown` race on it — so request the
+    # drain and wait for the supervisor to run it. Draining here is the fallback
+    # for when no live supervisor will: it never started (boot raised) or it
+    # died early; after one that drained it finds no children and no-ops. A
+    # supervisor still alive past the join is wedged mid-drain — leave it be
+    # rather than race it; its children self-terminate (OrphanGuard) once this
+    # process is gone.
+    def stop_swarm(swarm, supervisor, drain_wait)
+      return unless swarm.owner?
+
+      swarm.request_shutdown
+      supervisor&.join(drain_wait)
+      swarm.shutdown unless supervisor&.alive?
+    end
+
     # Sidekiq-embedded parity: a threads-only worker inside the web process, no
     # fork. Redis validation failure keeps the host serving HTTP (log + carry
     # on). at_exit drains on the graceful shutdown the web server runs on TERM.
-    def boot_embedded
-      instance = Wurk::Embedded.new(Wurk.configuration)
-      instance.run
+    def boot_embedded(embedded = Wurk::Embedded)
+      instance = embedded.new(Wurk.configuration)
+      # Registered BEFORE run, for the reason boot_swarm registers before the
+      # fork: run brings the heartbeat, pollers, managers and health listener up
+      # one at a time, and a raise partway through would otherwise leave the ones
+      # already running with nothing to drain them on host exit.
       at_exit { instance.stop }
+      instance.run
       logger.info { 'wurk: running embedded in the web process (config.wurk.embed_in_web) — threads only, no fork' }
       instance
     rescue StandardError => e
       logger.error { "wurk: embedded boot failed: #{e.class}: #{e.message}" }
+      instance&.stop
       nil
     end