wurk 1.3.1 → 1.5.0

data/lib/wurk/cron.rb

@@ -6,6 +6,7 @@ require 'time'
 require_relative 'component'
 require_relative 'client'
 require_relative 'leader'
+require_relative 'timer_loop'
 
 module Wurk
   # Sidekiq Enterprise periodic jobs. Pure leader-driven cron — only the
@@ -477,8 +478,8 @@ module Wurk
 
       private
 
-      def redis(&)
-        @config ? @config.redis(&) : Wurk.redis(&)
+      def redis(idempotent: false, &)
+        @config ? @config.redis(idempotent:, &) : Wurk.redis(idempotent:, &)
       end
     end
 
@@ -520,34 +521,35 @@ module Wurk
 
       def initialize(config)
         @config = config
-        @done = false
-        @mutex = ::Mutex.new
-        @sleeper = ::ConditionVariable.new
         @client = Client.new(config: config)
         @thread = nil
         # Operators never need to touch this; integration tests shrink it so a
         # due loop fires within the test window instead of waiting a full minute.
         @tick_interval = config[:cron_tick_interval] || DEFAULT_TICK_SECONDS
+        @timer = TimerLoop.new(@tick_interval)
       end
 
+      # TimerLoop waits one interval before the first tick: don't fire a
+      # catch-up burst the instant we boot (the leader is barely settled), and
+      # let a short-lived process exit without ticking at all.
       def start
-        @poller_thread ||= safe_thread('cron-poller') do # rubocop:disable Naming/MemoizedInstanceVariableName
-          # Wait one interval before the first tick: don't fire a catch-up burst
-          # the instant we boot (the leader is barely settled), and let a
-          # short-lived process exit without ticking at all.
-          wait
-          until @done
-            tick
-            wait
-          end
-        end
+        return @thread if @thread
+
+        @timer.reset
+        @thread = safe_thread('cron-poller') { @timer.run { tick } }
       end
 
+      # Blocks until the thread is really gone: the launcher releases the
+      # cluster lock immediately after this returns, and a tick still in flight
+      # would enqueue loops the next leader is about to fire itself.
+      #
+      # Cleared only on a confirmed join (Thread#join returns nil on timeout):
+      # a wedged thread must stay tracked so #start's guard returns it instead
+      # of calling @timer.reset, which would un-terminate the loop it is still
+      # inside and leave two tick threads double-enqueuing the same loops.
       def terminate
-        @mutex.synchronize do
-          @done = true
-          @sleeper.signal
-        end
+        @timer.terminate
+        @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
       end
 
       # Leader-gated by the single cluster lock (Component#leader? reads
@@ -562,18 +564,23 @@ module Wurk
         handle_exception(e, { context: 'cron-poller' })
       end
 
+      # The marks advance *before* the push, via CAS: the leader gate is a
+      # cached read, so a second poller can reach the same due loop for a few
+      # seconds after a handover. Losing the CAS means another tick owns this
+      # slot — enqueue nothing.
       def enqueue_if_due(loop_obj)
         return if loop_obj.paused?
 
         now = ::Time.now.to_i
-        prev_fire, next_fire = read_fire_marks(loop_obj.lid)
-        next_fire ||= loop_obj.next_fire_at(prev_fire || (now - @tick_interval))
-        return if next_fire.nil? || next_fire > now
+        slot, mark = due_slot(loop_obj, now)
+        return if slot.nil?
 
-        warn_missed_tick(loop_obj, next_fire, now)
-        jid = enqueue!(loop_obj)
-        future = loop_obj.next_fire_after(next_fire, now)
-        record_fire(loop_obj, jid, now, future)
+        future = loop_obj.next_fire_after(slot, now)
+        return unless claim_fire?(loop_obj, mark, now, future)
+
+        warn_missed_tick(loop_obj, slot, now)
+        jid = enqueue_claimed!(loop_obj)
+        record_history(loop_obj, jid, now)
         jid
       end
 
@@ -590,10 +597,16 @@ module Wurk
 
       private
 
-      def wait
-        @mutex.synchronize do
-          @sleeper.wait(@mutex, @tick_interval) unless @done
-        end
+      # The slot this tick would fire and the token that claims it, or nil when
+      # the loop has no resolvable occurrence or none is due yet. The token is
+      # the stored `nf` when there is one, otherwise the slot we derived — see
+      # the `cron_claim_fire` script for how the two cases differ.
+      def due_slot(loop_obj, now)
+        prev_fire, next_fire, mark = read_fire_marks(loop_obj.lid)
+        next_fire ||= loop_obj.next_fire_at(prev_fire || (now - @tick_interval))
+        return if next_fire.nil? || next_fire > now
+
+        [next_fire, mark || next_fire.to_s]
       end
 
       def warn_missed_tick(loop_obj, expected, now)
@@ -605,6 +618,34 @@ module Wurk
         )
       end
 
+      # Compare-and-swap the fire marks: only the tick still looking at the
+      # mark it read wins the slot. Manual `#fire` deliberately skips this —
+      # an operator-requested run contends with no schedule slot.
+      def claim_fire?(loop_obj, mark, fired_at, future)
+        @config.redis do |c|
+          Wurk::Lua::Loader.eval_cached(
+            c, :cron_claim_fire,
+            keys: ["#{LOOP_PREFIX}#{loop_obj.lid}"],
+            argv: [mark, fired_at.to_s, future.nil? ? '' : future.to_s]
+          )
+        end == 1
+      end
+
+      # The slot is already claimed, so a failed push loses this occurrence
+      # instead of leaving the next tick to fire it again. Ent leader election
+      # is explicitly best-effort coordination (sidekiq-ent.md §6), so a missed
+      # occurrence is tolerable where a duplicate one is not — log the gap so it
+      # stays attributable.
+      def enqueue_claimed!(loop_obj)
+        enqueue!(loop_obj)
+      rescue StandardError => e
+        logger.warn(
+          "[cron] fire lost lid=#{loop_obj.lid} klass=#{loop_obj.klass} " \
+          "mark already advanced, occurrence skipped: #{e.class}: #{e.message}"
+        )
+        raise
+      end
+
       def enqueue!(loop_obj)
         aj = active_job_class(loop_obj.klass)
         return enqueue_active_job(aj, loop_obj) if aj
@@ -643,24 +684,40 @@ module Wurk
         job.provider_job_id if job.respond_to?(:provider_job_id)
       end
 
+      # Third element is the raw `nf` string — the CAS token for #claim_fire.
+      # Comparing the bytes Redis holds, not the parsed integer, keeps the
+      # token exact regardless of how the value was written.
       def read_fire_marks(lid)
         @config.redis do |c|
-          vals = c.call('HMGET', "#{LOOP_PREFIX}#{lid}", 'lf', 'nf')
-          next_fire = vals[1]
+          prev_fire, next_fire = c.call('HMGET', "#{LOOP_PREFIX}#{lid}", 'lf', 'nf')
           # Preserve nil: treat missing or empty 'nf' as nil, not 0.
-          [vals[0]&.to_i, next_fire.nil? || next_fire.empty? ? nil : next_fire.to_i]
+          next_fire = nil if next_fire.nil? || next_fire.empty?
+          [prev_fire&.to_i, next_fire&.to_i, next_fire]
         end
       end
 
       def record_fire(loop_obj, jid, fired_at, future)
-        history_entry = Wurk.dump_json([fired_at, jid])
+        write_fire_marks(loop_obj.lid, fired_at, future)
+        record_history(loop_obj, jid, fired_at)
+      end
+
+      # Unconditional mark write, for the manual `#fire` path only. The
+      # scheduled path advances the marks through #claim_fire instead.
+      def write_fire_marks(lid, fired_at, future)
+        key = "#{LOOP_PREFIX}#{lid}"
         @config.redis do |c|
           if future.nil?
-            c.call('HSET', "#{LOOP_PREFIX}#{loop_obj.lid}", 'lf', fired_at.to_s)
-            c.call('HDEL', "#{LOOP_PREFIX}#{loop_obj.lid}", 'nf')
+            c.call('HSET', key, 'lf', fired_at.to_s)
+            c.call('HDEL', key, 'nf')
           else
-            c.call('HSET', "#{LOOP_PREFIX}#{loop_obj.lid}", 'lf', fired_at.to_s, 'nf', future.to_s)
+            c.call('HSET', key, 'lf', fired_at.to_s, 'nf', future.to_s)
           end
+        end
+      end
+
+      def record_history(loop_obj, jid, fired_at)
+        history_entry = Wurk.dump_json([fired_at, jid])
+        @config.redis do |c|
           c.call('LPUSH', "#{HISTORY_PREFIX}#{loop_obj.lid}", history_entry)
           c.call('LTRIM', "#{HISTORY_PREFIX}#{loop_obj.lid}", 0, HISTORY_CAP - 1)
         end

data/lib/wurk/deploy.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'pool_checkout'
+
 module Wurk
   # Records deploy markers into Redis so the history pane can overlay
   # "deployed at" lines onto job-throughput charts. Wire-compatible with
@@ -84,11 +86,11 @@ module Wurk
       nil
     end
 
-    def with_redis(&)
+    def with_redis(idempotent: false, &)
       if @pool
-        @pool.with(&)
+        PoolCheckout.with(@pool, idempotent, &)
       else
-        Wurk.redis(&)
+        Wurk.redis(idempotent:, &)
       end
     end
   end

data/lib/wurk/embedded.rb

@@ -41,6 +41,12 @@ module Wurk
       sleep 0.2
       logger.info { "Wurk running embedded, total process thread count: #{Thread.list.size}" }
       logger.debug { Thread.list.map(&:name).to_s }
+    rescue StandardError
+      # The host is left believing Wurk never started, so anything the launcher
+      # did get up would keep fetching, beating and campaigning for the leader
+      # lock unowned for the life of the process.
+      roll_back_partial_boot
+      raise
     end
 
     # Stop fetching new work; in-flight jobs continue.
@@ -56,6 +62,13 @@ module Wurk
 
     private
 
+    # Guarded so the caller sees why the boot failed, not why the rollback did.
+    def roll_back_partial_boot
+      stop
+    rescue StandardError => e
+      handle_exception(e, { context: 'embedded-boot-rollback' })
+    end
+
     # Extracted so tests can swap in a fake launcher without monkey-patching
     # Wurk::Launcher.new globally (which races other parallel tests).
     def build_launcher

data/lib/wurk/engine.rb

@@ -25,6 +25,25 @@ module Wurk
     class AssetMount
       PREFIX = '/wurk-assets'
 
+      # Vite fingerprints everything it emits into `assets/`
+      # (`Dashboard-cGKycyd0.js`), so the bytes behind one of those URLs can
+      # never change — cache them for a year and never revalidate. The rest of
+      # the bundle (index.html, the favicons, wurk-manifest.json,
+      # .vite/manifest.json) keeps a stable name across builds, so it must be
+      # revalidated or a client would pin a stale dashboard shell forever.
+      FINGERPRINTED_PREFIX = '/assets/'
+      IMMUTABLE_CACHE_CONTROL = 'public, max-age=31536000, immutable'
+      REVALIDATE_CACHE_CONTROL = 'public, no-cache'
+
+      # Only a served body gets a cache directive. Rack::Files answers more than
+      # file reads: OPTIONS with 200 + Allow, 405 for any other verb
+      # (`ALLOWED_VERBS = %w[GET HEAD OPTIONS]`), and 416 for an unsatisfiable
+      # Range. 405 is cacheable by default (RFC 9110 §15.5.6), so stamping it
+      # `immutable` would let a shared proxy serve "method not allowed" for a
+      # year to everyone behind it.
+      CACHEABLE_METHODS = %w[GET HEAD].freeze
+      CACHEABLE_STATUSES = [200, 206, 304].freeze
+
       def initialize(app, root:)
         @app   = app
         @files = ::Rack::Files.new(root)
@@ -38,7 +57,28 @@ module Wurk
         stripped = path.delete_prefix(PREFIX)
         inner[::Rack::PATH_INFO] = stripped.empty? ? '/' : stripped
         response = @files.call(inner)
-        response[0] == 404 ? @app.call(env) : response
+        return @app.call(env) if response[0] == 404
+
+        stamp_cache_control(response, env, stripped)
+        response
+      end
+
+      private
+
+      # This mount is inserted at index 0 of the HOST app's stack (see the
+      # initializer below), so Rack::ETag and Rack::ConditionalGet never see
+      # these responses — without this the dashboard's fingerprinted bundle
+      # shipped with no cache directive at all and was refetched every load.
+      def stamp_cache_control(response, env, stripped)
+        return unless CACHEABLE_METHODS.include?(env[::Rack::REQUEST_METHOD])
+        return unless CACHEABLE_STATUSES.include?(response[0])
+
+        response[1]['cache-control'] ||=
+          if stripped.start_with?(FINGERPRINTED_PREFIX)
+            IMMUTABLE_CACHE_CONTROL
+          else
+            REVALIDATE_CACHE_CONTROL
+          end
       end
     end
 

data/lib/wurk/errors.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Wurk
+  class Error < StandardError; end
+
+  # Raised inside a worker process to abort the run loop. User code must not
+  # rescue this — the swarm uses it to signal teardown across thread boundaries.
+  # Spec: docs/target/sidekiq-free.md §3.
+  #
+  # Lives in its own file, loaded before everything else, because Processor
+  # freezes its `Thread.handle_interrupt` masks into constants: those are
+  # evaluated the moment `wurk/processor` is required, so this class has to
+  # exist by then.
+  class Shutdown < Interrupt; end
+end

data/lib/wurk/fetcher/reaper.rb

@@ -3,13 +3,14 @@
 require_relative '../component'
 require_relative '../keys'
 require_relative '../middleware/poison_pill'
+require_relative '../timer_loop'
 
 module Wurk
   class Fetcher
     # Orphan reclamation for the reliable fetcher (Pro super_fetch §3.2).
     #
     # The Reliable fetcher moves each job from a public queue into a
-    # per-process private list (`queue:<public>|<host>|<pid>|<idx>`) and
+    # per-process private list (`queue:<public>|<host>|<pid>|<nonce>|<idx>`) and
     # leaves it there until the Processor ACKs. A SIGKILLed or crashed
     # worker therefore strands its in-flight jobs in private lists that
     # nobody will ever ACK. The Reaper is the recovery half: it periodically
@@ -17,15 +18,16 @@ module Wurk
     # moves their jobs back to the public queue so a live worker re-runs them.
     #
     # Liveness is decided per owner:
-    #   * same host — the OS is authoritative: `Process.kill(0, pid)`. This
-    #     is instant and ignores a stale `processes` SET entry whose 60s TTL
-    #     hasn't lapsed yet, so a `kill -9`ed sibling is reclaimed the moment
-    #     the supervisor reaps it rather than 60s later. (Pid reuse by an
-    #     unrelated local process is the one blind spot — the supervisor
-    #     respawns with a fresh pid, so it does not arise in practice.)
-    #   * other host — we cannot ping the pid, so we trust the heartbeat:
-    #     the owner is alive iff some live `processes` member (one whose
-    #     `info` hash still exists) shares its `host:pid`. Cross-host reclaim
+    #   * our own host and nonce — the pid was minted in our PID namespace, so
+    #     the OS is authoritative: `Process.kill(0, pid)`. This is instant and
+    #     ignores a stale `processes` SET entry whose 60s TTL hasn't lapsed
+    #     yet, so a `kill -9`ed sibling is reclaimed the moment the supervisor
+    #     reaps it rather than 60s later. (Pid reuse by an unrelated process in
+    #     the same tree is the one blind spot — the supervisor respawns with a
+    #     fresh pid, so it does not arise in practice.)
+    #   * any other incarnation — its pid means nothing in our namespace, so we
+    #     trust the heartbeat: the owner is alive iff its identity is a live
+    #     `processes` member (one whose `info` hash still exists). Such reclaim
     #     therefore waits out the 60s heartbeat TTL, exactly as the spec says.
     #
     # Re-pushed jobs run through Wurk::Middleware::PoisonPill, which caps a
@@ -90,13 +92,22 @@ module Wurk
         @thread
       end
 
+      # Bounded at TimerLoop::JOIN_TIMEOUT like every other periodic component.
+      # The full-keyspace sweep is exactly the tick that outlasts a stop: it
+      # SCANs the whole `queue:*|*` keyspace and drains what it finds, so on a
+      # large or slow Redis it can still be running when shutdown lands. Waiting
+      # it out held the entire process's teardown open past the swarm parent's
+      # SHUTDOWN_GRACE — which SIGKILLs the child mid-drain, the one outcome
+      # this component exists to recover from. A straggler is left running
+      # instead — and still referenced, like every other periodic component, so
+      # a restart after a timed-out stop can't spawn a second sweep loop
+      # alongside it. Its own tick_once rescues and reports.
       def stop
         @mutex.synchronize do
           @done = true
           @sleeper.signal
         end
-        @thread&.join
-        @thread = nil
+        @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
       end
 
       def running?
@@ -116,8 +127,8 @@ module Wurk
       # jobs reclaimed (re-queued or killed). Public so boot paths and tests
       # can drive a deterministic pass without the cluster lock.
       def reclaim!
-        prefixes = live_process_prefixes
-        served_queues.sum { |public_q| reclaim_queue(public_q, prefixes) }
+        owners = live_owners
+        served_queues.sum { |public_q| reclaim_queue(public_q, owners) }
       end
 
       # One unguarded full-keyspace sweep: every `queue:*|*` private list, even
@@ -125,10 +136,10 @@ module Wurk
       # of jobs reclaimed. Public so boot paths and tests can drive it without
       # the hourly lock.
       def reclaim_full!
-        prefixes = live_process_prefixes
+        owners = live_owners
         reclaimed = 0
-        each_full_private_list do |key, public_q, host, pid|
-          next if owner_alive?(host, pid, prefixes)
+        each_full_private_list do |key, public_q, host, pid, nonce|
+          next if owner_alive?(host, pid, nonce, owners)
 
           reclaimed += drain(key, public_q)
         end
@@ -148,39 +159,43 @@ module Wurk
       end
 
       # SCAN for this public queue's private lists, reclaim the orphaned ones.
-      def reclaim_queue(public_q, prefixes)
+      def reclaim_queue(public_q, owners)
         reclaimed = 0
-        each_private_list(public_q) do |key, host, pid|
-          next if owner_alive?(host, pid, prefixes)
+        each_private_list(public_q) do |key, host, pid, nonce|
+          next if owner_alive?(host, pid, nonce, owners)
 
           reclaimed += drain(key, public_q)
         end
         reclaimed
       end
 
-      # Yields [private_list_key, host, pid] for each private list of
+      # Yields [private_list_key, host, pid, nonce] for each private list of
       # `public_q`. MATCH `<public_q>|*` matches only this queue's private
       # lists (public queue keys carry no `|`).
       def each_private_list(public_q)
         cursor = '0'
         loop do
-          cursor, keys = redis { |c| c.call('SCAN', cursor, 'MATCH', "#{public_q}|*", 'COUNT', SCAN_COUNT) }
+          cursor, keys = redis(idempotent: true) do |c|
+            c.call('SCAN', cursor, 'MATCH', "#{public_q}|*", 'COUNT', SCAN_COUNT)
+          end
           keys.each do |key|
-            host, pid = parse_owner(public_q, key)
-            yield key, host, pid if pid
+            host, pid, nonce = parse_owner(public_q, key)
+            yield key, host, pid, nonce if pid
           end
           break if cursor == '0'
         end
       end
 
-      # Yields [private_list_key, public_q, host, pid] for every private list in
+      # Yields [private_list_key, public_q, host, pid, nonce] for every list in
       # the keyspace. MATCH `queue:*|*` matches only private lists (public queue
       # keys carry no `|`); parse_full_key drops anything that isn't a
-      # well-formed `queue:<public>|<host>|<pid>|<idx>`.
+      # well-formed `queue:<public>|<host>|<pid>|<nonce>|<idx>`.
       def each_full_private_list
         cursor = '0'
         loop do
-          cursor, keys = redis { |c| c.call('SCAN', cursor, 'MATCH', "#{Keys::QUEUE_PREFIX}*|*", 'COUNT', SCAN_COUNT) }
+          cursor, keys = redis(idempotent: true) do |c|
+            c.call('SCAN', cursor, 'MATCH', "#{Keys::QUEUE_PREFIX}*|*", 'COUNT', SCAN_COUNT)
+          end
           keys.each do |key|
             parsed = parse_full_key(key)
             yield key, *parsed if parsed
@@ -189,45 +204,88 @@ module Wurk
         end
       end
 
-      # `queue:<public>|<host>|<pid>|<idx>` → [public_q, host, pid], parsed from
-      # the right (pid + idx are integers, host precedes them) so a `|` inside
-      # the queue name is tolerated. nil when the key isn't a well-formed
-      # private list.
+      # `queue:<public>|<host>|<pid>|<nonce>|<idx>` → [public_q, host, pid,
+      # nonce], parsed from the right so a `|` inside the queue name is
+      # tolerated. nil when the key isn't a well-formed private list.
+      #
+      # With no known prefix to split on, both owner shapes are tried in
+      # preference order and the first one leaving a real public queue behind
+      # wins. The narrow reading is what saves a pre-nonce key from an all-digit
+      # host (`queue:q|123456789012|<pid>|<idx>` — a bare Docker hostname is 12
+      # hex chars): read wide, its host segment eats the whole queue name.
       def parse_full_key(key)
         parts = key.split('|')
-        return nil if parts.size < 4
-
-        host, pid, idx = parts.last(3)
-        return nil unless integer?(pid) && integer?(idx)
+        owner_tails(parts).each do |host, pid, nonce, width|
+          public_q = parts[0...-width].join('|')
+          next unless public_q.start_with?(Keys::QUEUE_PREFIX) && public_q != Keys::QUEUE_PREFIX
 
-        public_q = parts[0...-3].join('|')
-        return nil unless public_q.start_with?(Keys::QUEUE_PREFIX) && public_q != Keys::QUEUE_PREFIX
-
-        [public_q, host, pid.to_i]
+          return [public_q, host, pid, nonce]
+        end
+        nil
       end
 
-      # `<public_q>|<host>|<pid>|<idx>` → [host, pid] (pid as Integer), or
-      # [nil, nil] when the suffix isn't a well-formed `host|pid|idx` triple.
-      # Splitting the suffix off the known public-queue prefix tolerates a
-      # `|` inside the queue name itself.
+      # `<public_q>|<host>|<pid>|<nonce>|<idx>` → [host, pid, nonce] (pid as
+      # Integer), or all-nil when the suffix isn't a well-formed owner tail.
+      # Splitting the suffix off the known public-queue prefix tolerates a `|`
+      # inside the queue name itself, and leaves the tail unambiguous: exactly
+      # 4 segments for the current shape, exactly 3 for the pre-nonce one.
       def parse_owner(public_q, key)
         suffix = key.delete_prefix("#{public_q}|")
-        return [nil, nil] if suffix == key
+        return [nil, nil, nil] if suffix == key
 
-        host, pid, idx = suffix.split('|')
-        return [nil, nil] unless host && integer?(pid) && integer?(idx)
+        host, pid, nonce = owner_tails(suffix.split('|')).first
+        [host, pid, nonce]
+      end
 
-        [host, pid.to_i]
+      # Owner segments of a private-list key, taken from the right, as
+      # [host, pid, nonce, segment_count] readings in preference order (empty
+      # when nothing parses). The wide shape is preferred: an all-digit nonce is
+      # rare but reachable, and reading such a key narrow would take the pid for
+      # the host and the nonce for the pid — draining a live owner's list out
+      # from under it.
+      def owner_tails(parts)
+        return [] unless parts.size >= 3 && integer?(parts[-1])
+
+        [wide_tail(parts), narrow_tail(parts)].compact
+      end
+
+      # `<host>|<pid>|<nonce>|<idx>` — the shape every current process writes.
+      def wide_tail(parts)
+        [parts[-4], parts[-3].to_i, parts[-2], 4] if parts.size >= 4 && integer?(parts[-3])
+      end
+
+      # `<host>|<pid>|<idx>` — written before the nonce existed. Such a list can
+      # still hold a pre-upgrade process's in-flight jobs across a rolling
+      # upgrade, so it stays reclaimable even though nothing writes it anymore.
+      def narrow_tail(parts)
+        [parts[-3], parts[-2].to_i, nil, 3] if integer?(parts[-2])
       end
 
       def integer?(str)
         str.is_a?(String) && str.match?(/\A\d+\z/)
       end
 
-      def owner_alive?(host, pid, prefixes)
-        return local_pid_alive?(pid) if host == hostname
-
-        prefixes.include?("#{host}:#{pid}")
+      # `Process.kill(0, pid)` answers "does this pid exist *in my PID
+      # namespace*", which is the question we're actually asking only when the
+      # key was written from that same namespace. A shared hostname does not
+      # imply it: a container restarting under a fixed hostname comes back in a
+      # fresh namespace where the dead owner's pid is likely taken again (list
+      # read as live, jobs stranded forever), and two containers sharing a
+      # host's network namespace but not its pid namespace each hold pids the
+      # other lacks (live owner read as dead, list drained mid-job, duplicate
+      # run). So kill(0) can serve as neither a positive nor a negative signal
+      # off our own namespace.
+      #
+      # The nonce settles it: minted once per process image and inherited across
+      # fork, so a key carrying ours provably came from this very process tree.
+      # Every other owner goes through the namespace-blind heartbeat — alive iff
+      # its identity is a live `processes` member. A pre-nonce key can only be
+      # matched on the `<host>:<pid>` prefix of that identity.
+      def owner_alive?(host, pid, nonce, owners)
+        return local_pid_alive?(pid) if nonce == process_nonce && host == hostname
+        return owners.include?("#{host}:#{pid}:#{nonce}") if nonce
+
+        owners.include?("#{host}:#{pid}")
       end
 
       def local_pid_alive?(pid)
@@ -239,18 +297,21 @@ module Wurk
         true
       end
 
-      # `host:pid` of every live process — a member of `processes` whose
-      # `info` hash still exists. A bare SET membership isn't enough: the
-      # member lingers after its 60s hash TTL until ProcessSet#cleanup prunes
-      # it, and we must treat that window as dead for cross-host reclaim.
-      def live_process_prefixes
-        redis do |conn|
+      # Every live process indexed both ways: the full `<host>:<pid>:<nonce>`
+      # identity a nonce-bearing private list is matched on, and the
+      # `<host>:<pid>` prefix a pre-nonce one has to settle for. Live means a
+      # member of `processes` whose `info` hash still exists — a bare SET
+      # membership isn't enough, since the member lingers after its 60s hash
+      # TTL until ProcessSet#cleanup prunes it and that window must read as
+      # dead or the owner's jobs are never reclaimed.
+      def live_owners
+        redis(idempotent: true) do |conn|
           members = conn.call('SMEMBERS', Keys::PROCESSES)
           next ::Set.new if members.empty?
 
           infos = conn.pipelined { |pipe| members.each { |m| pipe.call('HGET', m, 'info') } }
           members.zip(infos).each_with_object(::Set.new) do |(member, info), set|
-            set << host_pid(member) if info
+            set << member << host_pid(member) if info
           end
         end
       end
@@ -265,6 +326,12 @@ module Wurk
       # poison check, so a crash mid-drain leaves the job safely in the public
       # queue (at-least-once), never lost. Poison jobs are killed to the dead
       # set by PoisonPill.track! and then LREM'd out of the public queue.
+      #
+      # Unlike the fetcher's LMOVE, this one does *not* claim apply-safety: a
+      # replay after a lost reply moves the next job and never poison-checks the
+      # one already on the public tail, so a job that kills its worker every time
+      # would get a free recovery past the cap. A raise instead lands in the
+      # rescue below, and the next sweep re-drains what's left.
       def drain(private_list, public_q)
         queue_name = public_q.delete_prefix(Keys::QUEUE_PREFIX)
         count = 0