wurk 1.3.0 → 1.4.0

data/lib/wurk/cron.rb

@@ -6,6 +6,7 @@ require 'time'
 require_relative 'component'
 require_relative 'client'
 require_relative 'leader'
+require_relative 'timer_loop'
 
 module Wurk
   # Sidekiq Enterprise periodic jobs. Pure leader-driven cron — only the
@@ -477,8 +478,8 @@ module Wurk
 
       private
 
-      def redis(&)
-        @config ? @config.redis(&) : Wurk.redis(&)
+      def redis(idempotent: false, &)
+        @config ? @config.redis(idempotent:, &) : Wurk.redis(idempotent:, &)
       end
     end
 
@@ -520,34 +521,35 @@ module Wurk
 
       def initialize(config)
         @config = config
-        @done = false
-        @mutex = ::Mutex.new
-        @sleeper = ::ConditionVariable.new
         @client = Client.new(config: config)
         @thread = nil
         # Operators never need to touch this; integration tests shrink it so a
         # due loop fires within the test window instead of waiting a full minute.
         @tick_interval = config[:cron_tick_interval] || DEFAULT_TICK_SECONDS
+        @timer = TimerLoop.new(@tick_interval)
       end
 
+      # TimerLoop waits one interval before the first tick: don't fire a
+      # catch-up burst the instant we boot (the leader is barely settled), and
+      # let a short-lived process exit without ticking at all.
       def start
-        @poller_thread ||= safe_thread('cron-poller') do # rubocop:disable Naming/MemoizedInstanceVariableName
-          # Wait one interval before the first tick: don't fire a catch-up burst
-          # the instant we boot (the leader is barely settled), and let a
-          # short-lived process exit without ticking at all.
-          wait
-          until @done
-            tick
-            wait
-          end
-        end
+        return @thread if @thread
+
+        @timer.reset
+        @thread = safe_thread('cron-poller') { @timer.run { tick } }
       end
 
+      # Blocks until the thread is really gone: the launcher releases the
+      # cluster lock immediately after this returns, and a tick still in flight
+      # would enqueue loops the next leader is about to fire itself.
+      #
+      # Cleared only on a confirmed join (Thread#join returns nil on timeout):
+      # a wedged thread must stay tracked so #start's guard returns it instead
+      # of calling @timer.reset, which would un-terminate the loop it is still
+      # inside and leave two tick threads double-enqueuing the same loops.
       def terminate
-        @mutex.synchronize do
-          @done = true
-          @sleeper.signal
-        end
+        @timer.terminate
+        @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
       end
 
       # Leader-gated by the single cluster lock (Component#leader? reads
@@ -562,18 +564,23 @@ module Wurk
         handle_exception(e, { context: 'cron-poller' })
       end
 
+      # The marks advance *before* the push, via CAS: the leader gate is a
+      # cached read, so a second poller can reach the same due loop for a few
+      # seconds after a handover. Losing the CAS means another tick owns this
+      # slot — enqueue nothing.
       def enqueue_if_due(loop_obj)
         return if loop_obj.paused?
 
         now = ::Time.now.to_i
-        prev_fire, next_fire = read_fire_marks(loop_obj.lid)
-        next_fire ||= loop_obj.next_fire_at(prev_fire || (now - @tick_interval))
-        return if next_fire.nil? || next_fire > now
+        slot, mark = due_slot(loop_obj, now)
+        return if slot.nil?
 
-        warn_missed_tick(loop_obj, next_fire, now)
-        jid = enqueue!(loop_obj)
-        future = loop_obj.next_fire_after(next_fire, now)
-        record_fire(loop_obj, jid, now, future)
+        future = loop_obj.next_fire_after(slot, now)
+        return unless claim_fire?(loop_obj, mark, now, future)
+
+        warn_missed_tick(loop_obj, slot, now)
+        jid = enqueue_claimed!(loop_obj)
+        record_history(loop_obj, jid, now)
         jid
       end
 
@@ -590,10 +597,16 @@ module Wurk
 
       private
 
-      def wait
-        @mutex.synchronize do
-          @sleeper.wait(@mutex, @tick_interval) unless @done
-        end
+      # The slot this tick would fire and the token that claims it, or nil when
+      # the loop has no resolvable occurrence or none is due yet. The token is
+      # the stored `nf` when there is one, otherwise the slot we derived — see
+      # the `cron_claim_fire` script for how the two cases differ.
+      def due_slot(loop_obj, now)
+        prev_fire, next_fire, mark = read_fire_marks(loop_obj.lid)
+        next_fire ||= loop_obj.next_fire_at(prev_fire || (now - @tick_interval))
+        return if next_fire.nil? || next_fire > now
+
+        [next_fire, mark || next_fire.to_s]
       end
 
       def warn_missed_tick(loop_obj, expected, now)
@@ -605,6 +618,34 @@ module Wurk
         )
       end
 
+      # Compare-and-swap the fire marks: only the tick still looking at the
+      # mark it read wins the slot. Manual `#fire` deliberately skips this —
+      # an operator-requested run contends with no schedule slot.
+      def claim_fire?(loop_obj, mark, fired_at, future)
+        @config.redis do |c|
+          Wurk::Lua::Loader.eval_cached(
+            c, :cron_claim_fire,
+            keys: ["#{LOOP_PREFIX}#{loop_obj.lid}"],
+            argv: [mark, fired_at.to_s, future.nil? ? '' : future.to_s]
+          )
+        end == 1
+      end
+
+      # The slot is already claimed, so a failed push loses this occurrence
+      # instead of leaving the next tick to fire it again. Ent leader election
+      # is explicitly best-effort coordination (sidekiq-ent.md §6), so a missed
+      # occurrence is tolerable where a duplicate one is not — log the gap so it
+      # stays attributable.
+      def enqueue_claimed!(loop_obj)
+        enqueue!(loop_obj)
+      rescue StandardError => e
+        logger.warn(
+          "[cron] fire lost lid=#{loop_obj.lid} klass=#{loop_obj.klass} " \
+          "mark already advanced, occurrence skipped: #{e.class}: #{e.message}"
+        )
+        raise
+      end
+
       def enqueue!(loop_obj)
         aj = active_job_class(loop_obj.klass)
         return enqueue_active_job(aj, loop_obj) if aj
@@ -643,24 +684,40 @@ module Wurk
         job.provider_job_id if job.respond_to?(:provider_job_id)
       end
 
+      # Third element is the raw `nf` string — the CAS token for #claim_fire.
+      # Comparing the bytes Redis holds, not the parsed integer, keeps the
+      # token exact regardless of how the value was written.
       def read_fire_marks(lid)
         @config.redis do |c|
-          vals = c.call('HMGET', "#{LOOP_PREFIX}#{lid}", 'lf', 'nf')
-          next_fire = vals[1]
+          prev_fire, next_fire = c.call('HMGET', "#{LOOP_PREFIX}#{lid}", 'lf', 'nf')
           # Preserve nil: treat missing or empty 'nf' as nil, not 0.
-          [vals[0]&.to_i, next_fire.nil? || next_fire.empty? ? nil : next_fire.to_i]
+          next_fire = nil if next_fire.nil? || next_fire.empty?
+          [prev_fire&.to_i, next_fire&.to_i, next_fire]
         end
       end
 
       def record_fire(loop_obj, jid, fired_at, future)
-        history_entry = Wurk.dump_json([fired_at, jid])
+        write_fire_marks(loop_obj.lid, fired_at, future)
+        record_history(loop_obj, jid, fired_at)
+      end
+
+      # Unconditional mark write, for the manual `#fire` path only. The
+      # scheduled path advances the marks through #claim_fire instead.
+      def write_fire_marks(lid, fired_at, future)
+        key = "#{LOOP_PREFIX}#{lid}"
         @config.redis do |c|
           if future.nil?
-            c.call('HSET', "#{LOOP_PREFIX}#{loop_obj.lid}", 'lf', fired_at.to_s)
-            c.call('HDEL', "#{LOOP_PREFIX}#{loop_obj.lid}", 'nf')
+            c.call('HSET', key, 'lf', fired_at.to_s)
+            c.call('HDEL', key, 'nf')
           else
-            c.call('HSET', "#{LOOP_PREFIX}#{loop_obj.lid}", 'lf', fired_at.to_s, 'nf', future.to_s)
+            c.call('HSET', key, 'lf', fired_at.to_s, 'nf', future.to_s)
           end
+        end
+      end
+
+      def record_history(loop_obj, jid, fired_at)
+        history_entry = Wurk.dump_json([fired_at, jid])
+        @config.redis do |c|
           c.call('LPUSH', "#{HISTORY_PREFIX}#{loop_obj.lid}", history_entry)
           c.call('LTRIM', "#{HISTORY_PREFIX}#{loop_obj.lid}", 0, HISTORY_CAP - 1)
         end

data/lib/wurk/deploy.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'pool_checkout'
+
 module Wurk
   # Records deploy markers into Redis so the history pane can overlay
   # "deployed at" lines onto job-throughput charts. Wire-compatible with
@@ -84,11 +86,11 @@ module Wurk
       nil
     end
 
-    def with_redis(&)
+    def with_redis(idempotent: false, &)
       if @pool
-        @pool.with(&)
+        PoolCheckout.with(@pool, idempotent, &)
       else
-        Wurk.redis(&)
+        Wurk.redis(idempotent:, &)
       end
     end
   end

data/lib/wurk/embedded.rb

@@ -41,6 +41,12 @@ module Wurk
       sleep 0.2
       logger.info { "Wurk running embedded, total process thread count: #{Thread.list.size}" }
       logger.debug { Thread.list.map(&:name).to_s }
+    rescue StandardError
+      # The host is left believing Wurk never started, so anything the launcher
+      # did get up would keep fetching, beating and campaigning for the leader
+      # lock unowned for the life of the process.
+      roll_back_partial_boot
+      raise
     end
 
     # Stop fetching new work; in-flight jobs continue.
@@ -56,6 +62,13 @@ module Wurk
 
     private
 
+    # Guarded so the caller sees why the boot failed, not why the rollback did.
+    def roll_back_partial_boot
+      stop
+    rescue StandardError => e
+      handle_exception(e, { context: 'embedded-boot-rollback' })
+    end
+
     # Extracted so tests can swap in a fake launcher without monkey-patching
     # Wurk::Launcher.new globally (which races other parallel tests).
     def build_launcher

data/lib/wurk/fetcher/reaper.rb

@@ -9,7 +9,7 @@ module Wurk
     # Orphan reclamation for the reliable fetcher (Pro super_fetch §3.2).
     #
     # The Reliable fetcher moves each job from a public queue into a
-    # per-process private list (`queue:<public>|<host>|<pid>|<idx>`) and
+    # per-process private list (`queue:<public>|<host>|<pid>|<nonce>|<idx>`) and
     # leaves it there until the Processor ACKs. A SIGKILLed or crashed
     # worker therefore strands its in-flight jobs in private lists that
     # nobody will ever ACK. The Reaper is the recovery half: it periodically
@@ -17,15 +17,16 @@ module Wurk
     # moves their jobs back to the public queue so a live worker re-runs them.
     #
     # Liveness is decided per owner:
-    #   * same host — the OS is authoritative: `Process.kill(0, pid)`. This
-    #     is instant and ignores a stale `processes` SET entry whose 60s TTL
-    #     hasn't lapsed yet, so a `kill -9`ed sibling is reclaimed the moment
-    #     the supervisor reaps it rather than 60s later. (Pid reuse by an
-    #     unrelated local process is the one blind spot — the supervisor
-    #     respawns with a fresh pid, so it does not arise in practice.)
-    #   * other host — we cannot ping the pid, so we trust the heartbeat:
-    #     the owner is alive iff some live `processes` member (one whose
-    #     `info` hash still exists) shares its `host:pid`. Cross-host reclaim
+    #   * our own host and nonce — the pid was minted in our PID namespace, so
+    #     the OS is authoritative: `Process.kill(0, pid)`. This is instant and
+    #     ignores a stale `processes` SET entry whose 60s TTL hasn't lapsed
+    #     yet, so a `kill -9`ed sibling is reclaimed the moment the supervisor
+    #     reaps it rather than 60s later. (Pid reuse by an unrelated process in
+    #     the same tree is the one blind spot — the supervisor respawns with a
+    #     fresh pid, so it does not arise in practice.)
+    #   * any other incarnation — its pid means nothing in our namespace, so we
+    #     trust the heartbeat: the owner is alive iff its identity is a live
+    #     `processes` member (one whose `info` hash still exists). Such reclaim
     #     therefore waits out the 60s heartbeat TTL, exactly as the spec says.
     #
     # Re-pushed jobs run through Wurk::Middleware::PoisonPill, which caps a
@@ -116,8 +117,8 @@ module Wurk
       # jobs reclaimed (re-queued or killed). Public so boot paths and tests
       # can drive a deterministic pass without the cluster lock.
       def reclaim!
-        prefixes = live_process_prefixes
-        served_queues.sum { |public_q| reclaim_queue(public_q, prefixes) }
+        owners = live_owners
+        served_queues.sum { |public_q| reclaim_queue(public_q, owners) }
       end
 
       # One unguarded full-keyspace sweep: every `queue:*|*` private list, even
@@ -125,10 +126,10 @@ module Wurk
       # of jobs reclaimed. Public so boot paths and tests can drive it without
       # the hourly lock.
       def reclaim_full!
-        prefixes = live_process_prefixes
+        owners = live_owners
         reclaimed = 0
-        each_full_private_list do |key, public_q, host, pid|
-          next if owner_alive?(host, pid, prefixes)
+        each_full_private_list do |key, public_q, host, pid, nonce|
+          next if owner_alive?(host, pid, nonce, owners)
 
           reclaimed += drain(key, public_q)
         end
@@ -148,39 +149,43 @@ module Wurk
       end
 
       # SCAN for this public queue's private lists, reclaim the orphaned ones.
-      def reclaim_queue(public_q, prefixes)
+      def reclaim_queue(public_q, owners)
         reclaimed = 0
-        each_private_list(public_q) do |key, host, pid|
-          next if owner_alive?(host, pid, prefixes)
+        each_private_list(public_q) do |key, host, pid, nonce|
+          next if owner_alive?(host, pid, nonce, owners)
 
           reclaimed += drain(key, public_q)
         end
         reclaimed
       end
 
-      # Yields [private_list_key, host, pid] for each private list of
+      # Yields [private_list_key, host, pid, nonce] for each private list of
       # `public_q`. MATCH `<public_q>|*` matches only this queue's private
       # lists (public queue keys carry no `|`).
       def each_private_list(public_q)
         cursor = '0'
         loop do
-          cursor, keys = redis { |c| c.call('SCAN', cursor, 'MATCH', "#{public_q}|*", 'COUNT', SCAN_COUNT) }
+          cursor, keys = redis(idempotent: true) do |c|
+            c.call('SCAN', cursor, 'MATCH', "#{public_q}|*", 'COUNT', SCAN_COUNT)
+          end
           keys.each do |key|
-            host, pid = parse_owner(public_q, key)
-            yield key, host, pid if pid
+            host, pid, nonce = parse_owner(public_q, key)
+            yield key, host, pid, nonce if pid
           end
           break if cursor == '0'
         end
       end
 
-      # Yields [private_list_key, public_q, host, pid] for every private list in
+      # Yields [private_list_key, public_q, host, pid, nonce] for every list in
       # the keyspace. MATCH `queue:*|*` matches only private lists (public queue
       # keys carry no `|`); parse_full_key drops anything that isn't a
-      # well-formed `queue:<public>|<host>|<pid>|<idx>`.
+      # well-formed `queue:<public>|<host>|<pid>|<nonce>|<idx>`.
       def each_full_private_list
         cursor = '0'
         loop do
-          cursor, keys = redis { |c| c.call('SCAN', cursor, 'MATCH', "#{Keys::QUEUE_PREFIX}*|*", 'COUNT', SCAN_COUNT) }
+          cursor, keys = redis(idempotent: true) do |c|
+            c.call('SCAN', cursor, 'MATCH', "#{Keys::QUEUE_PREFIX}*|*", 'COUNT', SCAN_COUNT)
+          end
           keys.each do |key|
             parsed = parse_full_key(key)
             yield key, *parsed if parsed
@@ -189,45 +194,88 @@ module Wurk
         end
       end
 
-      # `queue:<public>|<host>|<pid>|<idx>` → [public_q, host, pid], parsed from
-      # the right (pid + idx are integers, host precedes them) so a `|` inside
-      # the queue name is tolerated. nil when the key isn't a well-formed
-      # private list.
+      # `queue:<public>|<host>|<pid>|<nonce>|<idx>` → [public_q, host, pid,
+      # nonce], parsed from the right so a `|` inside the queue name is
+      # tolerated. nil when the key isn't a well-formed private list.
+      #
+      # With no known prefix to split on, both owner shapes are tried in
+      # preference order and the first one leaving a real public queue behind
+      # wins. The narrow reading is what saves a pre-nonce key from an all-digit
+      # host (`queue:q|123456789012|<pid>|<idx>` — a bare Docker hostname is 12
+      # hex chars): read wide, its host segment eats the whole queue name.
       def parse_full_key(key)
         parts = key.split('|')
-        return nil if parts.size < 4
-
-        host, pid, idx = parts.last(3)
-        return nil unless integer?(pid) && integer?(idx)
+        owner_tails(parts).each do |host, pid, nonce, width|
+          public_q = parts[0...-width].join('|')
+          next unless public_q.start_with?(Keys::QUEUE_PREFIX) && public_q != Keys::QUEUE_PREFIX
 
-        public_q = parts[0...-3].join('|')
-        return nil unless public_q.start_with?(Keys::QUEUE_PREFIX) && public_q != Keys::QUEUE_PREFIX
-
-        [public_q, host, pid.to_i]
+          return [public_q, host, pid, nonce]
+        end
+        nil
       end
 
-      # `<public_q>|<host>|<pid>|<idx>` → [host, pid] (pid as Integer), or
-      # [nil, nil] when the suffix isn't a well-formed `host|pid|idx` triple.
-      # Splitting the suffix off the known public-queue prefix tolerates a
-      # `|` inside the queue name itself.
+      # `<public_q>|<host>|<pid>|<nonce>|<idx>` → [host, pid, nonce] (pid as
+      # Integer), or all-nil when the suffix isn't a well-formed owner tail.
+      # Splitting the suffix off the known public-queue prefix tolerates a `|`
+      # inside the queue name itself, and leaves the tail unambiguous: exactly
+      # 4 segments for the current shape, exactly 3 for the pre-nonce one.
       def parse_owner(public_q, key)
         suffix = key.delete_prefix("#{public_q}|")
-        return [nil, nil] if suffix == key
+        return [nil, nil, nil] if suffix == key
+
+        host, pid, nonce = owner_tails(suffix.split('|')).first
+        [host, pid, nonce]
+      end
+
+      # Owner segments of a private-list key, taken from the right, as
+      # [host, pid, nonce, segment_count] readings in preference order (empty
+      # when nothing parses). The wide shape is preferred: an all-digit nonce is
+      # rare but reachable, and reading such a key narrow would take the pid for
+      # the host and the nonce for the pid — draining a live owner's list out
+      # from under it.
+      def owner_tails(parts)
+        return [] unless parts.size >= 3 && integer?(parts[-1])
+
+        [wide_tail(parts), narrow_tail(parts)].compact
+      end
 
-        host, pid, idx = suffix.split('|')
-        return [nil, nil] unless host && integer?(pid) && integer?(idx)
+      # `<host>|<pid>|<nonce>|<idx>` — the shape every current process writes.
+      def wide_tail(parts)
+        [parts[-4], parts[-3].to_i, parts[-2], 4] if parts.size >= 4 && integer?(parts[-3])
+      end
 
-        [host, pid.to_i]
+      # `<host>|<pid>|<idx>` — written before the nonce existed. Such a list can
+      # still hold a pre-upgrade process's in-flight jobs across a rolling
+      # upgrade, so it stays reclaimable even though nothing writes it anymore.
+      def narrow_tail(parts)
+        [parts[-3], parts[-2].to_i, nil, 3] if integer?(parts[-2])
       end
 
       def integer?(str)
         str.is_a?(String) && str.match?(/\A\d+\z/)
       end
 
-      def owner_alive?(host, pid, prefixes)
-        return local_pid_alive?(pid) if host == hostname
-
-        prefixes.include?("#{host}:#{pid}")
+      # `Process.kill(0, pid)` answers "does this pid exist *in my PID
+      # namespace*", which is the question we're actually asking only when the
+      # key was written from that same namespace. A shared hostname does not
+      # imply it: a container restarting under a fixed hostname comes back in a
+      # fresh namespace where the dead owner's pid is likely taken again (list
+      # read as live, jobs stranded forever), and two containers sharing a
+      # host's network namespace but not its pid namespace each hold pids the
+      # other lacks (live owner read as dead, list drained mid-job, duplicate
+      # run). So kill(0) can serve as neither a positive nor a negative signal
+      # off our own namespace.
+      #
+      # The nonce settles it: minted once per process image and inherited across
+      # fork, so a key carrying ours provably came from this very process tree.
+      # Every other owner goes through the namespace-blind heartbeat — alive iff
+      # its identity is a live `processes` member. A pre-nonce key can only be
+      # matched on the `<host>:<pid>` prefix of that identity.
+      def owner_alive?(host, pid, nonce, owners)
+        return local_pid_alive?(pid) if nonce == process_nonce && host == hostname
+        return owners.include?("#{host}:#{pid}:#{nonce}") if nonce
+
+        owners.include?("#{host}:#{pid}")
       end
 
       def local_pid_alive?(pid)
@@ -239,18 +287,21 @@ module Wurk
         true
       end
 
-      # `host:pid` of every live process — a member of `processes` whose
-      # `info` hash still exists. A bare SET membership isn't enough: the
-      # member lingers after its 60s hash TTL until ProcessSet#cleanup prunes
-      # it, and we must treat that window as dead for cross-host reclaim.
-      def live_process_prefixes
-        redis do |conn|
+      # Every live process indexed both ways: the full `<host>:<pid>:<nonce>`
+      # identity a nonce-bearing private list is matched on, and the
+      # `<host>:<pid>` prefix a pre-nonce one has to settle for. Live means a
+      # member of `processes` whose `info` hash still exists — a bare SET
+      # membership isn't enough, since the member lingers after its 60s hash
+      # TTL until ProcessSet#cleanup prunes it and that window must read as
+      # dead or the owner's jobs are never reclaimed.
+      def live_owners
+        redis(idempotent: true) do |conn|
           members = conn.call('SMEMBERS', Keys::PROCESSES)
           next ::Set.new if members.empty?
 
           infos = conn.pipelined { |pipe| members.each { |m| pipe.call('HGET', m, 'info') } }
           members.zip(infos).each_with_object(::Set.new) do |(member, info), set|
-            set << host_pid(member) if info
+            set << member << host_pid(member) if info
           end
         end
       end
@@ -265,6 +316,12 @@ module Wurk
       # poison check, so a crash mid-drain leaves the job safely in the public
       # queue (at-least-once), never lost. Poison jobs are killed to the dead
       # set by PoisonPill.track! and then LREM'd out of the public queue.
+      #
+      # Unlike the fetcher's LMOVE, this one does *not* claim apply-safety: a
+      # replay after a lost reply moves the next job and never poison-checks the
+      # one already on the public tail, so a job that kills its worker every time
+      # would get a free recovery past the cap. A raise instead lands in the
+      # rescue below, and the next sweep re-drains what's left.
       def drain(private_list, public_q)
         queue_name = public_q.delete_prefix(Keys::QUEUE_PREFIX)
         count = 0

data/lib/wurk/fetcher/reliable.rb

@@ -5,11 +5,12 @@ require_relative '../component'
 require_relative '../keys'
 require_relative '../lua'
 require_relative '../fetcher'
+require_relative '../middleware/poison_pill'
 
 module Wurk
   class Fetcher
     # Default fetcher. Each public queue is paired with a per-process
-    # private list (`queue:<name>|<host>|<pid>|<idx>`); a job is moved
+    # private list (`queue:<name>|<host>|<pid>|<nonce>|<idx>`); a job is moved
     # atomically from the public tail to the private head via LMOVE, and
     # stays there until the Processor explicitly ACKs (LREM). SIGKILL
     # between fetch and ack leaves the job in the private list, where the
@@ -30,15 +31,38 @@ module Wurk
       # Default BLMOVE block timeout; overridable via config.fetch_poll_interval.
       TIMEOUT = 2
 
+      # Backoff for the quieted short-circuit. Manager#quiet terminates the
+      # shared fetcher before it terminates the processors, and Processor#run
+      # loops on its *own* flag — so in that window every processor would spin
+      # on an instant nil. Kept below Manager::PAUSE_TIME, which #stop sleeps
+      # immediately after #quiet, so this pause adds no drain latency.
+      QUIET_PAUSE = 0.05
+
       # Carries the public queue key, the raw (still-JSON) job payload,
       # and the capsule we use to reach Redis. ACK removes from the private
       # list; requeue pushes back to the public queue head so the job is
       # next pulled. LREM count=1 is idempotent for our payloads since
       # each job's JSON contains a unique `jid`.
-      UnitOfWork = Struct.new(:queue, :job, :config, keyword_init: true) do
+      #
+      # `jid` is filled in by the Processor once it has parsed the payload —
+      # the fetcher never parses. It is only used to retire the job's
+      # poison-pill recovery counter, so an ACK without one is still a
+      # complete ACK.
+      UnitOfWork = Struct.new(:queue, :job, :config, :jid, keyword_init: true) do
+        # The counter DEL rides this round trip rather than taking one of its
+        # own: the ACK is the only Redis call the success path makes, and a
+        # per-job call would be a fetch+execute regression for the sake of a
+        # key that exists for roughly no jobs. See Middleware::PoisonPill.
         def acknowledge
+          private_list = Reliable.private_queue_name(queue)
+          job_jid = jid.to_s
+          return config.redis { |conn| conn.call('LREM', private_list, 1, job) } if job_jid.empty?
+
           config.redis do |conn|
-            conn.call('LREM', Reliable.private_queue_name(queue), 1, job)
+            conn.pipelined do |pipe|
+              pipe.call('LREM', private_list, 1, job)
+              Middleware::PoisonPill.clear_in(pipe, job_jid)
+            end
           end
         end
 
@@ -55,9 +79,16 @@ module Wurk
       # carrying a back-reference to its parent fetcher. Index defaults to
       # 0 — we run one fetcher per capsule today. Multi-processor topology
       # (one private list per processor slot) is a future Manager concern.
+      #
+      # The nonce marks the incarnation. host+pid alone is ambiguous once PID
+      # namespaces are in play: a restarted container reuses both, so the
+      # reaper's `kill(0)` liveness check would read a dead owner's list as
+      # live (jobs stranded) or a live owner's as dead (job run twice). Keys
+      # written before the nonce existed stay reclaimable — Reaper#parse_owner
+      # accepts both shapes.
       def self.private_queue_name(public_queue, index = 0)
         host = ENV['DYNO'] || Socket.gethostname
-        "#{public_queue}|#{host}|#{::Process.pid}|#{index}"
+        "#{public_queue}|#{host}|#{::Process.pid}|#{Component::PROCESS_NONCE}|#{index}"
       end
 
       def initialize(capsule)
@@ -66,11 +97,26 @@ module Wurk
         @done = false
       end
 
+      # Every pass that yields no job has to cost wall-clock time: Processor#run
+      # drives `process_one` in a bare `until @done` loop with no pause of its
+      # own, so any nil returned instantly turns N processor threads into a hot
+      # loop. The blocking BLMOVE pays that cost on the normal empty-queue path;
+      # the two short-circuits below have to pay it themselves.
       def retrieve_work
-        return nil if @done
+        if @done
+          sleep QUIET_PAUSE
+          return nil
+        end
 
         queues = queues_cmd
-        return nil if queues.empty?
+        # Nothing fetchable — every queue paused, or none configured. Back off a
+        # full poll interval rather than re-running queues_cmd (an SMEMBERS per
+        # pass, on the main pool) as fast as the CPU allows. Mirrors Sidekiq's
+        # BasicFetch guard, upstream #4825.
+        if queues.empty?
+          sleep poll_interval
+          return nil
+        end
 
         queues.each do |public_q|
           uow = lmove(public_q)
@@ -148,12 +194,19 @@ module Wurk
       # is dominated by the BLMOVE that follows. Returns a Set for O(1)
       # lookup against the (often weighted-expanded) queue list.
       def paused_names
-        config.redis { |conn| conn.call('SMEMBERS', Keys::PAUSED_SET) }.to_set
+        config.redis(idempotent: true) { |conn| conn.call('SMEMBERS', Keys::PAUSED_SET) }.to_set
       end
 
+      # Both LMOVE forms claim apply-safety, so fetch keeps the full
+      # connection-blip backoff the F5 split otherwise takes away: a move that
+      # applied but whose reply was lost leaves the job in *this* process's
+      # private list, un-ACKed — byte-for-byte the state a SIGKILL between fetch
+      # and ack leaves behind, which the next boot's Reaper already reclaims. The
+      # replay then pulls a different job; nothing duplicates and nothing is
+      # lost, at worst one job waits out this process's lifetime.
       def lmove(public_q)
         priv = self.class.private_queue_name(public_q)
-        job = config.redis { |conn| conn.call('LMOVE', public_q, priv, 'RIGHT', 'LEFT') }
+        job = config.redis(idempotent: true) { |conn| conn.call('LMOVE', public_q, priv, 'RIGHT', 'LEFT') }
         job ? UnitOfWork.new(queue: public_q, job: job, config: config) : nil
       end
 
@@ -165,7 +218,7 @@ module Wurk
         # starving the main pool's background loops (#101). Extend the socket
         # read-timeout one second past BLMOVE's own server-side timeout so the
         # connection's read timeout can't fire while BLMOVE is legitimately blocked.
-        job = config.fetch_redis do |conn|
+        job = config.fetch_redis(idempotent: true) do |conn|
           conn.blocking_call(timeout + 1, 'BLMOVE', public_q, priv, 'RIGHT', 'LEFT', timeout)
         end
         job ? UnitOfWork.new(queue: public_q, job: job, config: config) : nil