wurk 1.3.0 → 1.4.0

data/lib/wurk/limiter.rb

@@ -4,6 +4,7 @@ require 'json'
 require 'digest'
 require 'securerandom'
 require_relative 'lua'
+require_relative 'pool_checkout'
 
 module Wurk
   # Sidekiq Enterprise rate limiters: concurrent, bucket, window, leaky,
@@ -24,7 +25,10 @@ module Wurk
   # Layout (one file per type under `lib/wurk/limiter/`):
   #   * `Limiter::Base` owns the metadata write (lmtr:{name}) + the global
   #     `lmtr-list` registration so the Web UI can list every limiter, and
-  #     the uniform `status` shape.
+  #     the uniform `status` shape. Membership is added once per metadata
+  #     write and swept back out by `Web::Enterprise::Limits.list` once the
+  #     metadata expires — the two halves of keeping `lmtr-list` bounded
+  #     under interpolated names.
   #   * Per-type subclasses (Concurrent / Bucket / Window / Leaky / Points)
   #     own their acquire/wait loop. Each delegates the atomic step to a
   #     Lua script in `lib/wurk/lua/limiter_*.lua`.
@@ -146,9 +150,8 @@ module Wurk
       # Redis access: caller-supplied pool (Limiter.configure.redis = …) wins,
       # else fall back to the default Wurk pool. This is the same hierarchy
       # Sidekiq Ent documents — dedicated rate-limiter pool is opt-in.
-      def redis(&)
-        pool = config.pool || Wurk.redis_pool
-        pool.with(&)
+      def redis(idempotent: false, &)
+        PoolCheckout.with(config.pool || Wurk.redis_pool, idempotent, &)
       end
 
       # `ZRANGE key 0 0 WITHSCORES` yields a single [member, score] pair, but the

data/lib/wurk/lua.rb

@@ -131,8 +131,16 @@ module Wurk
     # success after the dead job is manually retried to success). The
     # `b-<bid>-death` notify dedup key is untouched, so `:death` cannot
     # re-fire.
+    #
+    # The two EXPIRE NX calls are what keep the batch bounded: `b-<bid>-jids` is
+    # born here, and only `Batch#ensure_first_flush!` ever stamped a TTL — on the
+    # hash alone — so an abandoned or invalidated batch leaked its live-jid set
+    # forever. A late re-push (retry, scheduled promotion) can also resurrect an
+    # already-expired `b-<bid>` through the HINCRBYs above, again TTL-less. NX
+    # stamps only keys that currently have no TTL, so a running batch's clock and
+    # the shorter post-success `linger` window (Callbacks#apply_linger) both win.
     # KEYS = [b-<bid>, b-<bid>-jids, queue_list, queues_set, b-<bid>-died, dead-batches]
-    # ARGV = [queue_name, jid, job_json, bid]
+    # ARGV = [queue_name, jid, job_json, bid, expiry_seconds]
     # Returns 1.
     BATCH_PUSH = <<~LUA
       if redis.call("srem", KEYS[5], ARGV[2]) == 1 then
@@ -147,6 +155,8 @@ module Wurk
           redis.call("hincrby", KEYS[1], "pending", 1)
         end
       end
+      redis.call("expire", KEYS[1], ARGV[5], "NX")
+      redis.call("expire", KEYS[2], ARGV[5], "NX")
       redis.call("sadd", KEYS[4], ARGV[1])
       redis.call("lpush", KEYS[3], ARGV[3])
       return 1
@@ -165,14 +175,19 @@ module Wurk
     # it, the re-push routes through BATCH_PUSH, whose guard finds the jid already
     # live (SADD == 0) → pure LPUSH, no recount. So registration happens exactly
     # once, here, at enqueue.
+    #
+    # The other path that can create `b-<bid>-jids`, so it carries the same NX
+    # expiry stamp as BATCH_PUSH.
     # KEYS = [schedule, b-<bid>, b-<bid>-jids]
-    # ARGV = [at_score, job_json, jid]
+    # ARGV = [at_score, job_json, jid, expiry_seconds]
     # Returns 1.
     BATCH_SCHEDULE = <<~LUA
       if redis.call("sadd", KEYS[3], ARGV[3]) == 1 then
         redis.call("hincrby", KEYS[2], "total", 1)
         redis.call("hincrby", KEYS[2], "pending", 1)
       end
+      redis.call("expire", KEYS[2], ARGV[4], "NX")
+      redis.call("expire", KEYS[3], ARGV[4], "NX")
       redis.call("zadd", KEYS[1], ARGV[1], ARGV[2])
       return 1
     LUA
@@ -208,13 +223,18 @@ module Wurk
     # currently in a failing/retrying state. Re-failures of the same jid are
     # idempotent. Cleared by BATCH_ACK_SUCCESS (retry passed) or
     # BATCH_ACK_COMPLETE (job died). Spec §2.5, §2.8.
+    #
+    # `b-<bid>-failed` is born here, so it carries the same NX expiry stamp as
+    # BATCH_PUSH.
     # KEYS = [b-<bid>, b-<bid>-failed]
-    # ARGV = [jid]
+    # ARGV = [jid, expiry_seconds]
     # Returns 1.
     BATCH_ACK_FAILED = <<~LUA
       if redis.call("sadd", KEYS[2], ARGV[1]) == 1 then
         redis.call("hincrby", KEYS[1], "failures", 1)
       end
+      redis.call("expire", KEYS[1], ARGV[2], "NX")
+      redis.call("expire", KEYS[2], ARGV[2], "NX")
       return 1
     LUA
 
@@ -224,8 +244,11 @@ module Wurk
     # live jids so the batch can fire `:complete` even with terminally failed
     # jobs. `b-<bid>-failed` holds only currently-retrying jids; `b-<bid>-died`
     # holds terminally-dead ones (spec §2.8 — the two sets are distinct).
+    #
+    # `b-<bid>-died` is born here, so it carries the same NX expiry stamp as
+    # BATCH_PUSH.
     # KEYS = [b-<bid>, b-<bid>-jids, b-<bid>-died, b-<bid>-failed]
-    # ARGV = [jid]
+    # ARGV = [jid, expiry_seconds]
     # Returns [live_jids_remaining, died_count, first_death]. `first_death`
     # is 1 the first time *any* jid is SADDed into the died set, 0 thereafter
     # — caller uses it to fire `:death` exactly once per batch.
@@ -236,6 +259,8 @@ module Wurk
         redis.call("hincrby", KEYS[1], "failures", -1)
       end
       local died_added = redis.call("sadd", KEYS[3], ARGV[1])
+      redis.call("expire", KEYS[1], ARGV[2], "NX")
+      redis.call("expire", KEYS[3], ARGV[2], "NX")
       local first_death = 0
       if was_pre_existing_death == 0 and died_added == 1 then
         first_death = 1
@@ -262,10 +287,21 @@ module Wurk
     # same reopened batch cannot lose each other's writes. Refuses to write
     # when the batch hash is gone — resurrecting a bare hash would create a
     # batch that can never fire anything.
+    #
+    # Appends are deduped and capped. Reopening the same batch inside every
+    # job and re-registering its callback is a common shape, and each append
+    # pays a decode + encode of the whole array: unbounded that is O(N^2) Lua
+    # work on the way in and N identical callback jobs at fire time. An
+    # identical triple is therefore a no-op — the registration is already
+    # satisfied by the entry that is there — and past the cap the append is
+    # refused rather than allowed to grow the hash field without limit.
+    # Both sides of the comparison are normalised through cjson so an entry
+    # written by Ruby's `to_json` at first flush matches one appended here.
     # KEYS = [b-<bid>]
-    # ARGV = [callback triple JSON, event name]
-    # Returns -1 when the batch hash does not exist; otherwise the event's
-    # fired flag ("1", or nil when it has not fired yet).
+    # ARGV = [callback triple JSON, event name, max callbacks]
+    # Returns -1 when the batch hash does not exist and -2 when the cap
+    # refused the append; otherwise the event's fired flag ("1", or nil when
+    # it has not fired yet).
     BATCH_APPEND_CALLBACK = <<~LUA
       if redis.call("exists", KEYS[1]) == 0 then
         return -1
@@ -277,19 +313,30 @@ module Wurk
       else
         list = {}
       end
-      list[#list + 1] = cjson.decode(ARGV[1])
+      local entry = cjson.decode(ARGV[1])
+      local encoded = cjson.encode(entry)
+      for i = 1, #list do
+        if cjson.encode(list[i]) == encoded then
+          return redis.call("hget", KEYS[1], ARGV[2])
+        end
+      end
+      if #list >= tonumber(ARGV[3]) then
+        return -2
+      end
+      list[#list + 1] = entry
       redis.call("hset", KEYS[1], "callbacks", cjson.encode(list))
       return redis.call("hget", KEYS[1], ARGV[2])
     LUA
 
     # Ent Unique (§3): atomic compare-and-delete of a lock key. Replaces the
     # two-command GET-then-DEL — between those calls the key can expire and a
-    # fresh enqueue can grab it, and the bare DEL would then drop the new
+    # fresh owner can grab it, and the bare DEL would then drop the new
     # owner's lock. Shared by `Unique::ServerMiddleware#release` (normal
-    # success/start release) and `Unique::DEATH_HANDLER` (automatic-death
-    # release) so the two paths cannot drift.
-    # KEYS = [unique:<sha256>]
-    # ARGV = [owning jid]
+    # success/start release), `Unique::DEATH_HANDLER` (automatic-death
+    # release) and `Leader#release` (stepping down from the cluster lock)
+    # so those paths cannot drift.
+    # KEYS = [the lock key — unique:<sha256> | dear-leader]
+    # ARGV = [the owner that must still hold it — jid | <host>:<pid>:<nonce>]
     # Returns 1 when the key was deleted, 0 otherwise.
     RELEASE_IF_OWNER = <<~LUA
       if redis.call("get", KEYS[1]) == ARGV[1] then
@@ -334,6 +381,41 @@ module Wurk
       return removed
     LUA
 
+    # Ent Periodic (§2): compare-and-swap the fire marks of `loops:<lid>`.
+    # The cron poller's leader gate is a cached read (`Component#leader?`,
+    # LEADER_CACHE_TTL_MS), so for a few seconds after a handover two
+    # processes both believe they lead and both reach the same due loop —
+    # HMGET → decide → enqueue → HSET then fires it twice. Claiming the slot
+    # atomically is what makes a tick fire exactly once; the loser gets 0 and
+    # enqueues nothing. Values written are the same decimal-epoch strings the
+    # Ruby path wrote, byte for byte — the dashboard reads these fields.
+    #
+    # `nf` present: the caller must still be looking at the mark it read
+    # (byte compare, so no parse can drift the token).
+    # `nf` absent: the caller derived the slot from `lf`, so refuse once `lf`
+    # has reached it — that is the exhausted-schedule case, where the winner
+    # cleared `nf` and a loser would otherwise re-fire the very same slot.
+    # KEYS = [loops:<lid>]
+    # ARGV = [expected `nf` (or the derived slot), new `lf`, new `nf` ('' → HDEL)]
+    # Returns 1 when this caller claimed the slot, 0 otherwise.
+    CRON_CLAIM_FIRE = <<~LUA
+      local key = KEYS[1]
+      local cur = redis.call("hget", key, "nf")
+      if cur and cur ~= "" then
+        if cur ~= ARGV[1] then return 0 end
+      else
+        local lf = tonumber(redis.call("hget", key, "lf") or "")
+        if lf and lf >= tonumber(ARGV[1]) then return 0 end
+      end
+      redis.call("hset", key, "lf", ARGV[2])
+      if ARGV[3] == "" then
+        redis.call("hdel", key, "nf")
+      else
+        redis.call("hset", key, "nf", ARGV[3])
+      end
+      return 1
+    LUA
+
     # Limiter scripts live in `lib/wurk/lua/limiter_*.lua` — one file per
     # type. Loaded at boot, the file's basename (minus `.lua`) becomes the
     # SCRIPTS key as a symbol. Keeping them as separate files makes diffing
@@ -358,7 +440,8 @@ module Wurk
       batch_append_callback: BATCH_APPEND_CALLBACK,
       fast_delete_job: FAST_DELETE_JOB,
       fast_delete_by_class: FAST_DELETE_BY_CLASS,
-      release_if_owner: RELEASE_IF_OWNER
+      release_if_owner: RELEASE_IF_OWNER,
+      cron_claim_fire: CRON_CLAIM_FIRE
     }.merge(FILE_SCRIPTS).freeze
 
     # SHA1 of each script source — matches what `SCRIPT LOAD` returns.

data/lib/wurk/manager.rb

@@ -24,12 +24,21 @@ module Wurk
 
     attr_reader :workers, :capsule
 
-    def initialize(capsule)
+    # `shutdown:` is the owning Launcher's process-wide shutdown request,
+    # invoked when concurrency can no longer be held (see #processor_result).
+    # Injected rather than reached for: only the entry point knows how this
+    # process exits — a swarm child / standalone CLI re-delivers TERM to itself
+    # so the installed trap drives the normal drain, while embedded mode owns
+    # no traps and must stop the launcher in place. A Manager built without an
+    # owner (Sidekiq's `Manager.new(capsule)` arity, kept for the alias) has no
+    # route to take, so it only reports.
+    def initialize(capsule, shutdown: nil)
       @config = @capsule = capsule
       @count = capsule.concurrency
       raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
 
       @done = false
+      @shutdown = shutdown
       @workers = Set.new
       @plock = ::Mutex.new
       @count.times do
@@ -86,8 +95,9 @@ module Wurk
     # cleanly or via raised exception. Removes the dead processor from the
     # pool and (unless we're already stopping) spawns a replacement so the
     # capsule's concurrency stays constant. If the replacement itself can't be
-    # spawned, crash the child (the swarm respawns it) rather than silently
-    # dropping concurrency. Snapshot under @plock; start the replacement — a
+    # spawned, ask the owner to shut this process down (the swarm respawns it at
+    # full concurrency) rather than silently dropping a worker for the life of
+    # the process. Snapshot under @plock; start the replacement — a
     # side effect — outside the lock.
     def processor_result(processor, _reason = nil)
       replacement = @plock.synchronize do
@@ -102,10 +112,15 @@ module Wurk
     rescue StandardError => e
       # Replacement spawn failed (e.g. ThreadError at the OS thread limit).
       # Silently running one Processor short for the life of the process is
-      # invisible degradation; instead report and crash the child on the main
-      # thread so the swarm respawns it at full concurrency (plan 02 §6).
+      # invisible degradation, so take the process down (the swarm respawns it
+      # at full concurrency) — but through the owner's TERM path, never
+      # `Thread.main.raise`: that raise unwound straight past Manager#stop, so
+      # the in-flight UnitsOfWork were never bulk_requeued and each one waited
+      # out a full reaper interval, and in embedded mode it killed the host's
+      # main thread instead of just the worker. Non-blocking by contract — we
+      # are on the dying Processor's own thread, which the drain will kill.
       @capsule.config.handle_exception(e, { context: 'Manager could not replace a dead Processor' })
-      main_thread.raise(e)
+      @shutdown&.call
     end
 
     # Reached when the deadline expired with workers still busy. Atomically
@@ -146,14 +161,15 @@ module Wurk
       @plock.synchronize { @workers.dup }
     end
 
+    # Drained means "no processor thread is still running", not "the Set is
+    # empty": a processor removes itself from @workers from inside its own
+    # thread, so the two agree — except for a processor that was never started
+    # (a launcher that raised mid-boot and rolled back). That one holds no
+    # thread and nothing will ever remove it, so an emptiness test would make
+    # #stop poll out the entire shutdown deadline waiting on a worker that
+    # never ran.
     def workers_empty?
-      @plock.synchronize { @workers.empty? }
-    end
-
-    # Seam over Thread.main: lets processor_result's replacement-failure crash
-    # be unit-tested against a controlled thread instead of the live runner.
-    def main_thread
-      Thread.main
+      @plock.synchronize { @workers.none? { |w| w.thread&.alive? } }
     end
 
     # Polls `condblock` until it returns true or the monotonic deadline

data/lib/wurk/metrics/history.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../middleware'
+require_relative '../pool_checkout'
 
 module Wurk
   module Metrics
@@ -124,11 +125,11 @@ module Wurk
         cfg.handle_exception(err, context: 'Wurk::Metrics::History')
       end
 
-      def self.with_pool(pool, &)
+      def self.with_pool(pool, idempotent: false, &)
         if pool
-          pool.with(&)
+          PoolCheckout.with(pool, idempotent, &)
         else
-          Wurk.redis(&)
+          Wurk.redis(idempotent:, &)
         end
       end
       private_class_method :with_pool

data/lib/wurk/metrics/queue_rollup.rb

@@ -51,11 +51,23 @@ module Wurk
       end
 
       def start
-        @thread ||= safe_thread('queue-metrics') { @timer.run { tick } } # rubocop:disable Naming/MemoizedInstanceVariableName
+        return @thread if @thread
+
+        @timer.reset
+        @thread = safe_thread('queue-metrics') { @timer.run { tick } }
       end
 
+      # Blocks until the thread is really gone: the launcher releases the
+      # cluster lock immediately after this returns, and a sample still in
+      # flight would write the same buckets as the next leader's first one.
+      #
+      # Cleared only on a confirmed join (Thread#join returns nil on timeout):
+      # a wedged thread must stay tracked so #start's guard returns it instead
+      # of calling @timer.reset, which would un-terminate the loop it is still
+      # inside and leave two threads HSETting the same buckets.
       def terminate
         @timer.terminate
+        @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
       end
 
       # Leader-gated: only the elected leader samples, so N workers don't each

data/lib/wurk/metrics/rollup.rb

@@ -59,11 +59,23 @@ module Wurk
       end
 
       def start
-        @thread ||= safe_thread('metrics-rollup') { @timer.run { tick } } # rubocop:disable Naming/MemoizedInstanceVariableName
+        return @thread if @thread
+
+        @timer.reset
+        @thread = safe_thread('metrics-rollup') { @timer.run { tick } }
       end
 
+      # Blocks until the thread is really gone: the launcher releases the
+      # cluster lock immediately after this returns, and a roll still in flight
+      # would write the same buckets as the next leader's first one.
+      #
+      # Cleared only on a confirmed join (Thread#join returns nil on timeout):
+      # a wedged thread must stay tracked so #start's guard returns it instead
+      # of calling @timer.reset, which would un-terminate the loop it is still
+      # inside and leave two threads HSETting the same buckets.
       def terminate
         @timer.terminate
+        @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
       end
 
       # Leader-gated: only the elected leader writes the cluster-total series,

data/lib/wurk/middleware/interrupt_handler.rb

@@ -8,14 +8,15 @@ module Wurk
   module Middleware
     # Server middleware. Catches `Wurk::Job::Interrupted` raised by an
     # IterableJob mid-iteration (or any cooperatively-cancelled job),
-    # re-pushes the job to the head of its queue so it resumes from the
+    # re-pushes the job to the tail of its queue so it resumes from the
     # persisted cursor, and raises `Wurk::JobRetry::Skip` so the retry
     # layer treats this as a clean exit rather than an error.
     #
-    # The re-push uses LPUSH (head of queue) so the same job is the next
-    # one to be fetched after restart. The job JSON is unchanged: cursor
-    # state lives in the `it-<jid>` HASH (see IterableJob persistence),
-    # not in the payload.
+    # The re-push uses RPUSH (tail of queue) so the same job is the next
+    # one to be fetched: the fetcher's LMOVE pops from the RIGHT (tail),
+    # so this job is fetched ahead of fresh LPUSH'd enqueues. The job
+    # JSON is unchanged: cursor state lives in the `it-<jid>` HASH (see
+    # IterableJob persistence), not in the payload.
     #
     # Auto-registered at the top of the server chain when this file is
     # required. Top-of-chain is important: a downstream middleware must
@@ -36,7 +37,7 @@ module Wurk
 
       def repush(job, queue)
         payload = Wurk.dump_json(job)
-        redis_pool.with { |conn| conn.call('LPUSH', "queue:#{queue}", payload) }
+        redis_pool.with { |conn| conn.call('RPUSH', "queue:#{queue}", payload) }
       end
     end
   end

data/lib/wurk/middleware/poison_pill.rb

@@ -19,15 +19,22 @@ module Wurk
     # Counter key TTL is wire-compat with Sidekiq Pro — third-party tooling
     # that watches `super_fetch:recovered:*` expects 72h.
     #
-    # No server-middleware registration: callers are reaper / bulk_requeue
-    # paths that drive the lifecycle directly via `track!(payload, queue:)`.
-    # When that integration lands, it just calls `PoisonPill.track!` on each
-    # orphan it about to RPUSH back to the public queue.
+    # No server-middleware registration: the producer is Reaper#drain, which
+    # drives the lifecycle directly via `track!(payload, queue:)` on each
+    # orphan it moves back to the public queue. The consumer is the ACK —
+    # `Fetcher::Reliable::UnitOfWork#acknowledge` pipelines {clear_in} next to
+    # its LREM, so an attempt that finished starts the next one at zero
+    # without costing a round trip of its own.
     module PoisonPill
       RECOVERY_THRESHOLD = 3
       RECOVERY_TTL = 72 * 60 * 60
       KEY_PREFIX = 'super_fetch:recovered:'
-      DEAD_RECORD_LIMIT = 100
+
+      # Handed to death handlers (and therefore to `:death` batch callbacks)
+      # as the cause when a recovered job is killed as a poison pill. Never
+      # raised: nothing is on a stack here — the reaper kills the job from the
+      # outside, on behalf of the workers it took down.
+      class Poisoned < ::StandardError; end
 
       # The `pill` handed to a Pro `super_fetch! { |jobstr, pill| }` recovery
       # callback on the kill path. Responds to .jid/.klass/.count/.queue so a
@@ -62,7 +69,7 @@ module Wurk
 
         jid = job['jid']
         klass = job['class']
-        emit_recovered_fetch(klass, queue)
+        emit('jobs.recovered.fetch', klass, queue)
 
         count = bump_counter(jid) if jid && !jid.empty?
         if count && count >= RECOVERY_THRESHOLD
@@ -80,15 +87,43 @@ module Wurk
       def recovery_count(jid)
         return 0 if jid.nil? || jid.to_s.empty?
 
-        Wurk.redis { |conn| conn.call('GET', "#{KEY_PREFIX}#{jid}") }.to_i
+        Wurk.redis { |conn| conn.call('GET', counter_key(jid)) }.to_i
       end
 
-      # Resets the counter for a jid — call after a successful perform so a
-      # job that recovered twice and then completed doesn't accumulate state.
+      # Resets the counter for a jid. The hot path uses {clear_in} instead —
+      # this is the standalone form for the API/dashboard and for callers with
+      # no pipeline of their own.
       def clear!(jid)
         return if jid.nil? || jid.to_s.empty?
 
-        Wurk.redis { |conn| conn.call('DEL', "#{KEY_PREFIX}#{jid}") }
+        Wurk.redis { |conn| conn.call('DEL', counter_key(jid)) }
+      end
+
+      # Queue the counter reset onto a pipeline the caller already has open.
+      #
+      # The ACK is the reset point. An attempt that got as far as acking —
+      # returned, or raised and booked its retry — is proof the job did not
+      # take its worker down, and that is the only thing this counter
+      # measures. Without the reset, three reclaims caused by three unrelated
+      # crashes inside 72h dead-set a job that has been completing all along
+      # (jids do come back: a UI/API retry re-pushes the same one, as does any
+      # client that supplies its own). Riding a round trip the ACK already
+      # makes is what keeps that free for the jobs — nearly all of them — that
+      # were never reclaimed at all.
+      # Guards the blank jid for the same reason {clear!} and {recovery_count}
+      # do, rather than relying on the one caller to do it: `counter_key(nil)`
+      # is the bare KEY_PREFIX, so an unguarded DEL here would delete a key
+      # that is shared rather than per-job. The caller's own check stays --
+      # it also skips queueing the command at all -- but the invariant belongs
+      # on the method that builds the key.
+      def clear_in(pipe, jid)
+        return if jid.nil? || jid.to_s.empty?
+
+        pipe.call('DEL', counter_key(jid))
+      end
+
+      def counter_key(jid)
+        "#{KEY_PREFIX}#{jid}"
       end
 
       # Register a callback fired when a poison pill is detected. Callbacks
@@ -113,19 +148,21 @@ module Wurk
       # ---- internals --------------------------------------------------
 
       def parse(payload)
-        case payload
-        when Hash then payload
-        when String
-          begin
-            Wurk.load_json(payload)
-          rescue ::JSON::ParserError
-            nil
-          end
-        end
+        return payload if payload.is_a?(::Hash)
+        return nil unless payload.is_a?(::String)
+
+        Wurk.load_json(payload)
+      rescue ::JSON::ParserError
+        nil
       end
 
+      # No apply-safety claim: INCR is additive, so a block replayed after a
+      # lost reply bumps twice and can carry a healthy job past
+      # RECOVERY_THRESHOLD into the dead set. Raising instead only leaves the
+      # count short — Reaper#drain rescues, and the job is already back on its
+      # public queue, so it just misses one poison check.
       def bump_counter(jid)
-        key = "#{KEY_PREFIX}#{jid}"
+        key = counter_key(jid)
         Wurk.redis do |conn|
           count = conn.call('INCR', key).to_i
           conn.call('EXPIRE', key, RECOVERY_TTL)
@@ -133,25 +170,29 @@ module Wurk
         end
       end
 
-      def emit_recovered_fetch(klass, queue)
+      def emit(metric, klass, queue)
         tags = []
         tags << "class:#{klass}" if klass
         tags << "queue:#{queue}" if queue
-        Wurk::Metrics::Statsd.increment('jobs.recovered.fetch', tags: tags.empty? ? nil : tags)
+        Wurk::Metrics::Statsd.increment(metric, tags: tags.empty? ? nil : tags)
       end
 
+      # Death handlers fire (`notify_failure` defaults to true). A poison kill
+      # is a death like any other exhaustion: suppressing it strands every
+      # batch that owns the job — Batch::DeathHandler is a death handler, so a
+      # silent kill leaves the batch's pending count stuck forever and neither
+      # `:death` nor `:complete` ever runs. Pro's spec is silent here
+      # (docs/target/sidekiq-pro.md §12); recorded in docs/idea/parity-divergences.md.
       def mark_poison(payload, job, queue:, count:)
-        emit_poison(job['class'], queue)
+        emit('jobs.poison', job['class'], queue)
         json = payload.is_a?(String) ? payload : Wurk.dump_json(job)
-        Wurk::DeadSet.new.kill(json, notify_failure: false)
+        Wurk::DeadSet.new.kill(json, ex: poisoned_error(job['class'], count))
         fire_callbacks(jid: job['jid'], klass: job['class'], count: count, queue: queue)
       end
 
-      def emit_poison(klass, queue)
-        tags = []
-        tags << "class:#{klass}" if klass
-        tags << "queue:#{queue}" if queue
-        Wurk::Metrics::Statsd.increment('jobs.poison', tags: tags.empty? ? nil : tags)
+      def poisoned_error(klass, count)
+        message = "#{klass || 'job'} was recovered #{count} times without completing"
+        Poisoned.new(message).tap { |e| e.set_backtrace(caller) }
       end
 
       def fire_callbacks(pill)

data/lib/wurk/middleware.rb

@@ -24,8 +24,8 @@ module Wurk
         config.logger
       end
 
-      def redis(&)
-        config.redis(&)
+      def redis(idempotent: false, &)
+        config.redis(idempotent:, &)
       end
     end
 

data/lib/wurk/pool_checkout.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative 'redis_pool'
+
+module Wurk
+  # The seam every `#redis`-style wrapper checks out through.
+  #
+  # `idempotent:` is a {Wurk::RedisPool} concept — it re-enables block replay
+  # after a command may already have applied server-side (see RedisPool#with).
+  # The object behind a wrapper is not always ours, though: the Sidekiq surface
+  # Wurk drops into lets a host supply any ConnectionPool-shaped object
+  # (`Sidekiq::Client#redis_pool=`, `Sidekiq::Web.redis_pool=`, the `pool:`
+  # kwarg on Stats::History / Deploy / Leader / Profiler / Metrics::History),
+  # and a bare `def with; yield conn; end` decorator — the shape people reach
+  # for to count or trace round trips — is a legal implementation of it. Handing
+  # that an unknown keyword is an ArgumentError, so the claim goes only to a pool
+  # that understands it.
+  #
+  # Dropping the claim is always sound: the pool falls back to the conservative
+  # no-replay default, which is exactly what a foreign pool did before the
+  # keyword existed. Apply-safety is an optimization, never a correctness input.
+  module PoolCheckout
+    def self.with(pool, idempotent, &)
+      return pool.with(&) unless idempotent && pool.is_a?(RedisPool)
+
+      pool.with(idempotent: true, &)
+    end
+  end
+end