wurk 1.3.1 → 1.4.0

data/lib/wurk/process_set.rb

@@ -32,7 +32,7 @@ module Wurk
     # absence (process never registered) and expiry (heartbeat lapsed,
     # info field gone) both return nil.
     def self.[](identity)
-      exists, fields = Wurk.redis do |conn|
+      exists, fields = Wurk.redis(idempotent: true) do |conn|
         conn.pipelined do |pipe|
           pipe.call('SISMEMBER', Keys::PROCESSES, identity)
           pipe.call('HMGET', identity, *LOOKUP_FIELDS)
@@ -48,6 +48,11 @@ module Wurk
     # don't dogpile the prune. Returns the number of identities removed
     # (or 0 when the lock was held by someone else).
     #
+    # No apply-safety claim, unlike the reads around it: a replay re-decides
+    # which identities are dead, so it could prune one that registered in
+    # between and hasn't written `info` yet. A blip here just skips one prune —
+    # the next caller a minute later does it.
+    #
     # Spec: docs/target/sidekiq-free.md §31.17.
     def cleanup
       return 0 unless acquired_cleanup_lock?
@@ -84,7 +89,7 @@ module Wurk
     # SCARD over `processes`. Not pruned — may include identities whose
     # heartbeat has lapsed. Use `each` for the accurate count.
     def size
-      Wurk.redis { |conn| conn.call('SCARD', Keys::PROCESSES) }
+      Wurk.redis(idempotent: true) { |conn| conn.call('SCARD', Keys::PROCESSES) }
     end
 
     # Sum of `concurrency` across live processes. Iterates `each` so dead
@@ -104,7 +109,7 @@ module Wurk
     # `||=` with empty-string fallback distinguishes "leader is unset" from
     # "memoization not yet computed".
     def leader
-      @leader ||= Wurk.redis { |c| c.call('GET', 'dear-leader') } || ''
+      @leader ||= Wurk.redis(idempotent: true) { |c| c.call('GET', 'dear-leader') } || ''
     end
 
     class << self
@@ -134,7 +139,7 @@ module Wurk
     private
 
     def fetch_each_rows
-      Wurk.redis do |conn|
+      Wurk.redis(idempotent: true) do |conn|
         procs = conn.call('SMEMBERS', Keys::PROCESSES).sort
         next [] if procs.empty?
 
@@ -223,7 +228,7 @@ module Wurk
     # Compares identity against the `dear-leader` STRING. Ent-only;
     # always false in OSS/free.
     def leader?
-      Wurk.redis { |c| c.call('GET', 'dear-leader') == identity }
+      Wurk.redis(idempotent: true) { |c| c.call('GET', 'dear-leader') == identity }
     end
 
     private

data/lib/wurk/processor.rb

@@ -164,6 +164,12 @@ module Wurk
       job_hash = parse_or_kill(jobstr, uow)
       return if job_hash.nil?
 
+      # The fetcher never parses, so hand it the jid we just read: the ACK
+      # retires this job's poison-pill recovery counter inside the round trip
+      # it already makes. A fetcher plugged in via `config[:fetch_class]` has
+      # no jid slot and simply ACKs — the counter then ages out on its 72h TTL.
+      uow.jid = job_hash['jid'] if uow.respond_to?(:jid=)
+
       ack = false
       begin
         Thread.handle_interrupt(Wurk::Shutdown => :never) do

data/lib/wurk/profiler.rb

@@ -5,6 +5,7 @@ require 'zlib'
 require 'stringio'
 require 'tempfile'
 require_relative 'keys'
+require_relative 'pool_checkout'
 
 module Wurk
   # Job profiling (Sidekiq 8.0+, OSS). When a job is pushed with a `profile`
@@ -113,8 +114,8 @@ module Wurk
         end
       end
 
-      def with_pool(pool, &)
-        pool ? pool.with(&) : Wurk.redis(&)
+      def with_pool(pool, idempotent: false, &)
+        pool ? PoolCheckout.with(pool, idempotent, &) : Wurk.redis(idempotent:, &)
       end
 
       def now

data/lib/wurk/queue.rb

@@ -25,7 +25,7 @@ module Wurk
 
     # @return [Array<Queue>] one per known queue, sorted by name.
     def self.all
-      names = Wurk.redis { |conn| conn.call('SMEMBERS', Keys::QUEUES_SET) }
+      names = Wurk.redis(idempotent: true) { |conn| conn.call('SMEMBERS', Keys::QUEUES_SET) }
       names.sort.map { |n| new(n) }
     end
 
@@ -35,12 +35,12 @@ module Wurk
     end
 
     def size
-      Wurk.redis { |conn| conn.call('LLEN', @rname) }
+      Wurk.redis(idempotent: true) { |conn| conn.call('LLEN', @rname) }
     end
 
     # Seconds since the oldest job (tail of LIST) was enqueued. 0.0 when empty.
     def latency
-      payload = Wurk.redis { |conn| conn.call('LRANGE', @rname, -1, -1).first }
+      payload = Wurk.redis(idempotent: true) { |conn| conn.call('LRANGE', @rname, -1, -1).first }
       return 0.0 if payload.nil?
 
       JobRecord.latency_from(Wurk.load_json(payload)['enqueued_at'])
@@ -51,19 +51,19 @@ module Wurk
     # True iff this queue's name is a member of the `paused` SET. Wurk
     # implements the Pro contract for free; fetchers consult the same set.
     def paused?
-      Wurk.redis { |conn| conn.call('SISMEMBER', Keys::PAUSED_SET, @name) } == 1
+      Wurk.redis(idempotent: true) { |conn| conn.call('SISMEMBER', Keys::PAUSED_SET, @name) } == 1
     end
 
     # Pause new fetches against this queue. Idempotent — `SADD` returns
     # 0 when the name was already present. In-flight jobs are untouched.
     def pause! # rubocop:disable Naming/PredicateMethod
-      Wurk.redis { |conn| conn.call('SADD', Keys::PAUSED_SET, @name) }
+      Wurk.redis(idempotent: true) { |conn| conn.call('SADD', Keys::PAUSED_SET, @name) }
       true
     end
 
     # Resume fetches. Idempotent.
     def unpause! # rubocop:disable Naming/PredicateMethod
-      Wurk.redis { |conn| conn.call('SREM', Keys::PAUSED_SET, @name) }
+      Wurk.redis(idempotent: true) { |conn| conn.call('SREM', Keys::PAUSED_SET, @name) }
       true
     end
 
@@ -74,7 +74,7 @@ module Wurk
       loop do
         start  = page * PAGE_SIZE
         stop   = start + PAGE_SIZE - 1
-        slice  = Wurk.redis { |conn| conn.call('LRANGE', @rname, start, stop) }
+        slice  = Wurk.redis(idempotent: true) { |conn| conn.call('LRANGE', @rname, start, stop) }
         slice.each { |value| yield JobRecord.new(value, @name) }
         break if slice.size < PAGE_SIZE
 
@@ -91,6 +91,9 @@ module Wurk
     # UNLINK the list + drop the queue from the `queues` set. Pipelined
     # so a partial failure leaves at most one of the two ops applied.
     # Method name is Sidekiq wire-compat — `clear?` would break the alias.
+    #
+    # Unlike pause!/unpause!, this one can't claim apply-safety: a replay after
+    # a lost reply would UNLINK whatever a producer enqueued in between.
     def clear # rubocop:disable Naming/PredicateMethod
       Wurk.redis do |conn|
         conn.pipelined do |pipe|

data/lib/wurk/rails_boot.rb

@@ -8,6 +8,11 @@ module Wurk
   # boot policy stays pure and unit-testable without the Railtie DSL.
   # See docs/idea/03-process-model.md for the exact ordering.
   module RailsBoot
+    # What `at_exit` waits for the supervise thread on top of the swarm's own
+    # drain budget: one supervise tick to notice the request, plus slack for
+    # the final reap.
+    DRAIN_JOIN_SLACK = 1
+
     module_function
 
     # Invoked from the `wurk.server_mode` initializer, before config/initializers
@@ -103,36 +108,62 @@ module Wurk
     end
 
     def boot_swarm
-      swarm = Wurk::Swarm.new(topology: Wurk.configuration.topology,
-                              shutdown_timeout: Wurk.configuration[:timeout] || Swarm::DEFAULT_SHUTDOWN_TIMEOUT)
+      timeout = Wurk.configuration[:timeout] || Swarm::DEFAULT_SHUTDOWN_TIMEOUT
+      swarm = Wurk::Swarm.new(topology: Wurk.configuration.topology, shutdown_timeout: timeout)
+      supervisor = nil
+      # Registered BEFORE boot, which forks the children one slot at a time: a
+      # fork that raises partway through would otherwise leave the children it
+      # already spawned with nothing to drain them on host exit. Children
+      # inherit the hook — stop_swarm no-ops off the process that forked them.
+      at_exit { stop_swarm(swarm, supervisor, timeout + Swarm::SHUTDOWN_GRACE + DRAIN_JOIN_SLACK) }
       # Co-hosted in the web process (e.g. Puma single mode): the host owns the
       # process-wide TERM/INT traps. Installing the swarm's own would hijack
       # them — a deploy TERM would drain the swarm but never stop the HTTP
       # server. Let the host keep signal ownership and drain the swarm on its
       # graceful exit (same contract as boot_embedded).
       swarm.boot(install_signals: false)
-      at_exit { swarm.shutdown }
       # supervise must still run somewhere or crashed children never respawn and
       # memory checks never fire. A background thread keeps the host's main
       # thread free to serve HTTP.
-      Thread.new do
+      supervisor = Thread.new do
         swarm.supervise
       rescue StandardError => e
         logger.error { "wurk supervisor thread died: #{e.class}: #{e.message}" }
       end
     end
 
+    # at_exit fires on the host's main thread, but the supervise thread owns the
+    # child table and two threads inside `shutdown` race on it — so request the
+    # drain and wait for the supervisor to run it. Draining here is the fallback
+    # for when no live supervisor will: it never started (boot raised) or it
+    # died early; after one that drained it finds no children and no-ops. A
+    # supervisor still alive past the join is wedged mid-drain — leave it be
+    # rather than race it; its children self-terminate (OrphanGuard) once this
+    # process is gone.
+    def stop_swarm(swarm, supervisor, drain_wait)
+      return unless swarm.owner?
+
+      swarm.request_shutdown
+      supervisor&.join(drain_wait)
+      swarm.shutdown unless supervisor&.alive?
+    end
+
     # Sidekiq-embedded parity: a threads-only worker inside the web process, no
     # fork. Redis validation failure keeps the host serving HTTP (log + carry
     # on). at_exit drains on the graceful shutdown the web server runs on TERM.
-    def boot_embedded
-      instance = Wurk::Embedded.new(Wurk.configuration)
-      instance.run
+    def boot_embedded(embedded = Wurk::Embedded)
+      instance = embedded.new(Wurk.configuration)
+      # Registered BEFORE run, for the reason boot_swarm registers before the
+      # fork: run brings the heartbeat, pollers, managers and health listener up
+      # one at a time, and a raise partway through would otherwise leave the ones
+      # already running with nothing to drain them on host exit.
       at_exit { instance.stop }
+      instance.run
       logger.info { 'wurk: running embedded in the web process (config.wurk.embed_in_web) — threads only, no fork' }
       instance
     rescue StandardError => e
       logger.error { "wurk: embedded boot failed: #{e.class}: #{e.message}" }
+      instance&.stop
       nil
     end
 

data/lib/wurk/redis_client_adapter.rb

@@ -21,13 +21,18 @@ module Wurk
 
     DEPRECATED_COMMANDS = %i[rpoplpush zrangebyscore zrevrange zrevrangebyscore getset hmset setex setnx].to_set
 
+    # Every method here dispatches through `call` rather than `@client.call` so
+    # the round-trip odometer below sees method-style commands too — a host
+    # block doing `conn.sadd(...)` then `conn.lpush(...)` has to be as
+    # replay-protected as one written with `conn.call`. On the pipeline
+    # decorator `call` is the same buffered forward it always was.
     module CompatMethods
       def info
-        @client.call('INFO') { |i| i.lines(chomp: true).map { |l| l.split(':', 2) }.select { |l| l.size == 2 }.to_h }
+        call('INFO') { |i| i.lines(chomp: true).map { |l| l.split(':', 2) }.select { |l| l.size == 2 }.to_h }
       end
 
       def evalsha(sha, keys, argv)
-        @client.call('EVALSHA', sha, keys.size, *keys, *argv)
+        call('EVALSHA', sha, keys.size, *keys, *argv)
       end
 
       # The Redis commands Sidekiq itself uses — defined eagerly so the
@@ -41,7 +46,7 @@ module Wurk
 
       USED_COMMANDS.each do |name|
         define_method(name) do |*args, **kwargs|
-          @client.call(name, *args, **kwargs)
+          call(name, *args, **kwargs)
         end
       end
 
@@ -52,7 +57,7 @@ module Wurk
         if DEPRECATED_COMMANDS.include?(args.first)
           warn("[sidekiq#5788] Redis has deprecated the `#{args.first}` command, called at #{caller(1..1)}")
         end
-        @client.call(*args, &)
+        call(*args, &)
       end
       ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
 
@@ -64,9 +69,48 @@ module Wurk
     CompatClient = RedisClient::Decorator.create(CompatMethods)
 
     class CompatClient
+      # Dispatch methods that put commands on the wire and wait for the reply.
+      # SCAN and friends are left out on purpose: they are pure reads, so
+      # re-running one applies nothing and cannot make a replay unsafe.
+      DISPATCH_METHODS = %i[call call_v call_once call_once_v
+                            blocking_call blocking_call_v pipelined multi].freeze
+
+      # Round trips this connection has completed, monotonic for its whole life.
+      #
+      # {Wurk::RedisPool} replays a failed block only while it can prove nothing
+      # in it applied, and a connect-phase error proves that for the command
+      # that raised — never for the ones before it. redis-client re-dials a
+      # dropped socket mid-block, so `CannotConnectError` surfaces on the second
+      # pipeline of a block whose first one already landed. The pool snapshots
+      # this counter around the block and refuses the replay once it has moved.
+      attr_reader :round_trips
+
+      def initialize(client)
+        super
+        @round_trips = 0
+      end
+
       def config
         @client.config
       end
+
+      # Counted *after* the call returns: a command that never reached a server
+      # leaves the odometer where it was, which is what keeps the pre-apply
+      # backoff alive for blocks that failed on their very first round trip.
+      DISPATCH_METHODS.each do |name|
+        class_eval(<<~RUBY, __FILE__, __LINE__ + 1)
+          # def call(...)
+          #   result = super
+          #   @round_trips += 1
+          #   result
+          # end
+          def #{name}(...)
+            result = super
+            @round_trips += 1
+            result
+          end
+        RUBY
+      end
     end
   end
 end

data/lib/wurk/redis_pool.rb

@@ -10,17 +10,29 @@ module Wurk
   # across forks: the parent closes the pool before fork, each child opens a
   # fresh one (see docs/idea/03-process-model.md, steps 3 and 5).
   #
-  # #with absorbs transient Redis failures (production incident #101):
+  # #with absorbs transient Redis failures (production incident #101), but only
+  # where replaying the caller's block cannot change what the server already
+  # did — the block is arbitrary Ruby, so a replay re-issues every command in it:
   #   * READONLY / NOREPLICAS / UNBLOCKED — a failover happened; close and retry
   #     once immediately so redis-client redials the new primary (spec §26).
-  #   * ConnectionError (incl. CannotConnect / Read- / WriteTimeout) — a blip;
-  #     close and retry with exponential backoff up to CONN_MAX_ATTEMPTS, then
-  #     raise. At-least-once tolerates the rare duplicate and JobRetry re-runs
-  #     the job anyway, so retrying at the pool layer is strictly better.
+  #   * CannotConnect / Failover — raised while dialing, so the command that hit
+  #     one never reached a server and cannot have applied; close and retry with
+  #     exponential backoff up to CONN_MAX_ATTEMPTS, then raise.
+  #   * Read-/WriteTimeout and bare ConnectionError — the command may already
+  #     have applied server-side, so these raise. Replaying would double-push a
+  #     job, double-count a stat, or drop a member ZPOPed by the lost reply.
+  #     Blocks that are safe to re-run (pure reads, an LMOVE the reaper reclaims,
+  #     owner-CAS scripts) opt back into the backoff with `with(idempotent: true)`.
   #   * ConnectionPool::TimeoutError — checkout starved; retry once after a
   #     short jittered pause, then raise (sizing is the fix, not queuing).
-  # Every retry and final give-up is reported through the injected `on_error`
-  # telemetry hook (Wurk::Configuration#on_redis_error).
+  # Those proofs are about the command that raised, not the block around it: a
+  # block is several round trips, and redis-client re-dials mid-block, so a
+  # CannotConnect can surface on the second pipeline of a block whose first one
+  # already landed. So the pool also watches the connection's round-trip
+  # odometer (RedisClientAdapter::CompatClient#round_trips) and refuses to
+  # replay a non-idempotent block that has already completed one.
+  # Every retry, refused replay, and final give-up is reported through the
+  # injected `on_error` telemetry hook (Wurk::Configuration#on_redis_error).
   class RedisPool
     DEFAULT_URL  = ENV.fetch('REDIS_URL', 'redis://localhost:6379/0')
     DEFAULT_NAME = 'default'
@@ -32,7 +44,9 @@ module Wurk
     # wait above. read/write are deliberately wider than connect so a briefly-
     # slow-but-alive Redis (RDB fork pause, a large BLMOVE payload) doesn't
     # spuriously ReadTimeout — the production incident (#101) the single
-    # dual-use timeout caused. reconnect_attempts re-dials a dropped socket once.
+    # dual-use timeout caused. reconnect_attempts re-dials a dropped socket once;
+    # note that redis-client's re-dial also re-sends the one in-flight command,
+    # so the apply-safety split below bounds block replay, not command replay.
     DEFAULT_CONNECT_TIMEOUT    = 1.0
     DEFAULT_READ_TIMEOUT       = 2.5
     DEFAULT_WRITE_TIMEOUT      = 2.5
@@ -53,6 +67,17 @@ module Wurk
     # backoff below (otherwise a failover would sleep instead of redialing).
     RETRYABLE_MSG = /\A(READONLY|NOREPLICAS|UNBLOCKED)/
 
+    # ConnectionErrors that can only be raised while dialing, so the block
+    # provably never applied and replaying it is safe whatever it contains.
+    # CannotConnect covers every connect-phase failure (redis-client converts a
+    # stalled handshake into it); Failover is the Sentinel resolver rejecting a
+    # server whose role changed. Any other ConnectionError — Read-/WriteTimeout
+    # or a reset mid-command — leaves the outcome unknown.
+    PRE_APPLY_ERRORS = [RedisClient::CannotConnectError, RedisClient::FailoverError].freeze
+
+    # The two retry_plan verdicts that re-run the block; the rest raise.
+    REPLAY_PLANS = %i[failover backoff].freeze
+
     # ConnectionError backoff: CONN_MAX_ATTEMPTS total tries, sleeping
     # (BASE * 2**attempt) + rand*JITTER before each retry. The 1.0s + 2.0s pair
     # rides out a sub-4s blip; the jitter de-syncs a fleet reconnecting at once.
@@ -89,10 +114,14 @@ module Wurk
     # Checkout a connection and run the block. ConnectionPool::TimeoutError is
     # raised by @pool.with *before* the block runs, so it is caught out here
     # (the in-block #run rescue never sees it) — one retry, then raise.
-    def with(&block)
+    #
+    # `idempotent: true` asserts the block can be re-run after a command may
+    # already have applied server-side, which buys back the full ConnectionError
+    # backoff. Only claim it for pure reads or writes whose repeat is a no-op.
+    def with(idempotent: false, &block)
       checkout_retried = false
       begin
-        @pool.with { |conn| run(conn, &block) }
+        @pool.with { |conn| run(conn, idempotent, &block) }
       rescue ConnectionPool::TimeoutError => e
         if checkout_retried
           notify_error(e, attempt: 2, retried: false)
@@ -113,7 +142,7 @@ module Wurk
     # slot counts merged in — one call gives a heartbeat both Redis health and
     # local pool saturation. (Real Redis INFO has no `size`/`available` field.)
     def info
-      with { |conn| parse_info(conn.call('INFO')) }
+      with(idempotent: true) { |conn| parse_info(conn.call('INFO')) }
         .merge('size' => @size, 'available' => available)
     end
 
@@ -159,17 +188,19 @@ module Wurk
     # Runs the block on the checked-out `conn`, retrying transient RedisClient
     # errors in place: the same slot is reused across retries (redis-client
     # redials a closed socket lazily), so a busy fetcher can't leak checkouts.
-    def run(conn)
+    def run(conn, idempotent)
       attempts = 0
       begin
         attempts += 1
+        odometer = conn.round_trips
         yield conn
       rescue RedisClient::Error => e
-        plan = retry_plan(e, attempts)
+        plan = retry_plan(e, attempts, idempotent, conn.round_trips != odometer)
         raise if plan == :propagate
 
-        notify_error(e, attempt: attempts, retried: plan != :exhausted)
-        raise if plan == :exhausted
+        replaying = REPLAY_PLANS.include?(plan)
+        notify_error(e, attempt: attempts, retried: replaying)
+        raise unless replaying
 
         safe_close(conn)
         sleep(backoff_delay(attempts)) if plan == :backoff
@@ -177,19 +208,33 @@ module Wurk
       end
     end
 
-    # Pure classification of a RedisClient error against the attempt count:
+    # Pure classification of a RedisClient error against the attempt count, the
+    # caller's apply-safety claim, and whether this attempt already completed a
+    # round trip (`dirty`):
     #   :failover  → close + immediate retry (a primary swap)
     #   :backoff   → close + sleep + retry (a connection blip)
-    #   :exhausted → ConnectionError past the cap; report and give up
+    #   :unsafe    → something may have applied; report the blip, then raise
+    #   :exhausted → replayable ConnectionError past the cap; report and give up
     #   :propagate → not transient (or a spent failover); raise as-is
-    def retry_plan(err, attempts)
-      if RETRYABLE_MSG.match?(err.message.to_s)
-        attempts > 1 ? :propagate : :failover
-      elsif err.is_a?(RedisClient::ConnectionError)
-        attempts >= CONN_MAX_ATTEMPTS ? :exhausted : :backoff
-      else
-        :propagate
-      end
+    def retry_plan(err, attempts, idempotent, dirty)
+      failover = RETRYABLE_MSG.match?(err.message.to_s)
+      return :propagate unless failover || err.is_a?(RedisClient::ConnectionError)
+      return :unsafe unless replayable?(err, idempotent, dirty, failover)
+      return attempts > 1 ? :propagate : :failover if failover
+
+      attempts >= CONN_MAX_ATTEMPTS ? :exhausted : :backoff
+    end
+
+    # A replay re-issues the whole block, so one completed round trip voids
+    # every pre-apply proof: however provably the *failing* command missed the
+    # server, the ones ahead of it in the block did not. On a still-clean block
+    # a failover reply is proof enough by itself (the command was rejected
+    # outright); a bare ConnectionError needs one of the connect-phase classes.
+    def replayable?(err, idempotent, dirty, failover)
+      return true if idempotent
+      return false if dirty
+
+      failover || PRE_APPLY_ERRORS.any? { |klass| err.is_a?(klass) }
     end
 
     def notify_error(error, attempt:, retried:)

data/lib/wurk/scheduled.rb

@@ -6,6 +6,7 @@ require_relative 'lua'
 require_relative 'lua/loader'
 require_relative 'client'
 require_relative 'process_set'
+require_relative 'timer_loop'
 
 module Wurk
   # Promotes due jobs from the `retry` and `schedule` sorted sets back onto
@@ -57,13 +58,30 @@ module Wurk
         loop do
           break if @done
 
-          jobstr = @config.redis { |conn| Wurk::Lua::Loader.eval_cached(conn, :zpopbyscore, keys: [sset], argv: [now]) }
+          jobstr = pop_due(sset, now)
           break unless jobstr
 
           push_promoted(jobstr, sset)
         end
       end
 
+      # ZPOPBYSCORE is destructive and carries its result in the reply, so this
+      # block never claims apply-safety: a replay discards whatever the lost
+      # reply already removed. The pool therefore raises on a Read-/WriteTimeout,
+      # which leaves the outcome of *this* pop unknown — a due job may or may not
+      # have come off the ZSET. Report it and end this set's drain (the nil makes
+      # #drain_set break) rather than pop again blind; a job caught in that window
+      # falls into the same pop→push loss the default scheduler already documents
+      # on #push_promoted, and `reliable_scheduler!` (ReliableEnq) is the loss-free
+      # fix. Rescuing here rather than around #drain_set keeps the sibling set
+      # draining on this tick.
+      def pop_due(sset, now)
+        @config.redis { |conn| Wurk::Lua::Loader.eval_cached(conn, :zpopbyscore, keys: [sset], argv: [now]) }
+      rescue RedisClient::ConnectionError => e
+        handle_exception(e, { context: 'scheduler_pop', set: sset })
+        nil
+      end
+
       # A raising `@client.push` (bad payload, transient Redis error) must not
       # abort the drain and strand the remaining due jobs until the next poll —
       # rescue per-job, report, continue. The already-popped job IS lost here
@@ -169,13 +187,23 @@ module Wurk
 
       # Idempotent. Wakes the sleeping thread so it observes @done and exits.
       # Also propagates the stop signal to @enq so any in-flight drain loop
-      # short-circuits instead of running to completion.
+      # short-circuits instead of running to completion. Terminal, not a pause:
+      # @enq's stop flag is one-way, so this poller never polls again.
+      #
+      # Joins before returning — the caller (Launcher#quiet, then #stop) clears
+      # the heartbeat right after, and a sweep still in flight would promote
+      # jobs on behalf of a process that no longer exists.
+      #
+      # Cleared only on a confirmed join (Thread#join returns nil on timeout):
+      # a wedged sweep must stay tracked so #start's ||= guard returns it
+      # rather than spawning a second scheduler thread alongside it.
       def terminate
         @mutex.synchronize do
           @done = true
           @enq.terminate
           @sleeper.signal
         end
+        @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
       end
 
       # Called on every wake. Any raise inside the Enq is reported and the

data/lib/wurk/stats.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'date'
+require_relative 'pool_checkout'
 
 module Wurk
   # Read-only inspector for cluster state in Redis. The cheap counters are
@@ -38,7 +39,7 @@ module Wurk
     # Sum of the `busy` HASH field across every live process identity.
     # Pipelined but unbounded by process count.
     def workers_size
-      Wurk.redis do |conn|
+      Wurk.redis(idempotent: true) do |conn|
         identities = conn.call('SMEMBERS', Keys::PROCESSES)
         next 0 if identities.empty?
 
@@ -55,7 +56,7 @@ module Wurk
     # and dashboards reading this rely on that order. Match it exactly with
     # `sort_by { |_, size| -size }`.
     def queues
-      Wurk.redis do |conn|
+      Wurk.redis(idempotent: true) do |conn|
         names = conn.call('SMEMBERS', Keys::QUEUES_SET)
         next {} if names.empty?
 
@@ -71,7 +72,7 @@ module Wurk
     # (`queue_summaries.sort_by { |qd| -qd.size }`) — this feeds the
     # dashboard's queue table (api_controller#queues).
     def queue_summaries
-      Wurk.redis do |conn|
+      Wurk.redis(idempotent: true) do |conn|
         names = conn.call('SMEMBERS', Keys::QUEUES_SET)
         next [] if names.empty?
 
@@ -89,13 +90,17 @@ module Wurk
     # Latency (secs) of the `default` queue — the most-asked-about gauge.
     def default_queue_latency
       now_ms = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
-      payload = Wurk.redis { |c| c.call('LRANGE', Keys.queue('default'), -1, -1) }.first
+      payload = Wurk.redis(idempotent: true) { |c| c.call('LRANGE', Keys.queue('default'), -1, -1) }.first
       compute_latency(payload, now_ms)
     end
 
     # Resets the named global counters. With no args, clears `processed`,
     # `failed`, and `expired`. SET … 0 (not DEL — keeps the key around so
     # reads stay `Integer` not `nil`).
+    #
+    # The only write here, and the only block in this class that can't claim
+    # apply-safety: a replay after a lost reply would re-zero the counters,
+    # discarding whatever the fleet counted in between.
     def reset(*stats)
       all      = %w[failed processed expired]
       to_clear = stats.empty? ? all : all & stats.flatten.map(&:to_s)
@@ -123,7 +128,7 @@ module Wurk
     private_constant :FAST_QUERIES, :FAST_KEYS
 
     def fetch_stats_fast!
-      raw = Wurk.redis do |conn|
+      raw = Wurk.redis(idempotent: true) do |conn|
         conn.pipelined { |pipe| FAST_QUERIES.each { |args| pipe.call(*args) } }
       end
       @stats = FAST_KEYS.zip(raw.map(&:to_i)).to_h
@@ -179,17 +184,17 @@ module Wurk
 
       def date_stat_hash(stat)
         keys = (0...@days_previous).map { |i| (@start_date - i).strftime('%Y-%m-%d') }
-        values = with_redis do |conn|
+        values = with_redis(idempotent: true) do |conn|
           conn.pipelined { |pipe| keys.each { |d| pipe.call('GET', "stat:#{stat}:#{d}") } }
         end
         keys.zip(values.map(&:to_i)).to_h
       end
 
-      def with_redis(&)
+      def with_redis(idempotent: false, &)
         if @pool
-          @pool.with(&)
+          PoolCheckout.with(@pool, idempotent, &)
         else
-          Wurk.redis(&)
+          Wurk.redis(idempotent:, &)
         end
       end
     end

data/lib/wurk/swarm/child_boot.rb

@@ -2,6 +2,7 @@
 
 require_relative '../component'
 require_relative '../launcher'
+require_relative '../client/buffered'
 require_relative '../fetcher/reliable'
 require_relative '../lua'
 require_relative 'orphan_guard'
@@ -106,8 +107,19 @@ module Wurk
 
       def reconnect_after_fork
         @config.reset_redis_pools!
+        # The reliable_push outage buffer, its drainer thread and its mutexes
+        # are process-global and were copied wholesale from the parent. The
+        # Process._fork hook normally beats us to it (making this a no-op) —
+        # the explicit call keeps the swarm path deterministic and ordered
+        # after the pool reset, so a re-armed drainer can only ever see the
+        # child's own pool.
+        Wurk::Client::Buffered.reset_after_fork!
         validate_redis!
         reconnect_active_record
+        # The dogstatsd client is memoized at the class level (Statsd.client),
+        # so without a reset every child would share the parent's UDP socket
+        # and thread-locals instead of building its own after fork.
+        Wurk::Metrics::Statsd.reset!
       end
 
       # Prove the child's fresh Redis socket reaches a live server before it