wurk 1.7.6 → 1.8.1

data/lib/wurk/dead_set.rb

@@ -23,22 +23,21 @@ module Wurk
 
     # Two-axis trim: `ZREMRANGEBYSCORE` evicts entries older than
     # `dead_timeout_in_seconds`, `ZREMRANGEBYRANK 0 -dead_max_jobs` keeps
-    # the count bounded. Pipelined — partial failure leaves at most one
-    # axis applied (acceptable; trim is non-critical, runs again next kill).
+    # the count bounded. One MULTI, as upstream — both axes apply together.
     #
     # `max_jobs:` / `timeout:` override the global config for this call.
     # Lets parallel tests run trim with isolated limits without mutating
     # `Wurk.configuration` (which is process-global and races across threads).
-    def trim(max_jobs: nil, timeout: nil) # rubocop:disable Naming/PredicateMethod
+    def trim(max_jobs: nil, timeout: nil)
       config = Wurk.configuration
       max_jobs ||= config[:dead_max_jobs] || 10_000
       timeout ||= config[:dead_timeout_in_seconds] || (180 * 24 * 60 * 60)
       cutoff = ::Process.clock_gettime(::Process::CLOCK_REALTIME) - timeout
 
       Wurk.redis do |conn|
-        conn.pipelined do |pipe|
-          pipe.call('ZREMRANGEBYSCORE', @name, '-inf', "(#{cutoff}")
-          pipe.call('ZREMRANGEBYRANK', @name, 0, -max_jobs)
+        conn.multi do |tx|
+          tx.call('ZREMRANGEBYSCORE', @name, '-inf', "(#{cutoff}")
+          tx.call('ZREMRANGEBYRANK', @name, 0, -max_jobs)
         end
       end
       true
@@ -65,7 +64,7 @@ module Wurk
     # synthesized RuntimeError when callers don't have one) — death handlers
     # receive `(job, ex)`. `max_jobs:` / `timeout:` propagate to the auto-trim;
     # see `#trim` for the rationale.
-    def kill(message, opts = {}) # rubocop:disable Naming/PredicateMethod
+    def kill(message, opts = {})
       notify = opts.fetch(:notify_failure, true)
       do_trim = opts.fetch(:trim, true)
       ex = opts[:ex] || RuntimeError.new(API_KILL_MESSAGE).tap { |e| e.set_backtrace(caller) }

data/lib/wurk/encryption.rb

@@ -34,7 +34,7 @@ module Wurk
   #   end
   #
   # Wire format (per docs/target/sidekiq-ent.md §4.4): the last arg becomes
-  # a plain JSON Hash `{"v"=>N, "iv"=>b64(iv), "ct"=>b64(ct), "tag"=>b64(tag)}`
+  # a plain JSON Hash `{"__wurk_enc__"=>true, "v"=>N, "iv"=>…, "ct"=>…, "tag"=>…}`
   # — *not* a base64 blob of a binary envelope — so the args array stays
   # valid JSON for inspectors that don't know about encryption.
   #
@@ -89,7 +89,7 @@ module Wurk
       #
       # Idempotent: re-calling rebinds the resolver and rebuilds the cache.
       # Middleware is installed at most once per chain.
-      def enable(active_version:, &resolver) # rubocop:disable Naming/PredicateMethod
+      def enable(active_version:, &resolver)
         raise ArgumentError, 'active_version is required' unless active_version
         raise ArgumentError, 'block returning the key bytes is required' unless resolver
 
@@ -117,7 +117,7 @@ module Wurk
         raise Error, 'Wurk::Encryption not enabled' unless enabled?
 
         @key_cache ||= {}
-        @key_cache[version] ||= validate_key!(version, @resolver.call(version))
+        @key_cache[version] ||= validate_key!(version, resolve(version))
       end
 
       # Encrypt `value` (any JSON-serializable Ruby value) under
@@ -167,8 +167,27 @@ module Wurk
           value.key?('v') && value.key?('iv') && value.key?('ct') && value.key?('tag')
       end
 
+      # `args` with its last element enveloped, unless it already is one. The
+      # single sealing rule shared by the client middleware and every server
+      # path that writes a payload back to Redis (InterruptHandler, limiter
+      # dead-routing): a job hash seen mid-chain may hold decrypted args.
+      def seal_args(args)
+        return args unless args.is_a?(::Array) && !args.empty? && !envelope?(args.last)
+
+        args[0..-2] + [encrypt(args.last)]
+      end
+
+      # `job` safe to write back to Redis: a copy with the last arg re-sealed
+      # when crypto is on and the job opted in, otherwise `job` itself.
+      def seal(job)
+        return job unless enabled? && job['encrypt']
+
+        sealed = seal_args(job['args'])
+        sealed.equal?(job['args']) ? job : job.merge('args' => sealed)
+      end
+
       # Web UI display helper (§4.7). Given a job hash, returns the args
-      # array with the last element replaced by the literal `"<encrypted>"`
+      # array with the last element replaced by the literal `"[encrypted data]"`
       # when the job opted in. Cleartext preceding args are untouched so
       # operators can still triage on user_id / object_id / etc.
       #
@@ -181,7 +200,7 @@ module Wurk
         return args if args.empty?
         return args unless job['encrypt'] || job[:encrypt] || envelope?(args.last)
 
-        args[0..-2] + ['<encrypted>']
+        args[0..-2] + ['[encrypted data]']
       end
 
       # A decryption failure means the key is gone (rotated away) or the
@@ -212,6 +231,16 @@ module Wurk
         cipher
       end
 
+      # A resolver built on `keys.fetch(v)` / `ENV.fetch(...)` raises KeyError
+      # (an IndexError) for a rotated-away version where a Hash#[] resolver
+      # returns nil. Both mean "no key for this version", so both must take
+      # the same dead-set path; a bare KeyError would bubble into 25 retries.
+      def resolve(version)
+        @resolver.call(version)
+      rescue ::IndexError => e
+        raise KeyMissingError, "key resolver has no key for version #{version} (#{e.class}: #{e.message})"
+      end
+
       def validate_key!(version, bytes)
         raise KeyMissingError, "key resolver returned nil for version #{version}" if bytes.nil?
 
@@ -240,17 +269,16 @@ module Wurk
       include Wurk::Middleware::ClientMiddleware
 
       def call(_worker, job, _queue, _redis_pool)
-        return yield unless Wurk::Encryption.enabled? && job['encrypt']
-
-        args = job['args']
-        if args.is_a?(::Array) && !args.empty? && !Wurk::Encryption.envelope?(args.last)
-          job['args'] = args[0..-2] + [Wurk::Encryption.encrypt(args.last)]
-        end
+        job['args'] = Wurk::Encryption.seal_args(job['args']) if Wurk::Encryption.enabled? && job['encrypt']
         yield
       end
     end
 
-    # Server middleware — peels the envelope before perform runs. A decrypt
+    # Server middleware — peels the envelope before perform runs and puts it
+    # back once the chain unwinds, success or failure. The job hash is shared
+    # with every outer middleware and the processor; leaving plaintext in it
+    # let any of them write the secret back to Redis (InterruptHandler's
+    # re-push, limiter dead-routing) or hand it to an error reporter. A decrypt
     # failure (missing/rotated key, bad tag) is terminal and non-retryable,
     # so rather than let it bubble into the 25× retry pipeline we route the
     # job straight to the dead set tagged `encryption_error` and ACK it via
@@ -262,8 +290,13 @@ module Wurk
       def call(_worker, job, _queue)
         return yield unless Wurk::Encryption.enabled? && job['encrypt']
 
+        sealed = job['args']
         decrypt_last_arg!(job)
-        yield
+        begin
+          yield
+        ensure
+          job['args'] = sealed
+        end
       end
 
       private

data/lib/wurk/fetcher/many_queues.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative '../lua'
+require_relative '../pid_cache'
+
+module Wurk
+  class Fetcher
+    # The rest of {Fetcher::Reliable#walk}'s non-blocking pass once the first
+    # queue came back empty: every remaining queue in one round trip
+    # (lib/wurk/lua/fetch_first.lua) instead of one LMOVE per queue. A sparse
+    # 100-queue capsule used to pay 100 round trips per pass to reach a job on
+    # its last queue; it now pays two. The script walks the order it is given,
+    # so strict order, the weighted shuffle and the paused filter all stay
+    # Reliable#queues_cmd's.
+    #
+    # Why not the first queue too, for one round trip: a script pays Redis per
+    # key it is handed, before it moves anything — measured on Redis 7.0 at
+    # ~13us server time for 10 queues and ~83us for 100, against ~0.2us for a
+    # bare LMOVE. A capsule whose first queue is busy is the common shape, and
+    # folding that queue into the script made its every fetch 1.4x slower
+    # end to end on 10 queues. Leading with the LMOVE keeps that path exactly
+    # where it was and spends the script only on a pass that already missed.
+    module ManyQueues
+      NO_ARGV = [].freeze
+
+      private
+
+      # Queues 2..n. Two queues is one more LMOVE — a one-key script costs more
+      # than the move it would wrap.
+      #
+      # Apply-safe for the reason #lmove is: a claim whose reply was lost left
+      # the job in this process's private list, un-ACKed, which the next boot's
+      # Reaper reclaims, and the replay claims a different job.
+      def claim_rest(queues)
+        return lmove(queues[1]) if queues.size == 2
+
+        keys = claim_keys(queues)
+        index, job = config.redis(idempotent: true) do |conn|
+          Wurk::Lua::Loader.eval_cached(conn, :fetch_first, keys: keys, argv: NO_ARGV)
+        end
+        return nil unless job
+
+        public_q = queues[index]
+        priv, name = queue_keys(public_q)
+        unit_of_work(public_q, priv, name, job)
+      end
+
+      # `[public, private, public, private, ...]` for queues 2..n, fetch_first's
+      # KEYS. Strict mode with nothing paused hands #walk the same frozen array
+      # every pass, so the steady state reuses one build; a shuffled or filtered
+      # list is a fresh array per pass and rebuilds. Keyed on the pid too, for
+      # the reason #queue_keys is. Copy-on-write: processor threads share this
+      # fetcher.
+      def claim_keys(queues)
+        pid = PidCache.pid
+        cached = @claim_keys
+        return cached[2] if cached && cached[0].equal?(queues) && cached[1] == pid
+
+        keys = queues.drop(1).flat_map { |public_q| [public_q, queue_keys(public_q).first] }.freeze
+        @claim_keys = [queues, pid, keys].freeze
+        keys
+      end
+    end
+  end
+end

data/lib/wurk/fetcher/orphan_grace.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Wurk
+  class Fetcher
+    # The Reaper's boot-grace check: has this heartbeat-judged orphan been
+    # left alone for at least `grace` seconds? A booting worker can claim a job
+    # a moment before its first heartbeat lands, and draining its list in that
+    # window runs the job twice, concurrently.
+    #
+    # The answer comes from OBJECT IDLETIME, which a live owner resets on every
+    # claim and ACK. Some servers refuse it — Redis under an LFU maxmemory
+    # policy, Dragonfly with no OBJECT at all — and the grace must not quietly
+    # vanish with it. Without IDLETIME the clock is this reaper's own: a list
+    # is drained only once it has been judged orphaned for `grace` seconds
+    # across sweeps, by which time a booting owner has beaten and the
+    # heartbeat re-check spares it. The switch is logged once.
+    class OrphanGrace
+      NO_IDLETIME_WARNING = 'reaper: OBJECT IDLETIME is unavailable on this Redis (%s); the boot grace ' \
+                            'before reclaiming a private list now runs on this process\'s own clock, so an ' \
+                            'orphan is drained one sweep later than it would be. Expected under an LFU ' \
+                            'maxmemory policy or on servers without OBJECT (e.g. Dragonfly).'
+
+      # Lists remembered for the fallback clock. A list leaves on the pass
+      # that clears it; one judged orphaned and then found alive can linger,
+      # so the memory is bounded and simply restarts when full — every clock
+      # restarting is the conservative direction.
+      MEMORY = 10_000
+
+      def initialize(config, grace)
+        @config = config
+        @grace = grace
+        @idletime = true
+        @first_seen = {}
+      end
+
+      # true while the list must be left alone.
+      def recent?(key)
+        return false unless @grace.positive?
+        return local_recent?(key) unless @idletime
+
+        idle = @config.redis(idempotent: true) { |c| c.call('OBJECT', 'IDLETIME', key) }
+        # nil: the list is already gone, nothing to drain.
+        idle.nil? || idle < @grace
+      rescue RedisClient::CommandError => e
+        idletime_unavailable!(e)
+        local_recent?(key)
+      end
+
+      private
+
+      def local_recent?(key)
+        now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        @first_seen.clear if @first_seen.size >= MEMORY && !@first_seen.key?(key)
+        seen = (@first_seen[key] ||= now)
+        return true if now - seen < @grace
+
+        @first_seen.delete(key)
+        false
+      end
+
+      def idletime_unavailable!(error)
+        @idletime = false
+        @config.logger.warn { format(NO_IDLETIME_WARNING, error.message.lines.first&.strip) }
+      end
+    end
+  end
+end

data/lib/wurk/fetcher/private_list_key.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative '../keys'
+
+module Wurk
+  class Fetcher
+    # Reads the owner back out of a reliable-fetch private list key
+    # (`queue:<public>|<host>|<pid>|<nonce>|<idx>`, see
+    # Reliable.private_queue_name) for the Reaper's liveness check. Pure string
+    # work, no Redis.
+    module PrivateListKey
+      module_function
+
+      # `queue:<public>|<host>|<pid>|<nonce>|<idx>` → [public_q, host, pid,
+      # nonce], parsed from the right so a `|` inside the queue name is
+      # tolerated. nil when the key isn't a well-formed private list.
+      #
+      # With no known prefix to split on, both owner shapes are tried in
+      # preference order and the first one leaving a real public queue behind
+      # wins. The narrow reading is what saves a pre-nonce key from an all-digit
+      # host (`queue:q|123456789012|<pid>|<idx>` — a bare Docker hostname is 12
+      # hex chars): read wide, its host segment eats the whole queue name.
+      def parse_full(key)
+        parts = key.split('|')
+        owner_tails(parts).each do |host, pid, nonce, width|
+          public_q = parts[0...-width].join('|')
+          next unless public_q.start_with?(Keys::QUEUE_PREFIX) && public_q != Keys::QUEUE_PREFIX
+
+          return [public_q, host, pid, nonce]
+        end
+        nil
+      end
+
+      # `<public_q>|<host>|<pid>|<nonce>|<idx>` → [host, pid, nonce] (pid as
+      # Integer), or all-nil when the suffix isn't a well-formed owner tail.
+      # Splitting the suffix off the known public-queue prefix tolerates a `|`
+      # inside the queue name itself, and leaves the tail unambiguous: exactly
+      # 4 segments for the current shape, exactly 3 for the pre-nonce one.
+      def parse_owner(public_q, key)
+        suffix = key.delete_prefix("#{public_q}|")
+        return [nil, nil, nil] if suffix == key
+
+        host, pid, nonce = owner_tails(suffix.split('|')).first
+        [host, pid, nonce]
+      end
+
+      # Owner segments of a private-list key, taken from the right, as
+      # [host, pid, nonce, segment_count] readings in preference order (empty
+      # when nothing parses). The wide shape is preferred: an all-digit nonce is
+      # rare but reachable, and reading such a key narrow would take the pid for
+      # the host and the nonce for the pid — draining a live owner's list out
+      # from under it.
+      def owner_tails(parts)
+        return [] unless parts.size >= 3 && integer?(parts[-1])
+
+        [wide_tail(parts), narrow_tail(parts)].compact
+      end
+
+      # `<host>|<pid>|<nonce>|<idx>` — the shape every current process writes.
+      def wide_tail(parts)
+        [parts[-4], parts[-3].to_i, parts[-2], 4] if parts.size >= 4 && integer?(parts[-3])
+      end
+
+      # `<host>|<pid>|<idx>` — written before the nonce existed. Such a list can
+      # still hold a pre-upgrade process's in-flight jobs across a rolling
+      # upgrade, so it stays reclaimable even though nothing writes it anymore.
+      def narrow_tail(parts)
+        [parts[-3], parts[-2].to_i, nil, 3] if integer?(parts[-2])
+      end
+
+      def integer?(str)
+        str.is_a?(String) && str.match?(/\A\d+\z/)
+      end
+    end
+  end
+end

data/lib/wurk/fetcher/reaper.rb

@@ -4,6 +4,9 @@ require_relative '../component'
 require_relative '../keys'
 require_relative '../middleware/poison_pill'
 require_relative '../timer_loop'
+require_relative 'private_list_key'
+require_relative 'unparseable_keys'
+require_relative 'orphan_grace'
 
 module Wurk
   class Fetcher
@@ -30,6 +33,15 @@ module Wurk
     #     `processes` member (one whose `info` hash still exists). Such reclaim
     #     therefore waits out the 60s heartbeat TTL, exactly as the spec says.
     #
+    # The heartbeat verdict is a snapshot taken before a SCAN that can run for
+    # a long time on a big keyspace, and a booting process can claim jobs a
+    # moment before its first beat lands. Draining on that snapshot alone
+    # reclaims a live worker's in-flight job — a double run. So a heartbeat-
+    # judged orphan is re-checked right before its drain: its list must have
+    # sat untouched for `grace` seconds (OBJECT IDLETIME; a live owner touches
+    # it on every claim and ACK), and its identity must still have no `info`
+    # hash. Our own incarnation skips both: kill(0) is not a snapshot.
+    #
     # Re-pushed jobs run through Wurk::Middleware::PoisonPill, which caps a
     # job at RECOVERY_THRESHOLD recoveries within 72h: past the cap the job
     # is killed into the dead set instead of re-queued, so a job that crashes
@@ -59,17 +71,31 @@ module Wurk
       # the fleet, since a global SCAN is far costlier than the scoped pass.
       FULL_INTERVAL = 3600
 
+      # Seconds a heartbeat-judged private list must have sat idle before it is
+      # drained. Far below the heartbeat TTL a dead owner's list already waits
+      # out, so it never delays a real reclaim; well above the gap between a
+      # booting process's first claim and its first beat.
+      DEFAULT_GRACE = 30
+
       LOCK_KEY = 'super_fetch:reaper'
       FULL_LOCK_KEY = 'super_fetch:reaper:full'
       SCAN_COUNT = 100
+
+      # The hourly sweep walks the whole keyspace, where almost nothing
+      # matches: COUNT is the work per SCAN call, so a bigger batch is the
+      # same server work in a tenth of the round trips — a million keys is
+      # 1,000 calls instead of 10,000. Still small enough that one call never
+      # holds Redis for more than a fraction of a millisecond.
+      FULL_SCAN_COUNT = 1000
       THREAD_NAME = 'wurk-reaper'
 
       attr_reader :interval
 
       def initialize(config, interval: DEFAULT_INTERVAL, lock_key: LOCK_KEY,
-                     full_interval: FULL_INTERVAL, full_lock_key: FULL_LOCK_KEY)
+                     full_interval: FULL_INTERVAL, full_lock_key: FULL_LOCK_KEY, grace: DEFAULT_GRACE)
         @config = config
         @interval = interval
+        @grace = OrphanGrace.new(config, grace)
         @lock_key = lock_key
         @full_interval = full_interval
         @full_lock_key = full_lock_key
@@ -77,6 +103,7 @@ module Wurk
         @done = false
         @mutex = ::Mutex.new
         @sleeper = ::ConditionVariable.new
+        @unparseable = UnparseableKeys.new(config)
       end
 
       # Spawns the sweep loop. Idempotent. The loop waits one interval before
@@ -139,7 +166,7 @@ module Wurk
         owners = live_owners
         reclaimed = 0
         each_full_private_list do |key, public_q, host, pid, nonce|
-          next if owner_alive?(host, pid, nonce, owners)
+          next unless orphaned?(key, host, pid, nonce, owners)
 
           reclaimed += drain(key, public_q)
         end
@@ -162,7 +189,7 @@ module Wurk
       def reclaim_queue(public_q, owners)
         reclaimed = 0
         each_private_list(public_q) do |key, host, pid, nonce|
-          next if owner_alive?(host, pid, nonce, owners)
+          next unless orphaned?(key, host, pid, nonce, owners)
 
           reclaimed += drain(key, public_q)
         end
@@ -179,8 +206,8 @@ module Wurk
             c.call('SCAN', cursor, 'MATCH', "#{public_q}|*", 'COUNT', SCAN_COUNT)
           end
           keys.each do |key|
-            host, pid, nonce = parse_owner(public_q, key)
-            yield key, host, pid, nonce if pid
+            host, pid, nonce = PrivateListKey.parse_owner(public_q, key)
+            pid ? yield(key, host, pid, nonce) : @unparseable.report(key)
           end
           break if cursor == '0'
         end
@@ -188,83 +215,22 @@ module Wurk
 
       # Yields [private_list_key, public_q, host, pid, nonce] for every list in
       # the keyspace. MATCH `queue:*|*` matches only private lists (public queue
-      # keys carry no `|`); parse_full_key drops anything that isn't a
+      # keys carry no `|`); PrivateListKey.parse_full drops anything that isn't a
       # well-formed `queue:<public>|<host>|<pid>|<nonce>|<idx>`.
       def each_full_private_list
         cursor = '0'
         loop do
           cursor, keys = redis(idempotent: true) do |c|
-            c.call('SCAN', cursor, 'MATCH', "#{Keys::QUEUE_PREFIX}*|*", 'COUNT', SCAN_COUNT)
+            c.call('SCAN', cursor, 'MATCH', "#{Keys::QUEUE_PREFIX}*|*", 'COUNT', FULL_SCAN_COUNT)
           end
           keys.each do |key|
-            parsed = parse_full_key(key)
-            yield key, *parsed if parsed
+            parsed = PrivateListKey.parse_full(key)
+            parsed ? yield(key, *parsed) : @unparseable.report(key)
           end
           break if cursor == '0'
         end
       end
 
-      # `queue:<public>|<host>|<pid>|<nonce>|<idx>` → [public_q, host, pid,
-      # nonce], parsed from the right so a `|` inside the queue name is
-      # tolerated. nil when the key isn't a well-formed private list.
-      #
-      # With no known prefix to split on, both owner shapes are tried in
-      # preference order and the first one leaving a real public queue behind
-      # wins. The narrow reading is what saves a pre-nonce key from an all-digit
-      # host (`queue:q|123456789012|<pid>|<idx>` — a bare Docker hostname is 12
-      # hex chars): read wide, its host segment eats the whole queue name.
-      def parse_full_key(key)
-        parts = key.split('|')
-        owner_tails(parts).each do |host, pid, nonce, width|
-          public_q = parts[0...-width].join('|')
-          next unless public_q.start_with?(Keys::QUEUE_PREFIX) && public_q != Keys::QUEUE_PREFIX
-
-          return [public_q, host, pid, nonce]
-        end
-        nil
-      end
-
-      # `<public_q>|<host>|<pid>|<nonce>|<idx>` → [host, pid, nonce] (pid as
-      # Integer), or all-nil when the suffix isn't a well-formed owner tail.
-      # Splitting the suffix off the known public-queue prefix tolerates a `|`
-      # inside the queue name itself, and leaves the tail unambiguous: exactly
-      # 4 segments for the current shape, exactly 3 for the pre-nonce one.
-      def parse_owner(public_q, key)
-        suffix = key.delete_prefix("#{public_q}|")
-        return [nil, nil, nil] if suffix == key
-
-        host, pid, nonce = owner_tails(suffix.split('|')).first
-        [host, pid, nonce]
-      end
-
-      # Owner segments of a private-list key, taken from the right, as
-      # [host, pid, nonce, segment_count] readings in preference order (empty
-      # when nothing parses). The wide shape is preferred: an all-digit nonce is
-      # rare but reachable, and reading such a key narrow would take the pid for
-      # the host and the nonce for the pid — draining a live owner's list out
-      # from under it.
-      def owner_tails(parts)
-        return [] unless parts.size >= 3 && integer?(parts[-1])
-
-        [wide_tail(parts), narrow_tail(parts)].compact
-      end
-
-      # `<host>|<pid>|<nonce>|<idx>` — the shape every current process writes.
-      def wide_tail(parts)
-        [parts[-4], parts[-3].to_i, parts[-2], 4] if parts.size >= 4 && integer?(parts[-3])
-      end
-
-      # `<host>|<pid>|<idx>` — written before the nonce existed. Such a list can
-      # still hold a pre-upgrade process's in-flight jobs across a rolling
-      # upgrade, so it stays reclaimable even though nothing writes it anymore.
-      def narrow_tail(parts)
-        [parts[-3], parts[-2].to_i, nil, 3] if integer?(parts[-2])
-      end
-
-      def integer?(str)
-        str.is_a?(String) && str.match?(/\A\d+\z/)
-      end
-
       # `Process.kill(0, pid)` answers "does this pid exist *in my PID
       # namespace*", which is the question we're actually asking only when the
       # key was written from that same namespace. A shared hostname does not
@@ -281,11 +247,26 @@ module Wurk
       # Every other owner goes through the namespace-blind heartbeat — alive iff
       # its identity is a live `processes` member. A pre-nonce key can only be
       # matched on the `<host>:<pid>` prefix of that identity.
-      def owner_alive?(host, pid, nonce, owners)
-        return local_pid_alive?(pid) if nonce == process_nonce && host == hostname
-        return owners.include?("#{host}:#{pid}:#{nonce}") if nonce
+      def orphaned?(key, host, pid, nonce, owners)
+        return !local_pid_alive?(pid) if nonce == process_nonce && host == hostname
+        return false if owners.include?(owner_key(host, pid, nonce))
+
+        settled_orphan?(key, host, pid, nonce)
+      end
+
+      def owner_key(host, pid, nonce)
+        nonce ? "#{host}:#{pid}:#{nonce}" : "#{host}:#{pid}"
+      end
+
+      # The re-check right before a heartbeat-judged drain (see the class doc):
+      # the `owners` snapshot may predate the owner's first beat, or the whole
+      # SCAN. Only reached for lists the snapshot already called orphaned, so
+      # its round trips are paid per orphan, never per live list.
+      def settled_orphan?(key, host, pid, nonce)
+        return false if @grace.recent?(key)
+        return !live_owners.include?(owner_key(host, pid, nil)) unless nonce
 
-        owners.include?("#{host}:#{pid}")
+        redis(idempotent: true) { |c| c.call('HGET', owner_key(host, pid, nonce), 'info') }.nil?
       end
 
       def local_pid_alive?(pid)

data/lib/wurk/fetcher/reliable.rb

@@ -7,6 +7,7 @@ require_relative '../keys'
 require_relative '../lua'
 require_relative '../fetcher'
 require_relative 'capped'
+require_relative 'many_queues'
 
 module Wurk
   class Fetcher
@@ -23,10 +24,13 @@ module Wurk
     # the paths that must send a held ACK before they stop fetching, and
     # docs/idea/parity-divergences.md for the window that widens.
     #
-    # Priority handling: iterate queues_cmd in order with non-blocking
-    # LMOVE, then fall back to a blocking BLMOVE on the first queue so an
-    # empty poll doesn't spin Redis. BLMOVE has no multi-key form, so
-    # blocking on a single queue is the best Redis gives us. The block
+    # Priority handling: walk queues_cmd in order with a non-blocking claim —
+    # an LMOVE on the first queue, then one fetch_first script call over all
+    # the others (ManyQueues) — then fall back to a blocking BLMOVE
+    # on the first queue so an empty poll doesn't spin Redis. BLMOVE has no
+    # multi-key form, so blocking on a single queue is the best Redis gives us:
+    # a job landing on a later queue while every thread is parked waits for
+    # the next pass, at most one block timeout. The block
     # timeout defaults to TIMEOUT (2s) and is overridable per the Pro
     # super_fetch §3.3 `config.fetch_poll_interval` knob.
     #
@@ -40,6 +44,7 @@ module Wurk
       # that caps nothing, which is a stronger guarantee that the unconfigured
       # path is unchanged than a second loop kept in step by hand.
       prepend Capped
+      include ManyQueues
 
       # Default BLMOVE block timeout; overridable via config.fetch_poll_interval.
       TIMEOUT = 2
@@ -72,7 +77,7 @@ module Wurk
       # namespaces are in play: a restarted container reuses both, so the
       # reaper's `kill(0)` liveness check would read a dead owner's list as
       # live (jobs stranded) or a live owner's as dead (job run twice). Keys
-      # written before the nonce existed stay reclaimable — Reaper#parse_owner
+      # written before the nonce existed stay reclaimable — PrivateListKey
       # accepts both shapes.
       def self.private_queue_name(public_queue, index = 0)
         host = ENV['DYNO'] || Socket.gethostname
@@ -112,6 +117,7 @@ module Wurk
         @queue_keys_pid = ::Process.pid
         @prefixed_queues = nil
         @prefixed_source = nil
+        @claim_keys = nil
       end
 
       # Take custody of a finished job's LREM instead of sending it now. One
@@ -246,20 +252,20 @@ module Wurk
         config.config.handle_exception(ex, ctx)
       end
 
-      # Non-blocking pass over the fetchable queues. The pending ACK rides the
-      # first LMOVE and only the first: the walk is one fetch, so a second copy
-      # of the same LREM would be a wasted command on every queue we find
-      # empty. By the time we fall through to BLMOVE this thread is holding
-      # nothing — which is the point, since a blocking call can't join a
-      # pipeline and would strand the ACK for a whole poll interval.
+      # Non-blocking pass over the fetchable queues, then the blocking
+      # fall-through. The pending ACK rides the first LMOVE and only the first.
+      # By the time we fall through to BLMOVE this thread is holding nothing —
+      # which is the point, since a blocking call can't join a pipeline and
+      # would strand the ACK for a whole poll interval.
+      #
+      # The first queue is always a bare LMOVE, so the busy-first-queue steady
+      # state (and the one-queue capsule `rake bench` gates) costs exactly what
+      # it did. Only a miss there pays for the rest — see ManyQueues.
       def walk(queues)
         ack = take_pending_ack
-        queues.each do |public_q|
-          uow = lmove(public_q, ack)
-          ack = nil
-          return uow if uow
-        end
-        blmove(queues.first)
+        uow = lmove(queues.first, ack)
+        uow ||= claim_rest(queues) if queues.size > 1
+        uow || blmove(queues.first)
       end
 
       # Exactly one, even when a failed flush left this thread holding two: the