wurk 1.3.1 → 1.5.0

data/lib/wurk/leader.rb

@@ -3,6 +3,9 @@
 require 'securerandom'
 require 'socket'
 require_relative 'component'
+require_relative 'lua'
+require_relative 'pool_checkout'
+require_relative 'timer_loop'
 
 module Wurk
   # Cluster leader election via Redis `SET NX EX`. Single-leader-per-cluster
@@ -19,9 +22,12 @@ module Wurk
   # too — it is never re-read on subsequent acquires, only on transitions.
   #
   # Cadence per spec: renew every 15s while leader, recheck every 60s as
-  # follower, lock TTL 30s. Opt out a process from campaigning entirely
-  # with `WURK_LEADER=false` (or its Sidekiq alias `SIDEKIQ_LEADER=false`),
-  # useful for hot-standby pools.
+  # follower, lock TTL 30s. The campaign itself starts `initial_wait` seconds
+  # after `#start` — the spec fixes the renew/follower/TTL cadence but says
+  # nothing about when the first campaign runs, and holding it back keeps a
+  # booting process's opening Redis round trips on the job path. Opt out a
+  # process from campaigning entirely with `WURK_LEADER=false` (or its Sidekiq
+  # alias `SIDEKIQ_LEADER=false`), useful for hot-standby pools.
   #
   # Spec: docs/target/sidekiq-ent.md §6.
   class Leader
@@ -30,6 +36,13 @@ module Wurk
     DEFAULT_TTL = 30
     DEFAULT_RENEW_INTERVAL = 15
     DEFAULT_FOLLOWER_INTERVAL = 60
+    # Delay before the campaign's first `SET NX EX`, so a booting process
+    # spends its opening Redis round trips on fetching jobs rather than on a
+    # leadership CAS. Every leader-gated consumer (cron, both rollups,
+    # history) waits out a 30-60s interval before its own first tick, so
+    # nothing observes the gap. Tune with `config[:leader_initial_wait]`;
+    # 0 campaigns immediately.
+    DEFAULT_INITIAL_WAIT = 1
     OPT_OUT_ENV = 'WURK_LEADER'            # native opt-out env
     SIDEKIQ_OPT_OUT_ENV = 'SIDEKIQ_LEADER' # Sidekiq Ent drop-in alias (§6.2/§7.2)
     THREAD_NAME = 'wurk-leader'
@@ -46,12 +59,14 @@ module Wurk
     def initialize(config: nil, key: DEFAULT_KEY, ttl: DEFAULT_TTL, # rubocop:disable Metrics/ParameterLists
                    renew_interval: DEFAULT_RENEW_INTERVAL,
                    follower_interval: DEFAULT_FOLLOWER_INTERVAL,
+                   initial_wait: DEFAULT_INITIAL_WAIT,
                    pool: nil, owner: nil)
       @config = config
       @key = key
       @ttl = ttl
       @renew_interval = renew_interval
       @follower_interval = follower_interval
+      @initial_wait = initial_wait.to_f
       @pool = pool
       @owner = owner || cluster_identity
       @held = false
@@ -89,9 +104,19 @@ module Wurk
 
     # CAS DEL — only drop the key if we still own it, otherwise a stale
     # release would yank leadership from whichever follower took over.
+    # GET-then-DEL is not that CAS: our key can lapse between the two
+    # commands and a follower can win the election in the gap, whereupon
+    # the bare DEL deletes *its* lock and leaves the cluster leaderless
+    # (or, once the ex-leader re-campaigns, doubly led) for up to one
+    # renew interval — double cron fires, double rollups. The compare and
+    # the delete happen in one server-side step instead, via the script
+    # `Unique.release_if_owner` already shares.
+    #
+    # `idempotent:` — a replay after a lost reply finds the key either
+    # already gone or no longer ours, so it can only be a no-op.
     def release
-      redis_call do |c|
-        c.call('DEL', @key) if c.call('GET', @key) == @owner
+      redis_call(idempotent: true) do |c|
+        Wurk::Lua::Loader.eval_cached(c, :release_if_owner, keys: [@key], argv: [@owner])
       end
       @held = false
       @token = nil
@@ -107,6 +132,11 @@ module Wurk
     # follower, it polls every `follower_interval`. Caller must invoke
     # `stop` for orderly shutdown — the thread also releases its lock on
     # exit.
+    #
+    # The spawn happens *inside* the lock: with it outside, two callers
+    # racing into `start` both clear the nil check and both spawn, and the
+    # loser's loop is never recorded — an unjoinable second campaigner that
+    # outlives `stop` and keeps renewing the cluster lock.
     def start
       return nil if disabled?
 
@@ -114,17 +144,25 @@ module Wurk
         return @thread if @thread
 
         @done = false
+        @thread = spawn_loop_thread
       end
-      @thread = spawn_loop_thread
+      @thread
     end
 
+    # Bounded at TimerLoop::JOIN_TIMEOUT like every other periodic component: a
+    # tick parked on a wedged Redis (the campaign is a `SET NX EX`, so it waits
+    # out the socket timeouts) must not hold the whole process's teardown open —
+    # the swarm parent SIGKILLs a child that overruns its shutdown grace. The
+    # CAS release below runs either way; the loop's own trailing release makes
+    # a straggler's second attempt a no-op. A straggler also stays referenced,
+    # so a restart after a timed-out stop can't campaign twice in parallel.
     def stop
-      @mutex.synchronize do
+      thread = @mutex.synchronize do
         @done = true
         @sleeper.signal
+        @thread
       end
-      @thread&.join
-      @thread = nil
+      @mutex.synchronize { @thread = nil } if thread.nil? || thread.join(TimerLoop::JOIN_TIMEOUT)
       release
     end
 
@@ -175,13 +213,13 @@ module Wurk
       @config.handle_exception(e, event: :leader) if @config.respond_to?(:handle_exception)
     end
 
-    def redis_call(&)
+    def redis_call(idempotent: false, &)
       if @pool
-        @pool.with(&)
+        PoolCheckout.with(@pool, idempotent, &)
       elsif @config
-        @config.redis(&)
+        @config.redis(idempotent:, &)
       else
-        Wurk.redis(&)
+        Wurk.redis(idempotent:, &)
       end
     end
 
@@ -205,6 +243,7 @@ module Wurk
     end
 
     def run_loop
+      wait_initial
       until done?
         tick_once
         wait_next
@@ -212,6 +251,13 @@ module Wurk
       release
     end
 
+    # Same condvar as #wait_next, so a `stop` landing inside the pre-campaign
+    # delay wakes the loop immediately instead of holding shutdown open for a
+    # whole initial_wait.
+    def wait_initial
+      wait_for(@initial_wait) if @initial_wait.positive?
+    end
+
     def tick_once
       acquire
     rescue StandardError => e
@@ -219,7 +265,10 @@ module Wurk
     end
 
     def wait_next
-      interval = @held ? @renew_interval : @follower_interval
+      wait_for(@held ? @renew_interval : @follower_interval)
+    end
+
+    def wait_for(interval)
       @mutex.synchronize { @sleeper.wait(@mutex, interval) unless @done }
     end
 

data/lib/wurk/limiter/base.rb

@@ -104,17 +104,15 @@ module Wurk
         end
       end
 
+      # Metadata HSET + EXPIRE + `lmtr-list` SADD in a single round trip. The
+      # script also gates the SADD on the metadata being newly written, so a
+      # per-job construction (`stripe-#{user_id}`) stops rewriting a set that
+      # grows one member per name — see limiter_register.lua for why that gate
+      # is only safe while it is atomic with the HSET.
       def register!
-        Wurk::Limiter.redis do |c|
-          c.call('SADD', LIST_KEY, @name)
-          c.call(
-            'HSET', meta_key,
-            'type', type.to_s,
-            'options', JSON.dump(serializable_options),
-            'fingerprint', fingerprint
-          )
-          c.call('EXPIRE', meta_key, @options[:ttl])
-        end
+        lua(:limiter_register,
+            keys: [meta_key, LIST_KEY],
+            argv: [@name, type.to_s, JSON.dump(serializable_options), fingerprint, ttl])
       end
 
       def ttl

data/lib/wurk/limiter/bucket.rb

@@ -38,7 +38,7 @@ module Wurk
           remaining = deadline - ::Time.now.to_f
           raise OverLimit, self if remaining <= 0
 
-          sleep [remaining, secs_to_next.to_f, 0.05].compact.min.clamp(0.0, remaining)
+          sleep [0.05, secs_to_next.to_f].max.clamp(0.0, remaining)
         end
       end
 

data/lib/wurk/limiter/concurrent.rb

@@ -6,7 +6,7 @@ module Wurk
   module Limiter
     # Atomic slot acquisition in a ZSET. Score = expiry epoch; the acquire
     # script first evicts expired slots (bumping the `reclaimed` metric)
-    # then ZADDs if there's headroom.
+    # then ZADDs if there's headroom (bumping the `held` metric).
     #
     # On exhaustion: spin loop with backoff. The spec says "blocks via
     # Redis stream XREAD" — that's a perf optimization; the visible
@@ -38,33 +38,15 @@ module Wurk
         raise ArgumentError, 'block required' unless block
 
         started = monotime
-        deadline = started + @options[:wait_timeout]
         slot = random_id
-        acquired_at = nil
-        loop do
-          result = acquire(slot)
-          if result[0].to_i == 1
-            acquired_at = monotime
-            break
-          end
-
-          return if @options[:policy] == :ignore
-
-          remaining = deadline - monotime
-          if remaining <= 0
-            bump_counter('overages')
-            raise OverLimit, self
-          end
-
-          sleep [remaining, WAIT_SLEEP].min
-        end
+        acquired_at = wait_for_slot(slot, started + @options[:wait_timeout])
+        return unless acquired_at
 
         begin
           incr_immediate_or_waited(acquired_at - started)
           block.call
         ensure
-          release(slot)
-          bump_counter('held_time', (monotime - acquired_at).to_i) if acquired_at
+          record_release(slot, acquired_at)
         end
       end
 
@@ -98,6 +80,29 @@ module Wurk
         "lmtr-stats:#{@name}"
       end
 
+      # A ZREM that removes nothing means an acquirer already evicted our slot:
+      # we outran `lock_timeout`, which is what `overages` counts. Exhausting
+      # `wait_timeout` is a different event — it raises OverLimit and the server
+      # middleware counts it in the job's `overrated` field.
+      def record_release(slot, acquired_at)
+        bump_counter('overages') if release(slot).to_i.zero?
+        bump_counter('held_time', (monotime - acquired_at).to_i)
+      end
+
+      # Spins until a slot frees up. Returns the monotonic acquire time, nil
+      # when `policy: :ignore` gives up; raises OverLimit past `wait_timeout`.
+      def wait_for_slot(slot, deadline)
+        loop do
+          return monotime if acquire(slot)[0].to_i == 1
+          return if @options[:policy] == :ignore
+
+          remaining = deadline - monotime
+          raise OverLimit, self if remaining <= 0
+
+          sleep [remaining, WAIT_SLEEP].min
+        end
+      end
+
       def acquire(slot)
         lua(:limiter_concurrent_acquire,
             keys: [state_key, stats_key],

data/lib/wurk/limiter/window.rb

@@ -19,18 +19,15 @@ module Wurk
       end
 
       def size
-        cutoff = ::Time.now.to_f - interval_seconds
-        Wurk::Limiter.redis do |c|
-          c.call('ZREMRANGEBYSCORE', state_key, '-inf', "(#{cutoff}")
-          c.call('ZCARD', state_key).to_i
-        end
+        window_state.first
       end
 
       # used = entries still inside the window; limit = count; reset_at =
       # when the oldest entry slides out (freeing a slot), or nil when
       # idle (#16).
       def status
-        build_status(used: size, limit: @options[:count], reset_at: oldest_expiry)
+        used, oldest = window_state
+        build_status(used: used, limit: @options[:count], reset_at: oldest && (oldest + interval_seconds))
       end
 
       def within_limit(used: 1, &block)
@@ -60,11 +57,16 @@ module Wurk
         "lmtr-w:#{@name}"
       end
 
-      # Oldest timestamp + interval = the moment it leaves the window.
-      def oldest_expiry
-        row = Wurk::Limiter.redis { |c| c.call('ZRANGE', state_key, 0, 0, 'WITHSCORES') }
-        score = Wurk::Limiter.first_score(row)
-        score && (score + interval_seconds)
+      # Count + oldest in-window score, both scoped by the Redis clock, in one
+      # read-only round trip. Reads never trim (the acquire script owns that):
+      # a dashboard host whose clock ran ahead used to evict live entries just
+      # by rendering `status`, freeing the window for a second full charge.
+      # Oldest timestamp + interval = the moment it leaves the window; -1 =
+      # nothing in window.
+      def window_state
+        count, oldest = lua(:limiter_window_status, keys: [state_key], argv: [interval_seconds])
+        score = oldest.to_f
+        [count.to_i, score.negative? ? nil : score]
       end
 
       def interval_seconds

data/lib/wurk/limiter.rb

@@ -4,6 +4,7 @@ require 'json'
 require 'digest'
 require 'securerandom'
 require_relative 'lua'
+require_relative 'pool_checkout'
 
 module Wurk
   # Sidekiq Enterprise rate limiters: concurrent, bucket, window, leaky,
@@ -24,7 +25,10 @@ module Wurk
   # Layout (one file per type under `lib/wurk/limiter/`):
   #   * `Limiter::Base` owns the metadata write (lmtr:{name}) + the global
   #     `lmtr-list` registration so the Web UI can list every limiter, and
-  #     the uniform `status` shape.
+  #     the uniform `status` shape. Membership is added once per metadata
+  #     write and swept back out by `Web::Enterprise::Limits.list` once the
+  #     metadata expires — the two halves of keeping `lmtr-list` bounded
+  #     under interpolated names.
   #   * Per-type subclasses (Concurrent / Bucket / Window / Leaky / Points)
   #     own their acquire/wait loop. Each delegates the atomic step to a
   #     Lua script in `lib/wurk/lua/limiter_*.lua`.
@@ -146,9 +150,8 @@ module Wurk
       # Redis access: caller-supplied pool (Limiter.configure.redis = …) wins,
       # else fall back to the default Wurk pool. This is the same hierarchy
       # Sidekiq Ent documents — dedicated rate-limiter pool is opt-in.
-      def redis(&)
-        pool = config.pool || Wurk.redis_pool
-        pool.with(&)
+      def redis(idempotent: false, &)
+        PoolCheckout.with(config.pool || Wurk.redis_pool, idempotent, &)
       end
 
       # `ZRANGE key 0 0 WITHSCORES` yields a single [member, score] pair, but the

data/lib/wurk/logger.rb

@@ -28,7 +28,7 @@ module Wurk
         }.freeze
 
         def tid
-          Thread.current[:wurk_tid] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+          Wurk::Component.tid
         end
 
         def format_context(ctxt = Wurk::Context.current)

data/lib/wurk/lua/loader.rb

@@ -23,9 +23,15 @@ module Wurk
         # connection errors are the pool wrapper's job (Wurk::RedisPool#with);
         # this only ships the loads.
         def script_load_all(redis)
-          redis.pipelined do |pipe|
-            SCRIPTS.each_value { |src| pipe.call('SCRIPT', 'LOAD', src) }
-          end
+          redis.pipelined { |pipe| queue_script_loads(pipe) }
+        end
+
+        # The same loads, queued onto a pipeline the caller already owns, for a
+        # caller with other work to batch with them: ChildBoot pairs them with
+        # its liveness PING so a child's whole Redis validation costs one round
+        # trip instead of two, on the boot-critical path.
+        def queue_script_loads(pipe)
+          SCRIPTS.each_value { |src| pipe.call('SCRIPT', 'LOAD', src) }
         end
 
         # @param redis [RedisClient] a single connection (not a pool)

data/lib/wurk/lua.rb

@@ -131,8 +131,16 @@ module Wurk
     # success after the dead job is manually retried to success). The
     # `b-<bid>-death` notify dedup key is untouched, so `:death` cannot
     # re-fire.
+    #
+    # The two EXPIRE NX calls are what keep the batch bounded: `b-<bid>-jids` is
+    # born here, and only `Batch#ensure_first_flush!` ever stamped a TTL — on the
+    # hash alone — so an abandoned or invalidated batch leaked its live-jid set
+    # forever. A late re-push (retry, scheduled promotion) can also resurrect an
+    # already-expired `b-<bid>` through the HINCRBYs above, again TTL-less. NX
+    # stamps only keys that currently have no TTL, so a running batch's clock and
+    # the shorter post-success `linger` window (Callbacks#apply_linger) both win.
     # KEYS = [b-<bid>, b-<bid>-jids, queue_list, queues_set, b-<bid>-died, dead-batches]
-    # ARGV = [queue_name, jid, job_json, bid]
+    # ARGV = [queue_name, jid, job_json, bid, expiry_seconds]
     # Returns 1.
     BATCH_PUSH = <<~LUA
       if redis.call("srem", KEYS[5], ARGV[2]) == 1 then
@@ -147,6 +155,8 @@ module Wurk
           redis.call("hincrby", KEYS[1], "pending", 1)
         end
       end
+      redis.call("expire", KEYS[1], ARGV[5], "NX")
+      redis.call("expire", KEYS[2], ARGV[5], "NX")
       redis.call("sadd", KEYS[4], ARGV[1])
       redis.call("lpush", KEYS[3], ARGV[3])
       return 1
@@ -165,14 +175,19 @@ module Wurk
     # it, the re-push routes through BATCH_PUSH, whose guard finds the jid already
     # live (SADD == 0) → pure LPUSH, no recount. So registration happens exactly
     # once, here, at enqueue.
+    #
+    # The other path that can create `b-<bid>-jids`, so it carries the same NX
+    # expiry stamp as BATCH_PUSH.
     # KEYS = [schedule, b-<bid>, b-<bid>-jids]
-    # ARGV = [at_score, job_json, jid]
+    # ARGV = [at_score, job_json, jid, expiry_seconds]
     # Returns 1.
     BATCH_SCHEDULE = <<~LUA
       if redis.call("sadd", KEYS[3], ARGV[3]) == 1 then
         redis.call("hincrby", KEYS[2], "total", 1)
         redis.call("hincrby", KEYS[2], "pending", 1)
       end
+      redis.call("expire", KEYS[2], ARGV[4], "NX")
+      redis.call("expire", KEYS[3], ARGV[4], "NX")
       redis.call("zadd", KEYS[1], ARGV[1], ARGV[2])
       return 1
     LUA
@@ -208,13 +223,18 @@ module Wurk
     # currently in a failing/retrying state. Re-failures of the same jid are
     # idempotent. Cleared by BATCH_ACK_SUCCESS (retry passed) or
     # BATCH_ACK_COMPLETE (job died). Spec §2.5, §2.8.
+    #
+    # `b-<bid>-failed` is born here, so it carries the same NX expiry stamp as
+    # BATCH_PUSH.
     # KEYS = [b-<bid>, b-<bid>-failed]
-    # ARGV = [jid]
+    # ARGV = [jid, expiry_seconds]
     # Returns 1.
     BATCH_ACK_FAILED = <<~LUA
       if redis.call("sadd", KEYS[2], ARGV[1]) == 1 then
         redis.call("hincrby", KEYS[1], "failures", 1)
       end
+      redis.call("expire", KEYS[1], ARGV[2], "NX")
+      redis.call("expire", KEYS[2], ARGV[2], "NX")
       return 1
     LUA
 
@@ -224,8 +244,11 @@ module Wurk
     # live jids so the batch can fire `:complete` even with terminally failed
     # jobs. `b-<bid>-failed` holds only currently-retrying jids; `b-<bid>-died`
     # holds terminally-dead ones (spec §2.8 — the two sets are distinct).
+    #
+    # `b-<bid>-died` is born here, so it carries the same NX expiry stamp as
+    # BATCH_PUSH.
     # KEYS = [b-<bid>, b-<bid>-jids, b-<bid>-died, b-<bid>-failed]
-    # ARGV = [jid]
+    # ARGV = [jid, expiry_seconds]
     # Returns [live_jids_remaining, died_count, first_death]. `first_death`
     # is 1 the first time *any* jid is SADDed into the died set, 0 thereafter
     # — caller uses it to fire `:death` exactly once per batch.
@@ -236,6 +259,8 @@ module Wurk
         redis.call("hincrby", KEYS[1], "failures", -1)
       end
       local died_added = redis.call("sadd", KEYS[3], ARGV[1])
+      redis.call("expire", KEYS[1], ARGV[2], "NX")
+      redis.call("expire", KEYS[3], ARGV[2], "NX")
       local first_death = 0
       if was_pre_existing_death == 0 and died_added == 1 then
         first_death = 1
@@ -262,10 +287,21 @@ module Wurk
     # same reopened batch cannot lose each other's writes. Refuses to write
     # when the batch hash is gone — resurrecting a bare hash would create a
     # batch that can never fire anything.
+    #
+    # Appends are deduped and capped. Reopening the same batch inside every
+    # job and re-registering its callback is a common shape, and each append
+    # pays a decode + encode of the whole array: unbounded that is O(N^2) Lua
+    # work on the way in and N identical callback jobs at fire time. An
+    # identical triple is therefore a no-op — the registration is already
+    # satisfied by the entry that is there — and past the cap the append is
+    # refused rather than allowed to grow the hash field without limit.
+    # Both sides of the comparison are normalised through cjson so an entry
+    # written by Ruby's `to_json` at first flush matches one appended here.
     # KEYS = [b-<bid>]
-    # ARGV = [callback triple JSON, event name]
-    # Returns -1 when the batch hash does not exist; otherwise the event's
-    # fired flag ("1", or nil when it has not fired yet).
+    # ARGV = [callback triple JSON, event name, max callbacks]
+    # Returns -1 when the batch hash does not exist and -2 when the cap
+    # refused the append; otherwise the event's fired flag ("1", or nil when
+    # it has not fired yet).
     BATCH_APPEND_CALLBACK = <<~LUA
       if redis.call("exists", KEYS[1]) == 0 then
         return -1
@@ -277,19 +313,30 @@ module Wurk
       else
         list = {}
       end
-      list[#list + 1] = cjson.decode(ARGV[1])
+      local entry = cjson.decode(ARGV[1])
+      local encoded = cjson.encode(entry)
+      for i = 1, #list do
+        if cjson.encode(list[i]) == encoded then
+          return redis.call("hget", KEYS[1], ARGV[2])
+        end
+      end
+      if #list >= tonumber(ARGV[3]) then
+        return -2
+      end
+      list[#list + 1] = entry
       redis.call("hset", KEYS[1], "callbacks", cjson.encode(list))
       return redis.call("hget", KEYS[1], ARGV[2])
     LUA
 
     # Ent Unique (§3): atomic compare-and-delete of a lock key. Replaces the
     # two-command GET-then-DEL — between those calls the key can expire and a
-    # fresh enqueue can grab it, and the bare DEL would then drop the new
+    # fresh owner can grab it, and the bare DEL would then drop the new
     # owner's lock. Shared by `Unique::ServerMiddleware#release` (normal
-    # success/start release) and `Unique::DEATH_HANDLER` (automatic-death
-    # release) so the two paths cannot drift.
-    # KEYS = [unique:<sha256>]
-    # ARGV = [owning jid]
+    # success/start release), `Unique::DEATH_HANDLER` (automatic-death
+    # release) and `Leader#release` (stepping down from the cluster lock)
+    # so those paths cannot drift.
+    # KEYS = [the lock key — unique:<sha256> | dear-leader]
+    # ARGV = [the owner that must still hold it — jid | <host>:<pid>:<nonce>]
     # Returns 1 when the key was deleted, 0 otherwise.
     RELEASE_IF_OWNER = <<~LUA
       if redis.call("get", KEYS[1]) == ARGV[1] then
@@ -334,6 +381,41 @@ module Wurk
       return removed
     LUA
 
+    # Ent Periodic (§2): compare-and-swap the fire marks of `loops:<lid>`.
+    # The cron poller's leader gate is a cached read (`Component#leader?`,
+    # LEADER_CACHE_TTL_MS), so for a few seconds after a handover two
+    # processes both believe they lead and both reach the same due loop —
+    # HMGET → decide → enqueue → HSET then fires it twice. Claiming the slot
+    # atomically is what makes a tick fire exactly once; the loser gets 0 and
+    # enqueues nothing. Values written are the same decimal-epoch strings the
+    # Ruby path wrote, byte for byte — the dashboard reads these fields.
+    #
+    # `nf` present: the caller must still be looking at the mark it read
+    # (byte compare, so no parse can drift the token).
+    # `nf` absent: the caller derived the slot from `lf`, so refuse once `lf`
+    # has reached it — that is the exhausted-schedule case, where the winner
+    # cleared `nf` and a loser would otherwise re-fire the very same slot.
+    # KEYS = [loops:<lid>]
+    # ARGV = [expected `nf` (or the derived slot), new `lf`, new `nf` ('' → HDEL)]
+    # Returns 1 when this caller claimed the slot, 0 otherwise.
+    CRON_CLAIM_FIRE = <<~LUA
+      local key = KEYS[1]
+      local cur = redis.call("hget", key, "nf")
+      if cur and cur ~= "" then
+        if cur ~= ARGV[1] then return 0 end
+      else
+        local lf = tonumber(redis.call("hget", key, "lf") or "")
+        if lf and lf >= tonumber(ARGV[1]) then return 0 end
+      end
+      redis.call("hset", key, "lf", ARGV[2])
+      if ARGV[3] == "" then
+        redis.call("hdel", key, "nf")
+      else
+        redis.call("hset", key, "nf", ARGV[3])
+      end
+      return 1
+    LUA
+
     # Limiter scripts live in `lib/wurk/lua/limiter_*.lua` — one file per
     # type. Loaded at boot, the file's basename (minus `.lua`) becomes the
     # SCRIPTS key as a symbol. Keeping them as separate files makes diffing
@@ -358,7 +440,8 @@ module Wurk
       batch_append_callback: BATCH_APPEND_CALLBACK,
       fast_delete_job: FAST_DELETE_JOB,
       fast_delete_by_class: FAST_DELETE_BY_CLASS,
-      release_if_owner: RELEASE_IF_OWNER
+      release_if_owner: RELEASE_IF_OWNER,
+      cron_claim_fire: CRON_CLAIM_FIRE
     }.merge(FILE_SCRIPTS).freeze
 
     # SHA1 of each script source — matches what `SCRIPT LOAD` returns.

data/lib/wurk/manager.rb

@@ -24,12 +24,21 @@ module Wurk
 
     attr_reader :workers, :capsule
 
-    def initialize(capsule)
+    # `shutdown:` is the owning Launcher's process-wide shutdown request,
+    # invoked when concurrency can no longer be held (see #processor_result).
+    # Injected rather than reached for: only the entry point knows how this
+    # process exits — a swarm child / standalone CLI re-delivers TERM to itself
+    # so the installed trap drives the normal drain, while embedded mode owns
+    # no traps and must stop the launcher in place. A Manager built without an
+    # owner (Sidekiq's `Manager.new(capsule)` arity, kept for the alias) has no
+    # route to take, so it only reports.
+    def initialize(capsule, shutdown: nil)
       @config = @capsule = capsule
       @count = capsule.concurrency
       raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
 
       @done = false
+      @shutdown = shutdown
       @workers = Set.new
       @plock = ::Mutex.new
       @count.times do
@@ -86,8 +95,9 @@ module Wurk
     # cleanly or via raised exception. Removes the dead processor from the
     # pool and (unless we're already stopping) spawns a replacement so the
     # capsule's concurrency stays constant. If the replacement itself can't be
-    # spawned, crash the child (the swarm respawns it) rather than silently
-    # dropping concurrency. Snapshot under @plock; start the replacement — a
+    # spawned, ask the owner to shut this process down (the swarm respawns it at
+    # full concurrency) rather than silently dropping a worker for the life of
+    # the process. Snapshot under @plock; start the replacement — a
     # side effect — outside the lock.
     def processor_result(processor, _reason = nil)
       replacement = @plock.synchronize do
@@ -102,10 +112,15 @@ module Wurk
     rescue StandardError => e
       # Replacement spawn failed (e.g. ThreadError at the OS thread limit).
       # Silently running one Processor short for the life of the process is
-      # invisible degradation; instead report and crash the child on the main
-      # thread so the swarm respawns it at full concurrency (plan 02 §6).
+      # invisible degradation, so take the process down (the swarm respawns it
+      # at full concurrency) — but through the owner's TERM path, never
+      # `Thread.main.raise`: that raise unwound straight past Manager#stop, so
+      # the in-flight UnitsOfWork were never bulk_requeued and each one waited
+      # out a full reaper interval, and in embedded mode it killed the host's
+      # main thread instead of just the worker. Non-blocking by contract — we
+      # are on the dying Processor's own thread, which the drain will kill.
       @capsule.config.handle_exception(e, { context: 'Manager could not replace a dead Processor' })
-      main_thread.raise(e)
+      @shutdown&.call
     end
 
     # Reached when the deadline expired with workers still busy. Atomically
@@ -146,14 +161,15 @@ module Wurk
       @plock.synchronize { @workers.dup }
     end
 
+    # Drained means "no processor thread is still running", not "the Set is
+    # empty": a processor removes itself from @workers from inside its own
+    # thread, so the two agree — except for a processor that was never started
+    # (a launcher that raised mid-boot and rolled back). That one holds no
+    # thread and nothing will ever remove it, so an emptiness test would make
+    # #stop poll out the entire shutdown deadline waiting on a worker that
+    # never ran.
     def workers_empty?
-      @plock.synchronize { @workers.empty? }
-    end
-
-    # Seam over Thread.main: lets processor_result's replacement-failure crash
-    # be unit-tested against a controlled thread instead of the live runner.
-    def main_thread
-      Thread.main
+      @plock.synchronize { @workers.none? { |w| w.thread&.alive? } }
     end
 
     # Polls `condblock` until it returns true or the monotonic deadline