wurk 1.9.0 → 1.10.0

data/lib/wurk/fetcher/orphan_grace.rb

@@ -2,65 +2,81 @@
 
 module Wurk
   class Fetcher
-    # The Reaper's boot-grace check: has this heartbeat-judged orphan been
-    # left alone for at least `grace` seconds? A booting worker can claim a job
-    # a moment before its first heartbeat lands, and draining its list in that
-    # window runs the job twice, concurrently.
+    # The Reaper's wait between judging a list orphaned by its heartbeat and
+    # draining it.
     #
-    # The answer comes from OBJECT IDLETIME, which a live owner resets on every
-    # claim and ACK. Some servers refuse it — Redis under an LFU maxmemory
-    # policy, Dragonfly with no OBJECT at all — and the grace must not quietly
-    # vanish with it. Without IDLETIME the clock is this reaper's own: a list
-    # is drained only once it has been judged orphaned for `grace` seconds
-    # across sweeps, by which time a booting owner has beaten and the
-    # heartbeat re-check spares it. The switch is logged once.
+    # A live owner can read as having no heartbeat hash: its first beat failed on
+    # a Redis blip, CPU-bound jobs held the beat back, or Redis restarted from an
+    # older snapshot. Its next beat lands within the wait, and that spares it.
+    #
+    # The clock is this process's own and it is shared by nothing else, so it
+    # costs the same on every server and at every fleet size. That is the
+    # difference from what it replaces, OBJECT IDLETIME: any read of the list
+    # (a monitor's LLEN) reset that clock, some servers do not have the command,
+    # and a clock kept per sweep only advanced in a process that won the sweep
+    # lock twice. The loop thread therefore finishes what any sweep here started,
+    # with or without the lock (Reaper#drain_settled).
     class OrphanGrace
-      NO_IDLETIME_WARNING = 'reaper: OBJECT IDLETIME is unavailable on this Redis (%s); the boot grace ' \
-                            'before reclaiming a private list now runs on this process\'s own clock, so an ' \
-                            'orphan is drained one sweep later than it would be. Expected under an LFU ' \
-                            'maxmemory policy or on servers without OBJECT (e.g. Dragonfly).'
-
-      # Lists remembered for the fallback clock. A list leaves on the pass
-      # that clears it; one judged orphaned and then found alive can linger,
-      # so the memory is bounded and simply restarts when full — every clock
-      # restarting is the conservative direction.
+      # Lists remembered at once. Past it the memory restarts, and every clock
+      # restarting is the conservative direction: the next sweep judges them anew.
       MEMORY = 10_000
 
-      def initialize(config, grace)
-        @config = config
+      Entry = Struct.new(:since, :owner)
+
+      def initialize(grace)
         @grace = grace
-        @idletime = true
-        @first_seen = {}
+        @lock = ::Mutex.new
+        @entries = {}
       end
 
-      # true while the list must be left alone.
-      def recent?(key)
+      # true while the list must be left alone. `owner` is what #take_settled
+      # hands back when the wait is over.
+      def recent?(key, owner)
         return false unless @grace.positive?
-        return local_recent?(key) unless @idletime
 
-        idle = @config.redis(idempotent: true) { |c| c.call('OBJECT', 'IDLETIME', key) }
-        # nil: the list is already gone, nothing to drain.
-        idle.nil? || idle < @grace
-      rescue RedisClient::CommandError => e
-        idletime_unavailable!(e)
-        local_recent?(key)
+        now = monotonic
+        @lock.synchronize do
+          @entries.clear if @entries.size >= MEMORY && !@entries.key?(key)
+          entry = (@entries[key] ||= Entry.new(now, owner))
+          next true if now - entry.since < @grace
+
+          @entries.delete(key)
+          false
+        end
       end
 
-      private
+      # The owner came back: its next judgement starts a new wait.
+      def forget(key)
+        @lock.synchronize { @entries.delete(key) }
+      end
 
-      def local_recent?(key)
-        now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-        @first_seen.clear if @first_seen.size >= MEMORY && !@first_seen.key?(key)
-        seen = (@first_seen[key] ||= now)
-        return true if now - seen < @grace
+      # [[key, owner], ...] whose wait is over, removed from memory.
+      def take_settled
+        now = monotonic
+        @lock.synchronize do
+          settled = @entries.select { |_, entry| now - entry.since >= @grace }
+          settled.each_key { |key| @entries.delete(key) }
+          settled.map { |key, entry| [key, entry.owner] }
+        end
+      end
+
+      # Seconds until the next wait ends, nil with nothing waiting.
+      def next_due_in
+        now = monotonic
+        @lock.synchronize do
+          oldest = @entries.each_value.map(&:since).min
+          oldest && [(oldest + @grace) - now, 0].max
+        end
+      end
 
-        @first_seen.delete(key)
-        false
+      def size
+        @lock.synchronize { @entries.size }
       end
 
-      def idletime_unavailable!(error)
-        @idletime = false
-        @config.logger.warn { format(NO_IDLETIME_WARNING, error.message.lines.first&.strip) }
+      private
+
+      def monotonic
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
     end
   end

data/lib/wurk/fetcher/private_list_key.rb

@@ -38,19 +38,6 @@ module Wurk
         nil
       end
 
-      # `<public_q>|<host>|<pid>|<nonce>|<idx>` → [host, pid, nonce] (pid as
-      # Integer), or all-nil when the suffix isn't a well-formed owner tail.
-      # Splitting the suffix off the known public-queue prefix tolerates a `|`
-      # inside the queue name itself, and leaves the tail unambiguous: exactly
-      # 4 segments for the current shape, exactly 3 for the pre-nonce one.
-      def parse_owner(public_q, key)
-        suffix = key.delete_prefix("#{public_q}|")
-        return [nil, nil, nil] if suffix == key
-
-        host, pid, nonce = owner_tails(suffix.split('|')).first
-        [host, pid, nonce]
-      end
-
       # Owner segments of a private-list key, taken from the right, as
       # [host, pid, nonce, segment_count] readings in preference order (empty
       # when nothing parses). The wide shape is preferred: an all-digit nonce is
@@ -67,9 +54,7 @@ module Wurk
       # The nonce slot must carry PROCESS_NONCE's exact shape (NONCE_PATTERN):
       # on a pre-nonce key read wide it holds the real pid, which is never 12
       # hex, so the guard is what falls such a key through to narrow_tail
-      # instead of misreading it. Guarded here rather than in parse_full
-      # because parse_owner reads through owner_tails too — one guard, and the
-      # two parsers agree on every input.
+      # instead of misreading it.
       def wide_tail(parts)
         [parts[-4], parts[-3].to_i, parts[-2], 4] if parts.size >= 4 && integer?(parts[-3]) && nonce?(parts[-2])
       end

data/lib/wurk/fetcher/reaper.rb

@@ -3,10 +3,13 @@
 require_relative '../component'
 require_relative '../keys'
 require_relative '../middleware/poison_pill'
+require_relative '../metrics/process_counters'
 require_relative '../timer_loop'
 require_relative 'private_list_key'
 require_relative 'unparseable_keys'
 require_relative 'orphan_grace'
+require_relative 'oom_drain'
+require_relative 'self_reconcile'
 
 module Wurk
   class Fetcher
@@ -31,87 +34,94 @@ module Wurk
     #     supervisor respawns with a fresh pid, so it does not arise in practice.)
     #   * any other incarnation — its pid means nothing in our namespace, so we
     #     trust the heartbeat: the owner is alive iff its identity's `info`
-    #     hash still exists. Such reclaim waits out the 60s heartbeat TTL.
+    #     hash still exists. Such reclaim waits out the 60s heartbeat TTL, the
+    #     next sweep and then the grace: 80-150s after the owner died.
     #
-    # The heartbeat verdict is a snapshot taken before a SCAN that can run for
-    # a long time on a big keyspace, and a booting process can claim jobs a
-    # moment before its first beat lands. Draining on that snapshot alone
-    # reclaims a live worker's in-flight job — a double run. So a heartbeat-
-    # judged orphan is re-checked right before its drain: its list must have
-    # sat untouched for `grace` seconds (OBJECT IDLETIME; a live owner touches
-    # it on every claim and ACK), and its identity must still have no `info`
-    # hash. Our own incarnation skips both: kill(0) is not a snapshot.
+    # A missing hash is not proof of death on its own. The verdict is a snapshot
+    # taken before a SCAN that can run for a long time on a big keyspace, a beat
+    # can be held back by CPU-bound jobs, and a Redis restarted from an older
+    # snapshot brings back the `processes` SET with every hash lapsed. So a
+    # heartbeat-judged orphan must stay orphaned for `grace` seconds on this
+    # process's own clock (OrphanGrace), and its identity must still have no
+    # `info` hash. Our own incarnation skips both: kill(0) is not a snapshot.
     #
     # Re-pushed jobs run through Wurk::Middleware::PoisonPill, which caps a
     # job at RECOVERY_THRESHOLD recoveries within 72h: past the cap the job
     # is killed into the dead set instead of re-queued, so a job that crashes
     # its worker every time can't loop forever.
     #
-    # The reaper runs two passes, exactly as super_fetch's sweeper does:
+    # One sweep an interval: one SCAN of `queue:*|*` acting on every list, gated
+    # by a cluster lock so about one process sweeps per interval. Every list, and
+    # not only those this process serves: the lock is shared across the fleet, so
+    # a scoped sweep would wait for its turn to land on one of the processes
+    # serving the queue. The list's own queue is where its jobs go back to,
+    # whichever process moves them.
     #
-    #   * a *scoped* sweep every interval ("1/min within process group"): SCANs
-    #     only the public queues this process serves, gated by a cluster `SET NX
-    #     EX` lock so one process sweeps per interval. The cheap common path.
-    #   * a *full* sweep at most once an hour ("full SCAN 1/hr"): SCANs the whole
-    #     `queue:*|*` keyspace, gated by its own hourly lock, so private lists
-    #     whose public queue no live process serves — a renamed/decommissioned
-    #     queue, or a dead host's queue no survivor consumes — are recovered too,
-    #     not stranded forever.
+    # Every tick also runs SelfReconcile over this process's own lists, which is
+    # the one case liveness cannot answer: the owner is alive and has lost track
+    # of a job.
     #
     # Spec: docs/target/sidekiq-pro.md §3.2.
-    class Reaper
+    class Reaper # rubocop:disable Metrics/ClassLength
       include Component
 
-      # Sweep cadence in seconds; also the cluster-lock TTL so exactly one
-      # process sweeps per interval. 60s matches the heartbeat TTL — the
-      # floor below which cross-host orphans can't be detected anyway.
+      # Sweep cadence in seconds. 60s matches the heartbeat TTL — the floor below
+      # which cross-host orphans can't be detected anyway.
       DEFAULT_INTERVAL = 60
 
-      # Full-keyspace sweep cadence + its lock TTL: at most once per hour across
-      # the fleet, since a global SCAN is far costlier than the scoped pass.
-      FULL_INTERVAL = 3600
+      # The lock lapses before the winner's next tick, or a sole survivor would
+      # still find its own lock there and sweep every second interval.
+      LOCK_TTL_RATIO = 0.8
 
-      # Seconds a heartbeat-judged private list must have sat idle before it is
-      # drained. Far below the heartbeat TTL a dead owner's list already waits
-      # out, so it never delays a real reclaim; well above the gap between a
-      # booting process's first claim and its first beat.
+      # Seconds a heartbeat-judged private list stays orphaned before it is
+      # drained. A beat is due every 10s, and 25 CPU-bound jobs were measured
+      # holding one back 17.4s: at a 15s grace one trial in nine drained all 25
+      # live jobs. The price is paid only by a host lost for good.
       DEFAULT_GRACE = 30
 
       LOCK_KEY = 'super_fetch:reaper'
-      FULL_LOCK_KEY = 'super_fetch:reaper:full'
-      SCAN_COUNT = 100
-
-      # The hourly sweep walks the whole keyspace, where almost nothing
-      # matches: COUNT is the work per SCAN call, so a bigger batch is the
-      # same server work in a tenth of the round trips — a million keys is
-      # 1,000 calls instead of 10,000. Still small enough that one call never
-      # holds Redis for more than a fraction of a millisecond.
-      FULL_SCAN_COUNT = 1000
+
+      # A sweep walks the whole keyspace, where almost nothing matches: COUNT is
+      # the work per SCAN call, so a bigger batch is the same server work in a
+      # tenth of the round trips — a million keys is 1,000 calls instead of
+      # 10,000. Still small enough that one call never holds Redis for more than
+      # a fraction of a millisecond.
+      SCAN_COUNT = 1000
       THREAD_NAME = 'wurk-reaper'
 
       attr_reader :interval
 
-      def initialize(config, interval: DEFAULT_INTERVAL, lock_key: LOCK_KEY,
-                     full_interval: FULL_INTERVAL, full_lock_key: FULL_LOCK_KEY, grace: DEFAULT_GRACE)
+      def initialize(config, interval: DEFAULT_INTERVAL, lock_key: LOCK_KEY, grace: DEFAULT_GRACE,
+                     reconcile_confirm: SelfReconcile::DEFAULT_CONFIRM)
         @config = config
         @interval = interval
-        @grace = OrphanGrace.new(config, grace)
+        @grace = OrphanGrace.new(grace)
         @lock_key = lock_key
-        @full_interval = full_interval
-        @full_lock_key = full_lock_key
         @thread = nil
         @done = false
         @mutex = ::Mutex.new
         @sleeper = ::ConditionVariable.new
         @unparseable = UnparseableKeys.new(config)
+        @next_sweep_at = 0.0
+        @self_reconcile = SelfReconcile.new(config, confirm: reconcile_confirm, stop: -> { done? },
+                                                    on_moved: method(:recovered))
       end
 
       # Spawns the sweep loop. Idempotent. The loop waits one interval before
       # its first sweep so booting processes don't dogpile Redis and so an
       # un-stopped launcher in a unit test never touches the keyspace.
+      #
+      # A LIVE loop is the one and only sweep loop and is handed back as-is. A
+      # dead one is not: a stop whose bounded join timed out leaves its
+      # straggler referenced, and once that straggler has wound down the
+      # reference is a thread that will never sweep again — handing it back
+      # would leave the reaper dead for good, silently, since `start` looked
+      # like it had worked. A fresh loop takes its place; while the straggler
+      # is still alive it keeps the sweep to itself, so a restart still cannot
+      # run two loops at once.
       def start
         @mutex.synchronize do
-          return @thread if @thread
+          return @thread if @thread&.alive?
 
           @done = false
           @thread = spawn_loop_thread
@@ -120,7 +130,7 @@ module Wurk
       end
 
       # Bounded at TimerLoop::JOIN_TIMEOUT like every other periodic component.
-      # The full-keyspace sweep is exactly the tick that outlasts a stop: it
+      # A sweep is exactly the tick that outlasts a stop: it
       # SCANs the whole `queue:*|*` keyspace and drains what it finds, so on a
       # large or slow Redis it can still be running when shutdown lands. Waiting
       # it out held the entire process's teardown open past the swarm parent's
@@ -144,18 +154,15 @@ module Wurk
         !@thread.nil? && @thread.alive?
       end
 
-      # One loop tick: the scoped sweep when this process wins the per-interval
-      # lock, plus the full-keyspace sweep when it also wins the hourly lock.
-      # Returns the total jobs reclaimed across both.
+      # One loop tick: a sweep when this process wins the per-interval lock.
+      # Returns the jobs reclaimed.
       def reap
-        reclaimed = acquire_lock? ? reclaim! : 0
-        reclaimed += reclaim_full! if acquire_full_lock?
-        reclaimed
+        acquire_lock? ? reclaim! : 0
       end
 
-      # One unguarded sweep over every served queue. Returns the number of
-      # jobs reclaimed (re-queued or killed). Public so boot paths and tests
-      # can drive a deterministic pass without the cluster lock.
+      # One unguarded sweep over every private list. Returns the number of jobs
+      # reclaimed (re-queued or killed). Public so boot paths and tests can drive
+      # a deterministic pass without the cluster lock.
       #
       # Honours #stop at each boundary: a stop bounding its join leaves the
       # sweep a straggler by design, and the straggler finishes the list it is
@@ -163,90 +170,65 @@ module Wurk
       # otherwise it keeps requeueing orphans and holding the sweep locks
       # past the stop, into whatever runs next.
       def reclaim!
-        owners = live_owners
-        reclaimed = 0
-        served_queues.each do |public_q|
-          break if done?
+        sweep.tap { wake_for_settling }
+      end
 
-          reclaimed += reclaim_queue(public_q, owners)
-        end
-        reclaimed
+      # #reclaim! for a booting process. Deliberately under no lock, not even a
+      # short one: a replacement's sweep is what recovers the sibling it replaces,
+      # and the sibling may have died a moment after another process booted (a
+      # 2s lock turned a child killed right after boot from 1s into 61s). One
+      # pass per process costs keyspace / SCAN_COUNT calls.
+      def boot_reclaim!
+        reclaim!
       end
 
-      # One unguarded full-keyspace sweep: every `queue:*|*` private list, even
-      # ones whose public queue this process doesn't serve. Returns the number
-      # of jobs reclaimed. Public so boot paths and tests can drive it without
-      # the hourly lock.
-      def reclaim_full!
-        owners = live_owners
-        reclaimed = 0
-        each_full_private_list do |key, public_q, host, pid, nonce|
-          break if done?
-          next unless orphaned?(key, host, pid, nonce, owners)
+      # Lists whose grace has run out, whichever sweep started it. Runs on every
+      # tick with no lock: the sweep lock rotates between processes, and waiting
+      # to win it again is what made reclaim slower as a fleet grew. Two processes
+      # draining one list is safe, each move takes one job once.
+      def drain_settled
+        @grace.take_settled.sum do |key, (public_q, host, pid, nonce)|
+          next 0 if done?
 
-          reclaimed += drain(key, public_q)
+          owner_gone?(host, pid, nonce) ? drain(key, public_q) : 0
         end
-        reclaimed
       end
 
       private
 
-      # Union of `queue:<name>` keys across every capsule this process serves.
-      # Scoping the scan to these keeps the keyspace we touch bounded and lets
-      # parallel test namespaces stay isolated.
-      def served_queues
-        @config.capsules.each_value
-               .flat_map(&:queues)
-               .uniq
-               .map { |name| Keys.queue(name) }
+      # A sweep off the loop thread (the boot sweep) may have started a grace the
+      # loop's current wait knows nothing about.
+      def wake_for_settling
+        @mutex.synchronize { @sleeper.signal }
       end
 
-      # SCAN for this public queue's private lists, reclaim the orphaned ones.
-      def reclaim_queue(public_q, owners)
+      def sweep
+        owners = live_owners
         reclaimed = 0
-        each_private_list(public_q) do |key, host, pid, nonce|
+        each_private_list do |key, owner|
           break if done?
-          next unless orphaned?(key, host, pid, nonce, owners)
+          next unless orphaned?(key, owner, owners)
 
-          reclaimed += drain(key, public_q)
+          reclaimed += drain(key, owner.first)
         end
         reclaimed
       end
 
-      # Yields [private_list_key, host, pid, nonce] for each private list of
-      # `public_q`. MATCH `<public_q>|*` matches only this queue's private
-      # lists (public queue keys carry no `|`).
-      def each_private_list(public_q)
-        cursor = '0'
-        loop do
-          break if done?
-
-          cursor, keys = redis(idempotent: true) do |c|
-            c.call('SCAN', cursor, 'MATCH', "#{public_q}|*", 'COUNT', SCAN_COUNT)
-          end
-          keys.each do |key|
-            host, pid, nonce = PrivateListKey.parse_owner(public_q, key)
-            pid ? yield(key, host, pid, nonce) : @unparseable.report(key)
-          end
-          break if cursor == '0'
-        end
-      end
-
-      # Yields [private_list_key, public_q, host, pid, nonce] for every list in
-      # the keyspace. MATCH `queue:*|*` matches only private lists (public queue
+      # Yields private_list_key, [public_q, host, pid, nonce] for every list.
+      # MATCH `queue:*|*` matches only private lists (public queue
       # keys carry no `|`); PrivateListKey.parse_full drops anything that isn't a
       # well-formed `queue:<public>|<host>|<pid>|<nonce>|<idx>`.
-      def each_full_private_list
+      def each_private_list
         cursor = '0'
         loop do
           break if done?
 
           cursor, keys = redis(idempotent: true) do |c|
-            c.call('SCAN', cursor, 'MATCH', "#{Keys::QUEUE_PREFIX}*|*", 'COUNT', FULL_SCAN_COUNT)
+            c.call('SCAN', cursor, 'MATCH', "#{Keys::QUEUE_PREFIX}*|*", 'COUNT', SCAN_COUNT)
           end
           keys.each do |key|
             parsed = PrivateListKey.parse_full(key)
-            parsed ? yield(key, *parsed) : @unparseable.report(key)
+            parsed ? yield(key, parsed) : @unparseable.report(key)
           end
           break if cursor == '0'
         end
@@ -271,11 +253,17 @@ module Wurk
       # proves it alive, and a nonce-bearing identity is alive on its hash
       # alone even once it has been pruned from the SET. A pre-nonce key can
       # only be matched on the `<host>:<pid>` prefix of that identity.
-      def orphaned?(key, host, pid, nonce, owners)
+      def orphaned?(key, owner, owners)
+        _public_q, host, pid, nonce = owner
         return !local_pid_alive?(pid) if nonce == process_nonce && host == hostname
-        return false if owners.include?(owner_key(host, pid, nonce))
 
-        settled_orphan?(key, host, pid, nonce)
+        if owners.include?(owner_key(host, pid, nonce))
+          @grace.forget(key)
+          return false
+        end
+        return false if @grace.recent?(key, owner)
+
+        owner_gone?(host, pid, nonce)
       end
 
       def owner_key(host, pid, nonce)
@@ -283,11 +271,10 @@ module Wurk
       end
 
       # The re-check right before a heartbeat-judged drain (see the class doc):
-      # the `owners` snapshot may predate the owner's first beat, or the whole
-      # SCAN. Only reached for lists the snapshot already called orphaned, so
-      # its round trips are paid per orphan, never per live list.
-      def settled_orphan?(key, host, pid, nonce)
-        return false if @grace.recent?(key)
+      # the `owners` snapshot may predate the owner's beat, or the whole SCAN.
+      # Only reached for lists the snapshot already called orphaned, so its round
+      # trips are paid per orphan, never per live list.
+      def owner_gone?(host, pid, nonce)
         return !live_owners.include?(owner_key(host, pid, nil)) unless nonce
 
         redis(idempotent: true) { |c| c.call('HGET', owner_key(host, pid, nonce), 'info') }.nil?
@@ -341,10 +328,11 @@ module Wurk
         queue_name = public_q.delete_prefix(Keys::QUEUE_PREFIX)
         count = 0
         loop do
-          job = redis { |c| c.call('LMOVE', private_list, public_q, 'RIGHT', 'RIGHT') }
+          job = redis { |c| reclaim_one(c, private_list, public_q) }
           break unless job
 
           count += 1
+          Metrics::ProcessCounters.increment(:reaper_reclaimed)
           poison_off(public_q, job, queue_name)
         end
         count
@@ -353,20 +341,42 @@ module Wurk
         count
       end
 
+      # A full Redis must not hold a dead worker's jobs back from the workers
+      # still able to drain the queue.
+      def reclaim_one(conn, private_list, public_q)
+        Wurk::Lua::Loader.eval_cached(conn, :reliable_reclaim, keys: [private_list, public_q], argv: [])
+      rescue RedisClient::OutOfMemoryError => e
+        OomDrain.reclaim(conn, e, private_list, public_q)
+      end
+
+      # SelfReconcile put a payload back: the same accounting as a reclaim, so a
+      # job that kills its processor thread every time still hits the poison cap.
+      def recovered(public_q, job)
+        Metrics::ProcessCounters.increment(:reaper_reclaimed)
+        poison_off(public_q, job, public_q.delete_prefix(Keys::QUEUE_PREFIX))
+      end
+
+      # The job is back on its queue whatever happens here. Past maxmemory the
+      # count can't be taken (nor the job killed, leaving it where it is): raising
+      # would end the drain at one job per pass, with the rest of the list waiting
+      # on memory they would help free.
       def poison_off(public_q, job, queue_name)
         return unless Middleware::PoisonPill.track!(job, queue: queue_name, config: @config) == :poison
 
         # track! already ZADDed the payload to the dead set; pull the copy we
-        # just LMOVE'd onto the public tail so it isn't also re-run.
+        # just moved onto the public tail so it isn't also re-run.
         redis { |c| c.call('LREM', public_q, -1, job) }
+      rescue RedisClient::OutOfMemoryError => e
+        report(e)
       end
 
       def acquire_lock?
-        redis { |c| c.call('SET', @lock_key, '1', 'NX', 'EX', @interval) } == 'OK'
-      end
-
-      def acquire_full_lock?
-        redis { |c| c.call('SET', @full_lock_key, '1', 'NX', 'EX', @full_interval) } == 'OK'
+        ttl = (@interval * LOCK_TTL_RATIO * 1000).ceil
+        redis do |c|
+          c.call('SET', @lock_key, '1', 'NX', 'PX', ttl)
+        rescue RedisClient::OutOfMemoryError => e
+          OomDrain.lock(c, e, @lock_key, ttl)
+        end == 'OK'
       end
 
       def spawn_loop_thread
@@ -377,6 +387,7 @@ module Wurk
       end
 
       def run_loop
+        @next_sweep_at = monotonic + @interval
         until done?
           wait_next
           break if done?
@@ -385,14 +396,45 @@ module Wurk
         end
       end
 
+      # Each step guarded on its own: a raise in the sweep (lock contention on a
+      # blip, a poison callback) must not cost this process its own reconcile.
+      # The wait is cut short for a grace running out (#wait_next), and that
+      # wake-up is not a sweep's turn.
       def tick_once
+        guarded { reap_when_due }
+        guarded { drain_settled }
+        guarded { @self_reconcile.sweep }
+      end
+
+      def reap_when_due
+        now = monotonic
+        return if now < @next_sweep_at
+
+        @next_sweep_at = now + @interval
         reap
+      end
+
+      def guarded
+        yield
       rescue StandardError => e
-        handle_exception(e, context: THREAD_NAME) if @config.respond_to?(:handle_exception)
+        report(e)
+      end
+
+      # The reporter is host code and may raise too (a closed log stream); that
+      # must not end the only thread that recovers orphans.
+      def report(error)
+        handle_exception(error, context: THREAD_NAME) if @config.respond_to?(:handle_exception)
+      rescue StandardError => e
+        warn "wurk-reaper: error reporter raised #{e.class}: #{e.message}"
       end
 
       def wait_next
-        @mutex.synchronize { @sleeper.wait(@mutex, @interval) unless @done }
+        wait = [@next_sweep_at - monotonic, @grace.next_due_in].compact.min
+        @mutex.synchronize { @sleeper.wait(@mutex, [wait, 0].max) unless @done }
+      end
+
+      def monotonic
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
 
       def done?