wurk 1.3.0 → 1.4.0

data/lib/wurk/heartbeat.rb

@@ -11,14 +11,17 @@ module Wurk
   # Wurk::Launcher so the launcher can stay focused on lifecycle and so the
   # heartbeat schema lives in one place readers can grep for.
   #
-  # Each beat is one pipelined round-trip:
+  # Each beat is two pipelined round-trips. The identity write:
   #   SADD   processes <identity>
   #   HSET   <identity>          info concurrency busy beat quiet rss rtt_us
   #   EXPIRE <identity>          60
   #   UNLINK <identity>:work
   #   HSET   <identity>:work     <tid> <json> ...    (only if WORK_STATE non-empty)
   #   EXPIRE <identity>:work     60                  (only if WORK_STATE non-empty)
+  # then the signal drain:
   #   LPOP   <identity>-signals  × BEAT_PAUSE
+  # Two rather than one because only the first is safe to replay — see
+  # #pipelined_beat and #drain_signals.
   #
   # The work hash is UNLINK-then-rewritten on every beat — a dropped beat
   # momentarily empties it, and ProcessSet#cleanup compensates by SREM-ing
@@ -90,16 +93,17 @@ module Wurk
 
     private
 
-    # Two extra writes (HSET + EXPIRE for the work mirror) when WORK_STATE
-    # has entries, so `lead` shifts to skip them when slicing signals out
-    # of the pipeline result.
+    # Every command in #write_beat converges on the state this snapshot
+    # describes however many times it lands, so the beat claims apply-safety and
+    # rides out a blip the way it did before the pool started splitting replay
+    # by it. `rtt_us` measures this write alone — the signal drain that follows
+    # is a separate checkout and deliberately not part of the gauge.
     def pipelined_beat
       work_snapshot = Processor::WORK_STATE.dup
-      lead = 4 + (work_snapshot.empty? ? 0 : 2)
       t0 = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
-      results = redis { |conn| conn.pipelined { |pipe| write_beat(pipe, work_snapshot) } }
+      redis(idempotent: true) { |conn| conn.pipelined { |pipe| write_beat(pipe, work_snapshot) } }
       rtt = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond) - t0
-      [results[lead, BEAT_PAUSE] || [], rtt]
+      [drain_signals, rtt]
     end
 
     def write_beat(pipe, work_snapshot)
@@ -107,7 +111,6 @@ module Wurk
       pipe.call('HSET', @identity, *beat_hash_args(work_snapshot.size))
       pipe.call('EXPIRE', @identity, TTL_SECONDS)
       write_work_hash(pipe, work_snapshot)
-      drain_signals(pipe)
     end
 
     def beat_hash_args(busy)
@@ -132,10 +135,19 @@ module Wurk
       pipe.call('EXPIRE', work_key, TTL_SECONDS)
     end
 
+    # Its own checkout, and never an idempotent one: LPOP is destructive and
+    # carries its result in the reply, so a replay after a lost reply discards
+    # whatever the first attempt already popped — and a discarded entry is a
+    # dashboard TERM or TSTP this process never acts on. Fused into the beat
+    # pipeline it would have dragged those writes down to the same no-replay
+    # default, or worse, invited a later sweep to claim the LPOPs alongside
+    # them. Runs after the beat so a signal only leaves Redis once the write
+    # that reports us alive has landed.
+    #
     # LPOP one entry per second of cadence so a flood of queued signals
     # can't stall the beat; anything older drains on the next beat.
-    def drain_signals(pipe)
-      BEAT_PAUSE.times { pipe.call('LPOP', "#{@identity}-signals") }
+    def drain_signals
+      redis { |conn| conn.pipelined { |pipe| BEAT_PAUSE.times { pipe.call('LPOP', "#{@identity}-signals") } } }
     end
 
     def info_hash

data/lib/wurk/history.rb

@@ -71,11 +71,23 @@ module Wurk
     end
 
     def start
-      @thread ||= safe_thread('history-snapshot') { @timer.run { tick } } # rubocop:disable Naming/MemoizedInstanceVariableName
+      return @thread if @thread
+
+      @timer.reset
+      @thread = safe_thread('history-snapshot') { @timer.run { tick } }
     end
 
+    # Blocks until the thread is really gone: the launcher releases the cluster
+    # lock immediately after this returns, and a snapshot still in flight would
+    # race the next leader's first one.
+    #
+    # Cleared only on a confirmed join (Thread#join returns nil on timeout): a
+    # wedged thread must stay tracked so #start's guard returns it instead of
+    # calling @timer.reset, which would un-terminate the loop it is still
+    # inside and leave two threads writing the same stream.
     def terminate
       @timer.terminate
+      @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
     end
 
     # Leader-gated: only the elected leader emits, so N workers don't each

data/lib/wurk/launcher.rb

@@ -13,6 +13,7 @@ require_relative 'metrics/rollup'
 require_relative 'metrics/queue_rollup'
 require_relative 'history'
 require_relative 'fetcher/reaper'
+require_relative 'timer_loop'
 
 module Wurk
   # Top-level supervisor inside each worker process. Owns the Manager pool
@@ -43,18 +44,23 @@ module Wurk
     # (Sidekiq's drop-in surface). The single source of truth is Heartbeat.
     BEAT_PAUSE = Heartbeat::BEAT_PAUSE
 
+    # Bound on how long #stop waits for the boot-time reclaim sweep before
+    # moving on — it can still be scanning a large keyspace when a fast
+    # shutdown lands right after boot; teardown must not hang on it.
+    BOOT_RECLAIM_JOIN_TIMEOUT = 5
+
     attr_accessor :managers, :poller, :cron_poller, :metrics_rollup, :queue_rollup, :history
 
     def initialize(config, embedded: false)
       @config = config
       @embedded = embedded
-      # Two separate flags, deliberately. @done = "quieted" (stop fetching, stay
-      # alive, report quiet=true). @stopped = "shutting down" (terminate the
-      # heartbeat loop). Quiet must NOT stop the heartbeat — otherwise a quieted
-      # process never publishes quiet=true and expires out of the live set (#236).
+      # @done is "quieted": stop fetching, stay alive, report quiet=true. It
+      # deliberately does NOT stop the heartbeat — a quieted process that stopped
+      # beating would never publish quiet=true and would expire out of the live
+      # set (#236). Only #stop ends the beat, by terminating @beat_timer.
       @done = false
-      @stopped = false
-      @managers = config.capsules.values.map { |cap| Manager.new(cap) }
+      @beat_timer = TimerLoop.new(BEAT_PAUSE)
+      @managers = build_managers
       @poller = build_poller
       @cron_poller = build_cron_poller
       @metrics_rollup = build_metrics_rollup
@@ -87,18 +93,21 @@ module Wurk
       @config.capsules.each_value(&:prepare!)
       @config.freeze!
       @heartbeat_thread = safe_thread('heartbeat', &method(:start_heartbeat)) if async_beat
-      @poller&.start
-      @leader&.start
-      @cron_poller&.start
-      @metrics_rollup&.start
-      @queue_rollup&.start
-      @history&.start
+      [@poller, @leader, @cron_poller, @metrics_rollup, @queue_rollup, @history].compact.each(&:start)
       @managers.each(&:start)
       @reaper.start
       # Run on a background thread so /ready probe isn't delayed by a large
       # orphan sweep (reaper.reclaim! is atomic, but can scan many entries).
       @boot_reclaim_thread = safe_thread('boot-reclaim', &method(:boot_reclaim))
       @health_server&.start
+    rescue StandardError
+      # Boot is not atomic: whatever raised (a health-check port already bound,
+      # ThreadError at the OS thread limit) leaves the steps before it holding
+      # threads, sockets and a leader campaign — and the caller is about to drop
+      # its only reference to us, so nothing else can ever release them. Guarded,
+      # because the caller must see the boot failure, not a rollback failure.
+      teardown_step('boot-rollback') { stop }
+      raise
     end
 
     # Idempotent. Flips `stopping?` true, halts fetching across every
@@ -118,26 +127,18 @@ module Wurk
 
     # Graceful shutdown. Deadline is monotonic so wall-clock skew can't
     # extend it. Managers stop in parallel threads so a slow capsule
-    # doesn't block its siblings.
+    # doesn't block its siblings — and each drain is guarded, because a
+    # capsule that blows up mid-drain (Redis down during bulk_requeue) used
+    # to surface out of `join` and skip the whole teardown tail, leaving the
+    # leader lock held, the process listed as live, and its port open.
     def stop
       deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + (@config[:timeout] || 25)
       quiet
-      stoppers = @managers.map { |m| Thread.new { m.stop(deadline) } }
+      stoppers = @managers.map { |m| Thread.new { teardown_step('manager') { m.stop(deadline) } } }
       fire_event(:shutdown, reverse: true)
       stoppers.each(&:join)
-      # Full shutdown stops periodic firing (it survived #quiet); do this before
-      # releasing the lock so no tick races a follower's promotion.
-      @cron_poller&.terminate
-      @metrics_rollup&.terminate
-      @queue_rollup&.terminate
-      @history&.terminate
-      @reaper&.stop
-      # CAS-release the cluster lock now (planned shutdown) so a follower can
-      # take over immediately instead of waiting out the TTL.
-      @leader&.stop
-      stop_heartbeat
-      clear_heartbeat
-      fire_event(:exit, reverse: true)
+    ensure
+      release_components
     end
 
     def stopping?
@@ -162,9 +163,12 @@ module Wurk
 
       write_stats(processed, failed, expired)
     rescue StandardError => e
-      # Replay-safety: counters were reset above, so a Redis blip would
-      # otherwise drop stats. We log and accept — the per-job at-least-once
-      # semantics don't apply to *counters*, and the next beat resets again.
+      # The counters were reset above, so a raise here drops this window's stats
+      # for good: #write_stats cannot claim apply-safety and the pool no longer
+      # replays it through a blip. Accepted deliberately — adding them back
+      # would double-count the case where the INCRBYs did land and only the
+      # reply was lost, and the per-job at-least-once semantics don't apply to
+      # *counters*. The next beat resets again.
       handle_exception(e, { context: 'flush_stats' })
     end
 
@@ -174,6 +178,60 @@ module Wurk
 
     private
 
+    # Teardown tail, driven from #stop's ensure. Every release is guarded on
+    # its own because they are independent: a Redis blip in one (leader CAS
+    # release, heartbeat clear) must not strand the ones after it — the process
+    # is exiting either way, so the only thing worse than a failed release is a
+    # skipped one. Order matters twice over: full shutdown stops the periodic
+    # loops (they survived #quiet) before the cluster lock is CAS-released, so
+    # no tick races a follower's promotion, and the release itself happens on a
+    # planned shutdown rather than waiting out the lock TTL.
+    def release_components
+      # Joined first, before anything below tears down the Redis pool it's
+      # still reading from: it started as a fire-and-forget thread in #run
+      # and a shutdown landing right after boot could otherwise race a
+      # pool disconnect mid-scan. Bounded: a full-keyspace scan can outlast
+      # the timeout, in which case the thread is left running rather than
+      # blocking shutdown on it — #boot_reclaim already rescues and logs on
+      # its own, so a straggler racing #reset_redis_pools! below just means
+      # a noisy log line, not a hang.
+      teardown_step('boot-reclaim-join') { @boot_reclaim_thread&.join(BOOT_RECLAIM_JOIN_TIMEOUT) }
+      stop_periodic_components
+      %i[stop_heartbeat clear_heartbeat].each { |step| teardown_step(step) { send(step) } }
+      # Unguarded: fire_event already reports and skips past a raising hook.
+      fire_event(:exit, reverse: true)
+      # Embedded only: a swarm child or standalone process exits right after
+      # #stop anyway, so disconnecting here would just make every unit test
+      # that inspects Redis post-stop rebuild a pool for no reason. Embedded
+      # hosts (Puma, a rake task) keep running and can `run` again later —
+      # without this a stop-then-run cycle doubles the live socket set.
+      teardown_step('redis-pools') { @config.reset_redis_pools! } if @embedded
+    ensure
+      # In an ensure of its own, not merely a guarded step: a leaked TCPServer
+      # FD survives even a non-StandardError unwind (a second TERM landing
+      # mid-teardown), and kubelet would keep getting 200s from a process that
+      # is already gone.
+      teardown_step('health-server') { @health_server&.stop }
+    end
+
+    # Split out of #release_components to keep it under the AbcSize/
+    # CyclomaticComplexity ceilings — these three are the "periodic loop"
+    # releases, independently guarded like everything else in the tail.
+    def stop_periodic_components
+      [@cron_poller, @metrics_rollup, @queue_rollup, @history].each { |t| teardown_step(t.class) { t&.terminate } }
+      teardown_step('reaper') { @reaper&.stop }
+      teardown_step('leader') { @leader&.stop }
+    end
+
+    def teardown_step(label)
+      yield
+    rescue StandardError => e
+      handle_exception(e, { context: "launcher-stop-#{label}" })
+    end
+
+    # INCRBY is additive, so a pipeline replayed after a lost reply counts this
+    # window twice on every `stat:` key it touches — no apply-safety claim. (The
+    # EXPIREs riding along are harmless to repeat; the INCRBYs pin the block.)
     def write_stats(processed, failed, expired)
       day = Time.now.utc.strftime('%F')
       @config.redis do |conn|
@@ -217,74 +275,83 @@ module Wurk
 
     # Erase the live-process footprint. flush_stats first so we don't drop
     # the final batch of counters; then Heartbeat#stop! removes us from the
-    # `processes` SET and UNLINK-s the identity + work hashes. The probe
-    # server is closed alongside so kubelet stops getting 200s after the
-    # process is no longer healthy.
+    # `processes` SET and UNLINK-s the identity + work hashes.
     def clear_heartbeat
       flush_stats
       @heartbeat&.stop!
-      @health_server&.stop
     end
 
     # Terminate the heartbeat loop and wait for it to exit before clear_heartbeat
     # removes us from the `processes` SET — otherwise a final in-flight beat could
-    # SADD us back right after the SREM. Wakes the thread out of its BEAT_PAUSE
-    # sleep so shutdown isn't delayed up to a full interval.
+    # SADD us back right after the SREM and the identity would linger for a full
+    # 60s TTL.
+    #
+    # The join is unbounded on purpose. This used to be `wakeup` + `join(BEAT_PAUSE)`,
+    # which lost the race whenever the beat was mid-Redis-call: `wakeup` does nothing
+    # to a thread that isn't sleeping, so the loop then slept a full interval and the
+    # bounded join returned with the thread still live — exactly the resurrect-after-
+    # SREM this ordering exists to prevent. A condvar can't be missed (terminate flips
+    # the flag inside the critical section the loop re-checks it in), so all that is
+    # left to wait out is one in-flight beat, whose Redis calls are timeout-bounded.
     def stop_heartbeat
-      @stopped = true
+      @beat_timer.terminate
       thread = @heartbeat_thread
       return unless thread
-      # Embedded dashboard-TERM runs `stop` from the beat itself; a self-join
-      # raises ThreadError. @stopped is set, so the loop exits after this beat.
+      # Embedded dashboard-TERM can run `stop` from the beat itself; a self-join
+      # raises ThreadError. The timer is terminated, so the loop exits after this beat.
       return if thread == Thread.current
 
-      begin
-        thread.wakeup
-      rescue ThreadError
-        nil
-      end
-      thread.join(BEAT_PAUSE)
+      thread.join
     end
 
-    # Heartbeat thread loop. `safe_thread` already wraps exceptions. Loops on
-    # `@stopped` — NOT `@done` — so a *quieted* process keeps beating and publishes
-    # `quiet=true` instead of vanishing from the live set (#236). Only `#stop`
-    # flips `@stopped`; its `Thread#wakeup` breaks the sleep so the loop re-checks
-    # `@stopped` and exits without waiting out the interval.
+    # Heartbeat thread loop. `safe_thread` already wraps exceptions. Beats once up
+    # front — TimerLoop#run waits before its first yield, and the dashboard has to
+    # see this process the moment it can pick up jobs — then ticks until #stop
+    # terminates the timer. Note it does NOT stop on `@done`: a *quieted* process
+    # keeps beating so it publishes `quiet=true` instead of vanishing (#236).
     def start_heartbeat
-      until @stopped
-        heartbeat
-        sleep BEAT_PAUSE
-      end
+      heartbeat
+      @beat_timer.run { heartbeat }
       logger.info('Heartbeat stopping...')
     end
 
     # Dashboard-queued signals must behave exactly like OS signals, so a
-    # standalone process re-delivers to itself and lets the installed trap
-    # run — that wakes the main thread (CLI self-pipe / child dispatcher)
-    # so the process actually exits instead of stopping its managers and
-    # then parking forever. Embedded mode owns no traps (and self-TERM
-    # would kill the host app), so it calls quiet/stop directly — stop on
-    # its own thread because `stop` joins the heartbeat thread we're on.
+    # standalone process re-delivers to itself and lets the installed trap run —
+    # that wakes the main thread (CLI self-pipe / child dispatcher) so the
+    # process actually exits instead of stopping its managers and then parking
+    # forever. Embedded mode owns no traps (and self-TERM would kill the host
+    # app), so quiet applies directly.
     def dispatch_signal(sig)
       case sig
-      when 'TSTP', 'TERM'
-        if @embedded
-          sig == 'TSTP' ? quiet : Thread.new { stop }
-        else
-          redeliver(sig)
-        end
+      when 'TSTP' then @embedded ? quiet : redeliver(sig)
+      when 'TERM' then request_shutdown
       else
         logger.warn { "Unknown signal in #{identity}-signals: #{sig.inspect}" }
       end
     end
 
+    # The one way anything inside this process asks it to shut down gracefully:
+    # dashboard-queued TERM, or a Manager that can no longer hold its
+    # concurrency. Standalone hands off to the installed TERM trap; embedded
+    # owns no traps, so it drains in place — on its own thread, because callers
+    # may be a thread `stop` itself joins (the heartbeat) or kills (a Processor).
+    def request_shutdown
+      @embedded ? Thread.new { stop } : redeliver('TERM')
+    end
+
     # Separate method so tests can stub it — really sending TERM/TSTP would
     # kill or suspend the test process.
     def redeliver(sig)
       ::Process.kill(sig, ::Process.pid)
     end
 
+    # One Manager per capsule, each holding our shutdown request: a Manager
+    # that can no longer replace a dead Processor has to take the process
+    # down, and only the Launcher knows how this process exits.
+    def build_managers
+      @config.capsules.values.map { |cap| Manager.new(cap, shutdown: method(:request_shutdown)) }
+    end
+
     def build_poller
       Wurk::Scheduled::Poller.new(@config)
     end

data/lib/wurk/leader.rb

@@ -3,6 +3,8 @@
 require 'securerandom'
 require 'socket'
 require_relative 'component'
+require_relative 'lua'
+require_relative 'pool_checkout'
 
 module Wurk
   # Cluster leader election via Redis `SET NX EX`. Single-leader-per-cluster
@@ -89,9 +91,19 @@ module Wurk
 
     # CAS DEL — only drop the key if we still own it, otherwise a stale
     # release would yank leadership from whichever follower took over.
+    # GET-then-DEL is not that CAS: our key can lapse between the two
+    # commands and a follower can win the election in the gap, whereupon
+    # the bare DEL deletes *its* lock and leaves the cluster leaderless
+    # (or, once the ex-leader re-campaigns, doubly led) for up to one
+    # renew interval — double cron fires, double rollups. The compare and
+    # the delete happen in one server-side step instead, via the script
+    # `Unique.release_if_owner` already shares.
+    #
+    # `idempotent:` — a replay after a lost reply finds the key either
+    # already gone or no longer ours, so it can only be a no-op.
     def release
-      redis_call do |c|
-        c.call('DEL', @key) if c.call('GET', @key) == @owner
+      redis_call(idempotent: true) do |c|
+        Wurk::Lua::Loader.eval_cached(c, :release_if_owner, keys: [@key], argv: [@owner])
       end
       @held = false
       @token = nil
@@ -107,6 +119,11 @@ module Wurk
     # follower, it polls every `follower_interval`. Caller must invoke
     # `stop` for orderly shutdown — the thread also releases its lock on
     # exit.
+    #
+    # The spawn happens *inside* the lock: with it outside, two callers
+    # racing into `start` both clear the nil check and both spawn, and the
+    # loser's loop is never recorded — an unjoinable second campaigner that
+    # outlives `stop` and keeps renewing the cluster lock.
     def start
       return nil if disabled?
 
@@ -114,17 +131,19 @@ module Wurk
         return @thread if @thread
 
         @done = false
+        @thread = spawn_loop_thread
       end
-      @thread = spawn_loop_thread
+      @thread
     end
 
     def stop
-      @mutex.synchronize do
+      thread = @mutex.synchronize do
         @done = true
         @sleeper.signal
+        @thread
       end
-      @thread&.join
-      @thread = nil
+      thread&.join
+      @mutex.synchronize { @thread = nil }
       release
     end
 
@@ -175,13 +194,13 @@ module Wurk
       @config.handle_exception(e, event: :leader) if @config.respond_to?(:handle_exception)
     end
 
-    def redis_call(&)
+    def redis_call(idempotent: false, &)
       if @pool
-        @pool.with(&)
+        PoolCheckout.with(@pool, idempotent, &)
       elsif @config
-        @config.redis(&)
+        @config.redis(idempotent:, &)
       else
-        Wurk.redis(&)
+        Wurk.redis(idempotent:, &)
       end
     end
 

data/lib/wurk/limiter/base.rb

@@ -104,17 +104,15 @@ module Wurk
         end
       end
 
+      # Metadata HSET + EXPIRE + `lmtr-list` SADD in a single round trip. The
+      # script also gates the SADD on the metadata being newly written, so a
+      # per-job construction (`stripe-#{user_id}`) stops rewriting a set that
+      # grows one member per name — see limiter_register.lua for why that gate
+      # is only safe while it is atomic with the HSET.
       def register!
-        Wurk::Limiter.redis do |c|
-          c.call('SADD', LIST_KEY, @name)
-          c.call(
-            'HSET', meta_key,
-            'type', type.to_s,
-            'options', JSON.dump(serializable_options),
-            'fingerprint', fingerprint
-          )
-          c.call('EXPIRE', meta_key, @options[:ttl])
-        end
+        lua(:limiter_register,
+            keys: [meta_key, LIST_KEY],
+            argv: [@name, type.to_s, JSON.dump(serializable_options), fingerprint, ttl])
       end
 
       def ttl

data/lib/wurk/limiter/bucket.rb

@@ -38,7 +38,7 @@ module Wurk
           remaining = deadline - ::Time.now.to_f
           raise OverLimit, self if remaining <= 0
 
-          sleep [remaining, secs_to_next.to_f, 0.05].compact.min.clamp(0.0, remaining)
+          sleep [0.05, secs_to_next.to_f].max.clamp(0.0, remaining)
         end
       end
 

data/lib/wurk/limiter/concurrent.rb

@@ -6,7 +6,7 @@ module Wurk
   module Limiter
     # Atomic slot acquisition in a ZSET. Score = expiry epoch; the acquire
     # script first evicts expired slots (bumping the `reclaimed` metric)
-    # then ZADDs if there's headroom.
+    # then ZADDs if there's headroom (bumping the `held` metric).
     #
     # On exhaustion: spin loop with backoff. The spec says "blocks via
     # Redis stream XREAD" — that's a perf optimization; the visible
@@ -38,33 +38,15 @@ module Wurk
         raise ArgumentError, 'block required' unless block
 
         started = monotime
-        deadline = started + @options[:wait_timeout]
         slot = random_id
-        acquired_at = nil
-        loop do
-          result = acquire(slot)
-          if result[0].to_i == 1
-            acquired_at = monotime
-            break
-          end
-
-          return if @options[:policy] == :ignore
-
-          remaining = deadline - monotime
-          if remaining <= 0
-            bump_counter('overages')
-            raise OverLimit, self
-          end
-
-          sleep [remaining, WAIT_SLEEP].min
-        end
+        acquired_at = wait_for_slot(slot, started + @options[:wait_timeout])
+        return unless acquired_at
 
         begin
           incr_immediate_or_waited(acquired_at - started)
           block.call
         ensure
-          release(slot)
-          bump_counter('held_time', (monotime - acquired_at).to_i) if acquired_at
+          record_release(slot, acquired_at)
         end
       end
 
@@ -98,6 +80,29 @@ module Wurk
         "lmtr-stats:#{@name}"
       end
 
+      # A ZREM that removes nothing means an acquirer already evicted our slot:
+      # we outran `lock_timeout`, which is what `overages` counts. Exhausting
+      # `wait_timeout` is a different event — it raises OverLimit and the server
+      # middleware counts it in the job's `overrated` field.
+      def record_release(slot, acquired_at)
+        bump_counter('overages') if release(slot).to_i.zero?
+        bump_counter('held_time', (monotime - acquired_at).to_i)
+      end
+
+      # Spins until a slot frees up. Returns the monotonic acquire time, nil
+      # when `policy: :ignore` gives up; raises OverLimit past `wait_timeout`.
+      def wait_for_slot(slot, deadline)
+        loop do
+          return monotime if acquire(slot)[0].to_i == 1
+          return if @options[:policy] == :ignore
+
+          remaining = deadline - monotime
+          raise OverLimit, self if remaining <= 0
+
+          sleep [remaining, WAIT_SLEEP].min
+        end
+      end
+
       def acquire(slot)
         lua(:limiter_concurrent_acquire,
             keys: [state_key, stats_key],

data/lib/wurk/limiter/window.rb

@@ -19,18 +19,15 @@ module Wurk
       end
 
       def size
-        cutoff = ::Time.now.to_f - interval_seconds
-        Wurk::Limiter.redis do |c|
-          c.call('ZREMRANGEBYSCORE', state_key, '-inf', "(#{cutoff}")
-          c.call('ZCARD', state_key).to_i
-        end
+        window_state.first
       end
 
       # used = entries still inside the window; limit = count; reset_at =
       # when the oldest entry slides out (freeing a slot), or nil when
       # idle (#16).
       def status
-        build_status(used: size, limit: @options[:count], reset_at: oldest_expiry)
+        used, oldest = window_state
+        build_status(used: used, limit: @options[:count], reset_at: oldest && (oldest + interval_seconds))
       end
 
       def within_limit(used: 1, &block)
@@ -60,11 +57,16 @@ module Wurk
         "lmtr-w:#{@name}"
       end
 
-      # Oldest timestamp + interval = the moment it leaves the window.
-      def oldest_expiry
-        row = Wurk::Limiter.redis { |c| c.call('ZRANGE', state_key, 0, 0, 'WITHSCORES') }
-        score = Wurk::Limiter.first_score(row)
-        score && (score + interval_seconds)
+      # Count + oldest in-window score, both scoped by the Redis clock, in one
+      # read-only round trip. Reads never trim (the acquire script owns that):
+      # a dashboard host whose clock ran ahead used to evict live entries just
+      # by rendering `status`, freeing the window for a second full charge.
+      # Oldest timestamp + interval = the moment it leaves the window; -1 =
+      # nothing in window.
+      def window_state
+        count, oldest = lua(:limiter_window_status, keys: [state_key], argv: [interval_seconds])
+        score = oldest.to_f
+        [count.to_i, score.negative? ? nil : score]
       end
 
       def interval_seconds