wurk 1.1.0 → 1.1.2

data/lib/wurk/swarm.rb

@@ -5,11 +5,20 @@ require_relative 'launcher'
 require_relative 'fetcher/reliable'
 require_relative 'keys'
 require_relative 'swarm/child_boot'
+require_relative 'swarm/backoff'
+require_relative 'swarm/restart'
+require_relative 'swarm/orphan_guard'
 
 module Wurk
   # Parent supervisor. Forks N children per the worker topology, monitors
-  # PIDs, relays signals, respawns crashed children, handles rolling
-  # restart on SIGUSR1, recycles RSS-bloated children.
+  # PIDs, relays signals, respawns crashed children with per-slot exponential
+  # backoff, handles rolling restart on SIGUSR1, recycles RSS-bloated children.
+  #
+  # The supervise loop never sleeps on behalf of a respawn or a restart: crash
+  # backoff is tracked as per-slot due-times (Swarm::Backoff) and rolling
+  # restart / recycle run as a non-blocking state machine (Swarm::Restart)
+  # advanced one phase per tick. TERM/INT is therefore honored within a tick
+  # regardless of restart or backoff state.
   #
   # Boot ordering (must be exact — see docs/idea/03-process-model.md):
   #   1. Host app boots fully; eager loads done.
@@ -21,11 +30,10 @@ module Wurk
   #   6. Parent calls `supervise` to enter the wait/relay loop.
   #
   # Signals (see docs/idea/04-signals.md):
-  #   TERM/INT  → `shutdown`           (graceful drain)
-  #   TSTP      → relay TSTP           (pause fetch)
-  #   CONT      → relay CONT           (resume fetch)
+  #   TERM/INT  → `shutdown`           (graceful drain; aborts any restart)
+  #   TSTP      → relay TSTP           (quiet — stop fetching; one-way, no resume)
   #   USR1      → `rolling_restart`    (zero-downtime cycle)
-  class Swarm
+  class Swarm # rubocop:disable Metrics/ClassLength
     include Component
 
     SUPERVISE_TICK = 0.2
@@ -34,6 +42,16 @@ module Wurk
     MEMORY_CHECK_INTERVAL = 10
     DEFAULT_SHUTDOWN_TIMEOUT = 25
 
+    # Children each hard_shutdown after their own drain deadline (bulk_requeue
+    # + a 3s ensure window + heartbeat cleanup); the parent must not SIGKILL
+    # them mid-tail, so its own wait always extends past theirs by this much.
+    SHUTDOWN_GRACE = 5
+
+    # USR2 is relayed (log reopen) — without a trap, a logrotate config that
+    # signals the master pid would hit USR2's default disposition and kill the
+    # whole swarm.
+    SWARM_SIGNALS = { 'TERM' => :term, 'INT' => :term, 'TSTP' => :tstp, 'USR1' => :usr1, 'USR2' => :usr2 }.freeze
+
     attr_reader :topology, :children
 
     def initialize(topology:, config: Wurk.configuration, memory_limit: config.memory_limit_kb,
@@ -45,28 +63,40 @@ module Wurk
       @children = {}
       @assignments = []
       @stopping = false
+      @quieted = false
       @last_memory_check = 0
-      @signal_queue = ::Thread::Queue.new
+      @signal_read = nil
+      @signal_write = nil
+      @respawn_backoff = Backoff.new(base: RESPAWN_BACKOFF)
+      @restart = build_restart
     end
 
     # `install_signals:` is false in tests so the integration suite can
     # drive `shutdown` / `rolling_restart` directly without poisoning the
     # test process's signal handlers.
+    #
+    # Traps go in BEFORE fork_children: a TERM landing in the (previously
+    # post-fork) window between fork and trap installation left the parent on
+    # its default disposition — it died instantly and orphaned live, fetching
+    # children. Installed first, the trap queues the TERM and the supervise
+    # loop drains it (relaying to children) even if it arrives mid-boot.
     def boot(install_signals: true)
       raise 'Wurk::Swarm already booted' unless @assignments.empty?
       raise ArgumentError, 'Topology has no slots' if @topology.empty?
 
       @assignments = @topology.assignments.freeze
+      install_signal_handlers if install_signals
       close_parent_sockets
       fork_children
-      install_signal_handlers if install_signals
       @children.keys
     end
 
     def supervise
       until done?
         drain_signals
-        reap_one_child
+        reap_children
+        spawn_due_respawns
+        @restart.advance unless @stopping
         check_memory_pressure
         sleep SUPERVISE_TICK
       end
@@ -74,32 +104,45 @@ module Wurk
 
     def shutdown(timeout: @shutdown_timeout)
       @stopping = true
+      @restart.abort
       relay_signal('TERM')
-      wait_for_children(timeout)
+      wait_for_children(timeout + SHUTDOWN_GRACE)
       hard_kill_stragglers
     end
 
-    # SIGUSR1. For each existing child, fork a replacement, wait for its
-    # first heartbeat, then TERM + drain the old one. Long-running jobs
-    # in the old slot get the full shutdown_timeout while the replacement
-    # is already serving new work.
+    # TSTP quiet is one-way and GLOBAL (spec §21.3): it must survive respawns
+    # and memory recycles, or a quieted-but-crashed child's replacement would
+    # resume fetching mid-maintenance. The flag makes every future fork boot
+    # already-quieted (see ChildBoot start_quiet).
+    def quiet_swarm
+      @quieted = true
+      relay_signal('TSTP')
+    end
+
+    # SIGUSR1: queue every live child for the rolling-restart state machine,
+    # which replaces one slot at a time (spawn replacement → await its
+    # heartbeat → TERM the old child → await its drain) without blocking the
+    # supervise thread, so TERM stays responsive throughout the cycle.
     def rolling_restart
-      @children.dup.each do |old_pid, meta|
-        replacement = fork_child(meta[:slot], meta[:index])
-        @children[replacement] = meta
-        unless wait_for_heartbeat(replacement)
-          logger.warn do
-            "swarm: replacement #{replacement} heartbeat not seen within #{HEARTBEAT_WAIT}s; proceeding anyway"
-          end
-        end
-        safe_kill(old_pid, 'TERM')
-        wait_pid(old_pid, @shutdown_timeout)
-        @children.delete(old_pid)
-      end
+      @restart.enqueue(@children.keys)
     end
 
     private
 
+    def build_restart
+      Restart.new(Restart::Config.new(
+                    spawn: method(:spawn_child),
+                    kill: method(:safe_kill),
+                    heartbeat: method(:heartbeat_seen?),
+                    describe: ->(pid) { @children[pid] },
+                    now: method(:monotonic),
+                    logger: logger,
+                    heartbeat_wait: HEARTBEAT_WAIT,
+                    drain_timeout: @shutdown_timeout + SHUTDOWN_GRACE,
+                    backoff: Backoff.new(base: RESPAWN_BACKOFF)
+                  ))
+    end
+
     # Step 3.
     def close_parent_sockets
       @config.reset_redis_pools!
@@ -117,82 +160,164 @@ module Wurk
 
     # Step 4.
     def fork_children
-      @assignments.each_with_index do |slot, idx|
-        @children[fork_child(slot, idx)] = { slot: slot, index: idx }
-      end
+      @assignments.each_index { |idx| spawn_child(@assignments[idx], idx) }
     end
 
+    # Fork one child for the slot and record its spawn time so crash backoff can
+    # tell a crash-loop (short-lived) from a healthy child that finally died.
+    # Returns the child PID; never returns in the child (ChildBoot exits).
+    def spawn_child(slot, idx)
+      pid = fork_child(slot, idx)
+      @children[pid] = { slot: slot, index: idx, spawned_at: monotonic }
+      pid
+    end
+
+    # Capture the parent PID BEFORE forking and hand it to the child: read
+    # from the parent, it is race-free even if the parent dies the instant
+    # after fork (getppid in the child could already return the reaper). The
+    # child's OrphanGuard compares live getppid against it.
     def fork_child(slot, idx)
+      parent_pid = ::Process.pid
       pid = ::Process.fork
       return pid if pid
 
-      ChildBoot.new(@config, slot, idx).run
+      # Drop the parent's self-pipe first: the inherited traps still write to
+      # it, so a signal landing in the window before ChildBoot resets them
+      # would surface in the PARENT's supervise loop (an operator TERMing one
+      # child pid would drain the whole swarm). Closed, the trap write no-ops.
+      @signal_read&.close
+      @signal_write&.close
+      ChildBoot.new(@config, slot, idx, parent_pid: parent_pid, start_quiet: @quieted).run
       exit 0 # unreachable; ChildBoot exits explicitly
     end
 
+    # Self-pipe pattern (same as Wurk::CLI): the trap only writes the signal
+    # name to a pipe — no Thread::Queue#push, which takes a mutex a trap can
+    # deadlock against. The supervise loop polls the read end each tick.
     def install_signal_handlers
-      { 'TERM' => :term, 'INT' => :term, 'TSTP' => :tstp,
-        'CONT' => :cont, 'USR1' => :usr1 }.each do |sig, sym|
-        ::Signal.trap(sig) { @signal_queue << sym }
+      @signal_read, @signal_write = ::IO.pipe
+      SWARM_SIGNALS.each_key do |sig|
+        ::Signal.trap(sig) { emit_signal(sig) }
+      rescue ArgumentError
+        # Platform without this signal (e.g. some JRuby builds) — skip it.
+        nil
       end
     end
 
+    # Non-blocking self-pipe write from trap context: a blocking `puts` could
+    # stall signal delivery if the pipe fills. `exception: false` returns
+    # :wait_writable instead of raising when full (drop the coalescible
+    # duplicate); a closed pipe during shutdown is ignored too.
+    def emit_signal(sig)
+      @signal_write.write_nonblock("#{sig}\n", exception: false)
+    rescue ::IOError, ::Errno::EPIPE, ::Errno::EBADF
+      nil
+    end
+
     def drain_signals
-      until @signal_queue.empty?
-        sym = next_signal_symbol
-        next if sym.nil?
+      return unless @signal_read
 
-        case sym
+      while (sig = read_pending_signal)
+        case SWARM_SIGNALS[sig]
         when :term then shutdown
-        when :tstp then relay_signal('TSTP')
-        when :cont then relay_signal('CONT')
+        when :tstp then quiet_swarm
         when :usr1 then rolling_restart
+        when :usr2 then relay_signal('USR2')
         end
       end
     end
 
-    def next_signal_symbol
-      @signal_queue.pop(true)
-    rescue ThreadError
-      nil
+    # One buffered line per pending signal, non-blocking (wait_readable(0)).
+    # nil once the pipe is drained, ending the loop for this tick.
+    def read_pending_signal
+      return nil unless @signal_read.wait_readable(0)
+
+      @signal_read.gets&.strip
     end
 
-    def reap_one_child
-      pid, status = ::Process.wait2(-1, ::Process::WNOHANG)
-      on_child_exit(pid, status) if pid
+    # Reap every exited child this tick (not one), so a fleet-wide death
+    # recovers in parallel rather than one child per SUPERVISE_TICK. ECHILD
+    # (momentarily no children — all crashed and awaiting backoff) is not a stop
+    # condition: the swarm only stops on an explicit TERM/INT.
+    def reap_children
+      loop do
+        pid, status = ::Process.wait2(-1, ::Process::WNOHANG)
+        break unless pid
+
+        on_child_exit(pid, status)
+      end
     rescue Errno::ECHILD
-      @stopping = true
+      nil
     end
 
     def on_child_exit(pid, status)
       meta = @children.delete(pid)
       return unless meta
+      return if @restart.claim_exit(pid)
 
       if @stopping
         logger.info { "swarm: child #{pid} exited (status=#{status.exitstatus})" }
       else
-        logger.warn { "swarm: child #{pid} died (status=#{status.exitstatus}); respawning slot #{meta[:index]}" }
-        sleep RESPAWN_BACKOFF
-        @children[fork_child(meta[:slot], meta[:index])] = meta
+        schedule_respawn(pid, status, meta)
       end
     end
 
+    # Arm the slot's backoff instead of sleeping the supervise thread; the next
+    # due tick respawns it. A child that lived past the reset window counts as a
+    # fresh failure (base delay), so only a genuine crash-loop escalates toward
+    # the cap.
+    def schedule_respawn(pid, status, meta)
+      idx = meta[:index]
+      delay = @respawn_backoff.fail(idx, lifetime: monotonic - meta[:spawned_at])
+      logger.warn do
+        "swarm: child #{pid} died (status=#{status.exitstatus}); respawning slot #{idx} in #{delay}s"
+      end
+    end
+
+    # Respawn any slot whose backoff window has elapsed. Runs every tick; a
+    # no-op until a scheduled respawn comes due.
+    def spawn_due_respawns
+      return if @stopping
+
+      @assignments.each_index do |idx|
+        next unless @respawn_backoff.pending?(idx) && @respawn_backoff.ready?(idx)
+
+        respawn_slot(idx)
+      end
+    end
+
+    # Consume the backoff only after the fork lands. A fork/resource failure
+    # (EAGAIN, ENOMEM) must neither drop the pending respawn nor escape the
+    # supervise loop: on failure we re-arm the backoff and leave the slot
+    # pending so the next due tick retries instead of hot-looping.
+    def respawn_slot(idx)
+      spawn_child(@assignments[idx], idx)
+      @respawn_backoff.consume(idx)
+    rescue StandardError => e
+      delay = @respawn_backoff.fail(idx)
+      logger.warn { "swarm: respawn of slot #{idx} failed (#{e.class}: #{e.message}); retrying in #{delay}s" }
+    end
+
     def check_memory_pressure
       return unless @memory_limit
 
-      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      now = monotonic
       return if now - @last_memory_check < MEMORY_CHECK_INTERVAL
 
       @last_memory_check = now
       @children.dup.each_key { |pid| recycle_if_bloated(pid) }
     end
 
+    # Route a bloated child through the restart state machine (same path as a
+    # rolling restart) so recycle is graceful — a healthy replacement takes over
+    # before the old child is TERMed — and can't overlap a restart already in
+    # flight on the slot.
     def recycle_if_bloated(pid)
       rss = pid_rss_kb(pid)
       return if rss.nil? || rss < @memory_limit
 
       logger.warn { "swarm: child #{pid} RSS #{rss}KB >= #{@memory_limit}KB; recycling" }
-      safe_kill(pid, 'TERM')
+      @restart.enqueue([pid])
     end
 
     def pid_rss_kb(pid)
@@ -213,22 +338,10 @@ module Wurk
       nil
     end
 
-    def wait_pid(pid, timeout)
-      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout
-      while ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) < deadline
-        return true if ::Process.wait(pid, ::Process::WNOHANG)
-
-        sleep 0.1
-      end
-      false
-    rescue Errno::ECHILD
-      true
-    end
-
     def wait_for_children(timeout)
-      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout
-      while ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) < deadline && @children.any?
-        reap_one_child
+      deadline = monotonic + timeout
+      while monotonic < deadline && @children.any?
+        reap_children
         sleep 0.1
       end
     end
@@ -238,21 +351,24 @@ module Wurk
       @children.clear
     end
 
-    # Identity is `<hostname>:<pid>:<nonce>`. PROCESS_NONCE is set when
-    # Component loads in the parent and inherited by every fork — the
-    # parent can compute a child's identity from its PID alone.
-    # Returns true if the heartbeat was observed before the deadline.
-    def wait_for_heartbeat(pid) # rubocop:disable Naming/PredicateMethod
+    # Has the child written its first heartbeat yet? One non-blocking SISMEMBER,
+    # polled by the restart state machine each tick. Identity is
+    # `<hostname>:<pid>:<nonce>`; PROCESS_NONCE is set when Component loads in
+    # the parent and inherited by every fork, so the parent computes a child's
+    # identity from its PID alone. A Redis blip returns false (not seen yet) so a
+    # transient error can't crash the supervisor — the restart deadline still
+    # forces progress.
+    def heartbeat_seen?(pid)
       identity = "#{hostname}:#{pid}:#{Component::PROCESS_NONCE}"
-      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + HEARTBEAT_WAIT
-      while ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) < deadline
-        return true if @config.redis { |c| c.call('SISMEMBER', Keys::PROCESSES, identity) } == 1
-
-        sleep 0.5
-      end
+      @config.redis { |c| c.call('SISMEMBER', Keys::PROCESSES, identity) } == 1
+    rescue StandardError
       false
     end
 
+    def monotonic
+      ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+    end
+
     def done?
       @stopping && @children.empty?
     end

data/lib/wurk/timer_loop.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Wurk
+  # Mutex/CV "tick every N seconds until told to stop" primitive shared by
+  # every leader-gated periodic component (History, Metrics::Rollup,
+  # Metrics::QueueRollup). Each of those used to hand-roll the same
+  # `@mutex`/`@sleeper`/`@done` dance; this is that dance, extracted once.
+  #
+  # The host owns thread spawning (it needs its own `safe_thread` from
+  # Component for logger/handle_exception context) and the leader-gate check
+  # inside its `tick` — TimerLoop only owns the interval wait and the
+  # start/terminate signaling around it:
+  #
+  #   @timer = Wurk::TimerLoop.new(interval)
+  #   @thread ||= safe_thread('my-loop') { @timer.run { tick } }
+  #   def terminate = @timer.terminate
+  class TimerLoop
+    def initialize(interval)
+      @interval = interval
+      @done = false
+      @mutex = ::Mutex.new
+      @sleeper = ::ConditionVariable.new
+    end
+
+    # Waits one interval, then yields repeatedly (waiting between calls)
+    # until #terminate is called. Matches the existing components' "don't
+    # tick immediately on boot" behavior.
+    def run
+      wait
+      until @done
+        yield
+        wait
+      end
+    end
+
+    def terminate
+      @mutex.synchronize do
+        @done = true
+        @sleeper.signal
+      end
+    end
+
+    def wait
+      @mutex.synchronize do
+        @sleeper.wait(@mutex, @interval) unless @done
+      end
+    end
+  end
+end

data/lib/wurk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wurk
-  VERSION = "1.1.0"
+  VERSION = "1.1.2"
 end

data/lib/wurk/web/extension.rb

@@ -271,7 +271,10 @@ module Wurk
             end
           rescue ::StandardError => e
             ::Wurk.configuration.handle_exception(e, context: 'web-extension-render')
-            [500, html_headers, "Extension render error: #{::CGI.escapeHTML(e.message)}"]
+            # Message stays server-side (handle_exception logs it): exception
+            # text can carry file paths or Redis connection details, and every
+            # dashboard viewer sees this body.
+            [500, html_headers, 'Extension render error (see server logs)']
           end
 
           def action_for(ext, route_params, env, ctx)

data/lib/wurk/web/pool_scope.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Wurk
+  class Web
+    # Routes `Wurk.redis` onto the dedicated web pool (Wurk::Configuration
+    # #web_redis_pool) for the duration of a block.
+    #
+    # The dashboard, JSON API, and SSE stream run in the host's web process and
+    # reach Redis through the same inspector objects (Stats, Queue, Search, …) a
+    # worker uses — all of which call `Wurk.redis`. Left on the default pool, a
+    # burst of dashboard traffic or a long-held SSE stream could drain the
+    # connections a co-located (embedded) worker needs to make progress, and
+    # vice versa: the #101 pool-exhaustion incident.
+    #
+    # `Wurk.redis_pool` resolves its pool by calling `#redis_pool` on
+    # `Thread.current[:wurk_capsule]` (falling back to the default capsule).
+    # Pointing that thread-local at a handle whose `#redis_pool` is the web pool
+    # diverts every nested `Wurk.redis` call with no per-call-site change. The
+    # prior value is restored on exit so a reused web-server thread — or a
+    # nested scope — is left untouched.
+    module PoolScope
+      class << self
+        def scope
+          prev = Thread.current[:wurk_capsule]
+          Thread.current[:wurk_capsule] = handle
+          yield
+        ensure
+          Thread.current[:wurk_capsule] = prev
+        end
+
+        def handle
+          @handle ||= Handle.new
+        end
+      end
+
+      # Minimal capsule-shaped duck type: Wurk.redis_pool only ever calls
+      # `#redis_pool` on the thread-local. Resolved lazily so a post-fork
+      # `reset_redis_pools!` rebuild is picked up transparently.
+      class Handle
+        def redis_pool
+          Wurk.configuration.web_redis_pool
+        end
+      end
+    end
+  end
+end

data/lib/wurk/web/rack_app.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'extension'
+require_relative 'pool_scope'
 
 module Wurk
   class Web
@@ -29,7 +30,7 @@ module Wurk
       # auth here would break `run Sidekiq::Web` / rack-test parity with
       # upstream Sidekiq, which also doesn't auth-gate `Sidekiq::Web.call`.
       def call(env)
-        config.rack_app(method(:dispatch)).call(env)
+        PoolScope.scope { config.rack_app(method(:dispatch)).call(env) }
       end
 
       private