wurk 1.1.0 → 1.1.2

data/lib/wurk/swarm/orphan_guard.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Wurk
+  class Swarm
+    # Orphan protection for a swarm child. A SIGKILL'd (or crashed, or OOM'd)
+    # supervisor leaves its children with no parent — they keep fetching
+    # forever, and the next redeploy that boots a fresh supervisor then runs
+    # doubled concurrency against the same queues. This arms two independent
+    # mechanisms so a child self-terminates the moment it is orphaned:
+    #
+    #   * Linux: PR_SET_PDEATHSIG delivers SIGTERM the instant the forking
+    #     thread dies (kernel-level, zero latency). It is set right after the
+    #     fork; the fork race — the parent can die in the window before the
+    #     syscall lands, so the signal is never queued — is closed by the
+    #     watchdog's immediate getppid check below.
+    #   * Everywhere: a watchdog thread compares getppid against the PID the
+    #     parent captured *before* forking (race-free — never the reaper PID).
+    #     On its first tick, and every `interval` seconds after, a mismatch
+    #     means the parent is gone. This is the only mechanism on non-Linux
+    #     (JRuby, macOS dev) and a backstop for pdeathsig's per-thread caveat.
+    #
+    # Both funnel through the child's own SIGTERM handler (Process.kill self),
+    # so orphan death is an ordinary graceful drain — in-flight jobs finish and
+    # the private list is requeued — not a hard kill.
+    class OrphanGuard
+      WATCHDOG_INTERVAL = 5
+      # <linux/prctl.h>: PR_SET_PDEATHSIG is option 1.
+      PR_SET_PDEATHSIG = 1
+
+      # pdeathsig via libc/prctl is Linux-only. Off elsewhere; the watchdog
+      # thread covers those platforms on its own.
+      def self.pdeathsig_supported?
+        ::RbConfig::CONFIG['host_os'].include?('linux')
+      end
+
+      # `on_orphan` is injectable so unit tests can observe the trip without
+      # actually signalling the test process. Production leaves it nil and
+      # self-TERMs, routing through the child's normal graceful-drain handler.
+      def initialize(parent_pid, logger:, interval: WATCHDOG_INTERVAL,
+                     pdeathsig: pdeathsig_supported?, on_orphan: nil)
+        @parent_pid = parent_pid
+        @logger = logger
+        @interval = interval
+        @pdeathsig = pdeathsig
+        @on_orphan = on_orphan || -> { ::Process.kill('TERM', ::Process.pid) }
+      end
+
+      # Arm both mechanisms. Returns the watchdog thread so the caller can
+      # retain it (otherwise GC could reap it).
+      def arm
+        set_pdeathsig
+        start_watchdog
+      end
+
+      def orphaned?
+        ::Process.ppid != @parent_pid
+      end
+
+      private
+
+      def pdeathsig_supported?
+        self.class.pdeathsig_supported?
+      end
+
+      # `sleep until orphaned?` checks first, so an already-orphaned child
+      # (the pdeathsig fork race) drains on the very first tick with no wait.
+      def start_watchdog
+        ::Thread.new do
+          ::Thread.current.name = 'wurk-orphan-watchdog'
+          ::Thread.current.report_on_exception = false
+          sleep @interval until orphaned?
+          drain
+        end
+      end
+
+      def drain
+        @logger&.warn { "swarm child #{::Process.pid}: parent #{@parent_pid} gone; draining orphan" }
+        @on_orphan.call
+      rescue StandardError => e
+        @logger&.error { "swarm child #{::Process.pid}: orphan drain failed: #{e.class}: #{e.message}" }
+      end
+
+      def set_pdeathsig
+        return unless @pdeathsig
+
+        arm_pdeathsig
+      rescue ::StandardError, ::LoadError => e
+        @logger&.debug { "swarm child #{::Process.pid}: PR_SET_PDEATHSIG unavailable (#{e.class}: #{e.message})" }
+      end
+
+      # prctl(int option, unsigned long arg2, …) — option is int, the rest are
+      # unsigned long. Fiddle opens the already-loaded libc (dlopen nil).
+      def arm_pdeathsig
+        require 'fiddle'
+        libc = ::Fiddle.dlopen(nil)
+        func = ::Fiddle::Function.new(
+          libc['prctl'],
+          [::Fiddle::TYPE_INT, ::Fiddle::TYPE_LONG, ::Fiddle::TYPE_LONG, ::Fiddle::TYPE_LONG, ::Fiddle::TYPE_LONG],
+          ::Fiddle::TYPE_INT
+        )
+        func.call(PR_SET_PDEATHSIG, ::Signal.list.fetch('TERM'), 0, 0, 0)
+      end
+    end
+  end
+end

data/lib/wurk/swarm/restart.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module Wurk
+  class Swarm
+    # Non-blocking rolling-restart / recycle state machine. One slot is in
+    # flight at a time; `advance` moves it a single phase per supervise tick so
+    # the supervisor never blocks on a restart and keeps honoring TERM:
+    #
+    #   spawn_replacement → await_heartbeat(heartbeat_wait) → term_old
+    #     → await_exit(drain_timeout) → done
+    #
+    # The reaper reports child exits via `claim_exit`, so a replacement that
+    # dies before it heartbeats is seen as dead (not "slow"): the old child is
+    # kept, a per-slot backoff applied, and the slot retried. `abort` drops
+    # everything — the swarm's TERM handler then drains the in-flight
+    # replacement + old as ordinary children.
+    #
+    # Collaborators are injected (Config) so the machine is decoupled from the
+    # swarm's fork/kill/Redis internals and unit-testable against fakes.
+    class Restart
+      Config = Struct.new(:spawn, :kill, :heartbeat, :describe, :now, :logger,
+                          :heartbeat_wait, :drain_timeout, :backoff, keyword_init: true)
+
+      def initialize(config)
+        @spawn = config.spawn            # ->(slot, idx) => replacement pid (registered by the swarm)
+        @kill = config.kill              # ->(pid, sig)
+        @heartbeat = config.heartbeat    # ->(pid) => truthy once the child has beaten
+        @describe = config.describe      # ->(pid) => { slot:, index: } | nil
+        @now = config.now                # -> monotonic seconds
+        @logger = config.logger
+        @heartbeat_wait = config.heartbeat_wait
+        @drain_timeout = config.drain_timeout
+        @backoff = config.backoff
+        @queue = []
+        @current = nil
+      end
+
+      def idle?
+        @current.nil? && @queue.empty?
+      end
+
+      # Queue live child PIDs for restart, skipping any already queued or in
+      # flight so recycle + rolling restart can't double up on one slot.
+      def enqueue(pids)
+        pids.each do |pid|
+          next if in_flight?(pid) || @queue.include?(pid)
+
+          @queue << pid
+        end
+      end
+
+      # Reaper hook. Returns true when `pid` belonged to the in-flight restart
+      # so the swarm skips crash-respawn for it. A replacement is only claimed
+      # while awaiting its heartbeat — once it takes over the slot (await_exit),
+      # its death is an ordinary crash the swarm respawns.
+      def claim_exit(pid) # rubocop:disable Naming/PredicateMethod
+        return false unless @current
+
+        if pid == @current[:old_pid]
+          @current[:old_exited] = true
+          true
+        elsif pid == @current[:replacement] && @current[:phase] == :await_heartbeat
+          @current[:replacement_dead] = true
+          true
+        else
+          false
+        end
+      end
+
+      def advance
+        start_next if @current.nil?
+        return if @current.nil?
+
+        case @current[:phase]
+        when :await_heartbeat then advance_await_heartbeat
+        when :await_exit then advance_await_exit
+        end
+      end
+
+      # TERM/INT: forget queued + in-flight work. The replacement and old child
+      # are ordinary children the swarm's shutdown TERMs and drains.
+      def abort
+        @queue.clear
+        @current = nil
+      end
+
+      private
+
+      def in_flight?(pid)
+        return false unless @current
+
+        pid == @current[:old_pid] || pid == @current[:replacement]
+      end
+
+      def start_next
+        pid = next_ready_pid
+        return unless pid
+
+        meta = @describe.call(pid)
+        replacement = spawn_replacement(pid, meta)
+        return unless replacement
+
+        @current = { old_pid: pid, index: meta[:index], replacement: replacement,
+                     phase: :await_heartbeat, deadline: now + @heartbeat_wait }
+        @logger.info { "swarm: restarting slot #{meta[:index]} (old #{pid} -> new #{replacement})" }
+      end
+
+      # Spawn the replacement, requeuing the slot on a fork/resource failure so
+      # the restart work item isn't lost (and the error can't escape the
+      # supervise loop). A per-slot backoff paces the retry — next_ready_pid
+      # leaves the head queued until it elapses.
+      def spawn_replacement(pid, meta)
+        @spawn.call(meta[:slot], meta[:index])
+      rescue StandardError => e
+        delay = @backoff.fail(meta[:index], lifetime: 0.0)
+        @queue.unshift(pid)
+        @logger.warn do
+          "swarm: replacement spawn for slot #{meta[:index]} failed (#{e.class}: #{e.message}); retry in #{delay}s"
+        end
+        nil
+      end
+
+      # Head of the queue if its old child still exists and its retry backoff
+      # has elapsed. Drops a stale head whose old child already vanished; leaves
+      # the head queued (returns nil) while a retry backoff is pending, so a
+      # slot whose replacement won't boot doesn't cascade into the next slot.
+      def next_ready_pid
+        pid = @queue.first
+        return nil unless pid
+
+        meta = @describe.call(pid)
+        if meta.nil?
+          @queue.shift
+          return nil
+        end
+        return nil unless @backoff.ready?(meta[:index])
+
+        @queue.shift
+      end
+
+      def advance_await_heartbeat
+        cur = @current
+        return retry_slot if cur[:replacement_dead]
+
+        seen = @heartbeat.call(cur[:replacement])
+        timed_out = now >= cur[:deadline]
+        return unless seen || timed_out
+
+        warn_heartbeat_timeout(cur) if timed_out && !seen
+        @kill.call(cur[:old_pid], 'TERM')
+        cur[:phase] = :await_exit
+        cur[:deadline] = now + @drain_timeout
+      end
+
+      def advance_await_exit
+        cur = @current
+        return finish(cur) if cur[:old_exited]
+        return if now < cur[:deadline]
+
+        # SIGKILL an overrunning old child once, then wait for the reaper to
+        # confirm the exit before finishing — finishing early would let the
+        # later reap look like a fresh crash and spawn a surplus child.
+        return if cur[:killed]
+
+        @kill.call(cur[:old_pid], 'KILL')
+        cur[:killed] = true
+      end
+
+      def finish(cur)
+        @backoff.clear(cur[:index])
+        @current = nil
+      end
+
+      def retry_slot
+        cur = @current
+        delay = @backoff.fail(cur[:index], lifetime: 0.0)
+        @queue.unshift(cur[:old_pid])
+        @logger.warn do
+          "swarm: replacement #{cur[:replacement]} for slot #{cur[:index]} died; " \
+            "keeping #{cur[:old_pid]}, retry in #{delay}s"
+        end
+        @current = nil
+      end
+
+      def warn_heartbeat_timeout(cur)
+        @logger.warn do
+          "swarm: replacement #{cur[:replacement]} heartbeat not seen in #{@heartbeat_wait}s; proceeding to TERM old"
+        end
+      end
+
+      def now
+        @now.call
+      end
+    end
+  end
+end