kino 0.6.0 → 0.7.1

data/ext/kino/src/server.rs

@@ -68,6 +68,10 @@ pub fn server_start(ruby: &Ruby, config: magnus::RHash) -> Result<(u64, u16, Opt
     let log_requests: bool = cfg_opt(ruby, config, "log_requests")?.unwrap_or(false);
     let mode: String = cfg_opt(ruby, config, "mode")?.unwrap_or_else(|| "threaded".to_string());
     let workers: usize = cfg_opt(ruby, config, "workers")?.unwrap_or(0);
+    // Absent or below the floor means a fixed pool.
+    let max_workers: usize = cfg_opt::<usize>(ruby, config, "max_workers")?
+        .unwrap_or(workers)
+        .max(workers);
     let threads: usize = cfg_opt(ruby, config, "threads")?.unwrap_or(0);
     let batch: usize = cfg_opt(ruby, config, "batch")?.unwrap_or(1);
     let acceptor = match (&tls_cert, &tls_key) {
@@ -113,6 +117,7 @@ pub fn server_start(ruby: &Ruby, config: magnus::RHash) -> Result<(u64, u16, Opt
     let (req_tx, req_rx) = flume::bounded(queue_depth);
     let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false);
 
+    let pin_slab = Arc::new(crate::pin::PinSlab::for_responses(mode == "ractor"));
     let server = Arc::new(ServerInner {
         id: registry::next_server_id(),
         req_tx: Mutex::new(Some(req_tx)),
@@ -130,9 +135,13 @@ pub fn server_start(ruby: &Ruby, config: magnus::RHash) -> Result<(u64, u16, Opt
         state: std::sync::atomic::AtomicU8::new(registry::STATE_BOOTING),
         respawns: std::sync::atomic::AtomicU64::new(0),
         quarantine_replacements: std::sync::atomic::AtomicU64::new(0),
+        active_workers: std::sync::atomic::AtomicUsize::new(workers),
+        scale_ups: std::sync::atomic::AtomicU64::new(0),
+        scale_downs: std::sync::atomic::AtomicU64::new(0),
         topology: registry::Topology {
             mode,
             workers,
+            max_workers,
             threads,
             batch,
         },
@@ -142,7 +151,7 @@ pub fn server_start(ruby: &Ruby, config: magnus::RHash) -> Result<(u64, u16, Opt
         access_log: log_requests.then(|| crate::logsink::Sink::new(std::io::stdout())),
         lanes,
         lane_cursor: std::sync::atomic::AtomicUsize::new(0),
-        pin_slab: Arc::new(crate::pin::PinSlab::new()),
+        pin_slab,
         queue_histogram: registry::QueueHistogram::new(),
     });
 
@@ -383,6 +392,12 @@ fn advertised_streams(workers: usize, threads: usize) -> u32 {
     slots.clamp(8, 1024) as u32
 }
 
+/// The pool can grow to its ceiling to meet what an h2 balancer sends, so
+/// the ceiling is the admission to advertise (the floor for a fixed pool).
+fn stream_capacity(topology: &registry::Topology) -> u32 {
+    advertised_streams(topology.max_workers, topology.threads)
+}
+
 fn conn_builder(http2: bool, max_streams: u32) -> auto::Builder<TokioExecutor> {
     let mut builder = auto::Builder::new(TokioExecutor::new());
     builder
@@ -418,7 +433,7 @@ async fn serve_connection<I>(
     I: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
 {
     let http2 = server.http2;
-    let max_streams = advertised_streams(server.topology.workers, server.topology.threads);
+    let max_streams = stream_capacity(&server.topology);
     let mut drain = server.shutdown_tx.subscribe();
     let service =
         service_fn(move |req| handle_request(server.clone(), remote_addr, local_addr, req));
@@ -712,6 +727,11 @@ fn try_dispatch(server: &ServerInner, mut ctx: BoxedCtx) -> Dispatch {
             let guard = slot.lane_tx.lock();
             let Some(tx) = guard.as_ref() else { continue };
             any_open = true;
+            // Checked under the lane lock, which WorkerSlot::retire also
+            // takes: see there for the ordering.
+            if slot.retired.load(Ordering::SeqCst) {
+                continue;
+            }
             match tx.try_send(ctx) {
                 Ok(()) => return Dispatch::Sent,
                 Err(flume::TrySendError::Full(c)) | Err(flume::TrySendError::Disconnected(c)) => {
@@ -900,7 +920,7 @@ pub fn queue_time(_ruby: &Ruby, server_id: u64) -> Result<(u64, f64), Error> {
 }
 
 /// One worker slot's [index, served, in_flight, busy_ms, quarantined] row.
-pub type WorkerStatRow = (usize, u64, usize, u64, bool);
+pub type WorkerStatRow = (usize, u64, usize, u64, bool, bool);
 
 /// Per-slot rows for Server#stats parity: [index, served, in_flight,
 /// busy_ms, quarantined] each. Empty when the server is gone.
@@ -910,7 +930,16 @@ pub fn worker_stats(_ruby: &Ruby, server_id: u64) -> Result<Vec<WorkerStatRow>,
     };
     Ok(crate::control::collect_worker_status(&server)
         .into_iter()
-        .map(|w| (w.index, w.served, w.in_flight, w.busy_ms, w.quarantined))
+        .map(|w| {
+            (
+                w.index,
+                w.served,
+                w.in_flight,
+                w.busy_ms,
+                w.quarantined,
+                w.retired,
+            )
+        })
         .collect())
 }
 
@@ -923,6 +952,61 @@ pub fn quarantine_slot(ruby: &Ruby, server_id: u64, worker_id: usize) -> Result<
     Ok(())
 }
 
+/// Send a slot's worker home once idle (pool scale-down); see
+/// registry::WorkerSlot::retire. An unknown id is a caller bug: raise.
+pub fn retire_slot(ruby: &Ruby, server_id: u64, worker_id: usize) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        server.slot(ruby, worker_id)?.retire();
+    }
+    Ok(())
+}
+
+/// Hand a retired slot to a new worker (pool scale-up reusing a slot).
+pub fn reset_slot(ruby: &Ruby, server_id: u64, worker_id: usize) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        server.slot(ruby, worker_id)?.reset();
+    }
+    Ok(())
+}
+
+/// The Ruby pool reports how many workers are alive and serving.
+pub fn set_active_workers(_ruby: &Ruby, server_id: u64, count: usize) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        server.active_workers.store(count, Ordering::Relaxed);
+    }
+    Ok(())
+}
+
+/// One worker added by the pool scaler.
+pub fn record_scale_up(_ruby: &Ruby, server_id: u64) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        server.scale_ups.fetch_add(1, Ordering::Relaxed);
+    }
+    Ok(())
+}
+
+/// One worker retired by the pool scaler.
+pub fn record_scale_down(_ruby: &Ruby, server_id: u64) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        server.scale_downs.fetch_add(1, Ordering::Relaxed);
+    }
+    Ok(())
+}
+
+/// [active_workers, max_workers, scale_ups, scale_downs] for Server#stats.
+/// Zeros when the server is gone.
+pub fn pool_stats(_ruby: &Ruby, server_id: u64) -> Result<(usize, usize, u64, u64), Error> {
+    let Some(server) = registry::try_get(server_id) else {
+        return Ok((0, 0, 0, 0));
+    };
+    Ok((
+        server.active_workers.load(Ordering::Relaxed),
+        server.topology.max_workers,
+        server.scale_ups.load(Ordering::Relaxed),
+        server.scale_downs.load(Ordering::Relaxed),
+    ))
+}
+
 /// One replacement spawned by the quarantine monitor.
 pub fn record_quarantine_replacement(_ruby: &Ruby, server_id: u64) -> Result<(), Error> {
     if let Some(server) = registry::try_get(server_id) {
@@ -1355,6 +1439,32 @@ mod tests {
         assert_eq!(advertised_streams(8, 0), 200);
     }
 
+    #[test]
+    fn stream_capacity_is_advertised_from_the_pool_ceiling() {
+        // An elastic pool grows to meet load an h2 balancer sends, so the
+        // ceiling, not the floor, is the admission to advertise.
+        let topology = registry::Topology {
+            mode: "ractor".to_string(),
+            workers: 2,
+            max_workers: 8,
+            threads: 3,
+            batch: 1,
+        };
+        assert_eq!(stream_capacity(&topology), 24);
+    }
+
+    #[test]
+    fn stream_capacity_of_a_fixed_pool_is_its_slot_count() {
+        let topology = registry::Topology {
+            mode: "threaded".to_string(),
+            workers: 4,
+            max_workers: 4,
+            threads: 3,
+            batch: 1,
+        };
+        assert_eq!(stream_capacity(&topology), 12);
+    }
+
     #[tokio::test]
     async fn streams_beyond_the_advertised_cap_queue_instead_of_failing() {
         use http_body_util::Full;
@@ -1566,4 +1676,110 @@ mod tests {
         assert!(matches!(try_dispatch(&server, test_ctx()), Dispatch::Sent));
         assert_eq!(server.lane_depths(), Some(vec![0, 2]));
     }
+
+    #[test]
+    fn dispatch_skips_retired_lanes() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        server.register_worker();
+        server.slots.read()[0].retire();
+
+        // A retiring worker gets nothing new: both land on slot 1.
+        assert!(matches!(try_dispatch(&server, test_ctx()), Dispatch::Sent));
+        assert!(matches!(try_dispatch(&server, test_ctx()), Dispatch::Sent));
+        assert_eq!(server.lane_depths(), Some(vec![0, 2]));
+    }
+
+    #[test]
+    fn dispatch_retries_rather_than_closing_when_only_retired_lanes_remain() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        server.slots.read()[0].retire();
+
+        // Retired is not draining: the caller keeps retrying until the
+        // queue timeout, so a replacement worker can still pick this up.
+        assert!(matches!(
+            try_dispatch(&server, test_ctx()),
+            Dispatch::Full(_)
+        ));
+    }
+
+    #[test]
+    fn dispatch_skips_a_retired_lane_even_when_it_is_the_only_awake_one() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        server.register_worker();
+        server.slots.read()[0].retire();
+        server.slots.read()[1].parked.store(true, Ordering::Relaxed);
+
+        // The second pass (parked lanes allowed) still skips the retired one.
+        assert!(matches!(try_dispatch(&server, test_ctx()), Dispatch::Sent));
+        assert_eq!(server.lane_depths(), Some(vec![0, 1]));
+    }
+
+    #[test]
+    fn dispatch_keeps_retrying_when_the_other_lanes_are_closed() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        server.register_worker();
+        server.slots.read()[0].lane_tx.lock().take(); // a crashed worker's lane
+        server.slots.read()[1].retire();
+
+        assert!(matches!(
+            try_dispatch(&server, test_ctx()),
+            Dispatch::Full(_)
+        ));
+    }
+
+    #[test]
+    fn a_reset_lane_receives_dispatches_again() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        server.register_worker();
+        server.slots.read()[0].retire();
+        assert!(matches!(try_dispatch(&server, test_ctx()), Dispatch::Sent));
+        assert_eq!(server.lane_depths(), Some(vec![0, 1]));
+
+        server.slots.read()[0].reset();
+        for _ in 0..2 {
+            assert!(matches!(try_dispatch(&server, test_ctx()), Dispatch::Sent));
+        }
+        // Back in the round-robin.
+        assert_eq!(server.lane_depths(), Some(vec![1, 2]));
+    }
+
+    #[test]
+    fn nothing_lands_in_a_lane_after_its_retirement_and_final_drain() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        server.register_worker();
+        let stop = Arc::new(std::sync::atomic::AtomicBool::new(false));
+        let dispatcher = std::thread::spawn({
+            let server = server.clone();
+            let stop = stop.clone();
+            move || {
+                while !stop.load(Ordering::Relaxed) {
+                    let _ = try_dispatch(&server, test_ctx());
+                    // Lane 1 keeps draining, so dispatch never stalls on Full.
+                    let slots = server.slots.read();
+                    if let Some(rx) = slots[1].lane_rx.as_ref() {
+                        while rx.try_recv().is_ok() {}
+                    }
+                }
+            }
+        });
+        std::thread::sleep(Duration::from_millis(20));
+
+        // The scaler retires slot 0; its worker then drains what it holds.
+        let slot = server.slots.read()[0].clone();
+        slot.retire();
+        if let Some(rx) = slot.lane_rx.as_ref() {
+            while rx.try_recv().is_ok() {}
+        }
+
+        std::thread::sleep(Duration::from_millis(50));
+        stop.store(true, Ordering::Relaxed);
+        dispatcher.join().expect("dispatcher thread");
+        assert_eq!(server.lane_depths(), Some(vec![0, 0]));
+    }
 }

data/ext/kino/src/test_support.rs

@@ -92,3 +92,47 @@ pub fn close(ruby: &magnus::Ruby, id: u64) -> Result<(), magnus::Error> {
     chan.tx.lock().take();
     Ok(())
 }
+
+/// Build the cache-backed slice of a Rack env directly, the way `build_env`
+/// does for a real request: REMOTE_ADDR from `ip`, SERVER_NAME/SERVER_PORT
+/// from `host` (the Host-header path) or, when `authority` is given, from
+/// it (the :authority path, which also sets HTTP_HOST), plus one interned
+/// header value. The rooting stress spec calls this from several ractors
+/// at once with more distinct inputs than the LRU caches hold.
+pub fn env_probe(
+    ruby: &magnus::Ruby,
+    host: String,
+    authority: Option<String>,
+    ip: String,
+    user_agent: String,
+) -> Result<magnus::RHash, magnus::Error> {
+    use crate::env_strings;
+    use crate::request::split_host_port;
+
+    let ip: std::net::IpAddr = ip.parse().map_err(|e| {
+        magnus::Error::new(ruby.exception_arg_error(), format!("bad ip {ip:?}: {e}"))
+    })?;
+    let env = ruby.hash_new();
+    env_strings::set_addr_env(ruby, env, ip)?;
+    match &authority {
+        Some(authority) => {
+            env_strings::set_authority_env(ruby, env, authority, || split_host_port(authority, 80))?
+        }
+        None => {
+            env_strings::set_host_env(ruby, env, host.as_bytes(), || split_host_port(&host, 80))?
+        }
+    }
+    let strings = env_strings::get();
+    let key = strings
+        .header_names
+        .get("user-agent")
+        .expect("user-agent is a common header");
+    env_strings::set_value_env(
+        ruby,
+        env,
+        ruby.get_inner(*key),
+        "user-agent",
+        user_agent.as_bytes(),
+    )?;
+    Ok(env)
+}

data/lib/kino/cli.rb

@@ -127,7 +127,7 @@ module Kino
       stats = server.stats
       puts dim("- ruby:      #{RUBY_DESCRIPTION}")
       puts dim("- env:       #{ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"}")
-      puts dim("- mode:      #{server.mode}, #{count(stats[:workers], "worker")} × #{count(stats[:threads], "thread")}")
+      puts dim("- mode:      #{server.mode}, #{workers_label(stats)} × #{count(stats[:threads], "thread")}")
       puts dim("- pid:       #{Process.pid}")
       puts dim("- listening: #{server.url}")
       puts dim("- control:   #{server.control_url}") if server.control_url
@@ -140,6 +140,14 @@ module Kino
       "#{number} #{noun}#{"s" unless number == 1}"
     end
 
+    # The pool as configured: "8 workers", or "8-32 workers" when it can
+    # grow.
+    def workers_label(stats)
+      return count(stats[:workers], "worker") if stats[:max_workers] == stats[:workers]
+
+      "#{stats[:workers]}-#{stats[:max_workers]} workers"
+    end
+
     # Roll credits when the process ends: normal exit or crash (at_exit
     # also runs after an uncaught exception; only a force-exit skips it).
     # @return [void]
@@ -234,7 +242,7 @@ module Kino
     end
 
     def write_sample(path)
-      require "kino"
+      require "kino" # audition:disable runtime-require
       Configuration.write_sample(path)
       puts "Kino: wrote sample config to #{path}"
       0
@@ -245,8 +253,8 @@ module Kino
 
     # Resolve the full configuration once: file + CLI flag overrides.
     def resolve_config(options)
-      require "kino"
-      require "rack"
+      require "kino" # audition:disable runtime-require
+      require "rack" # audition:disable runtime-require
 
       config_file = options[:config_file] || Configuration.default_path
 

data/lib/kino/configuration.rb

@@ -10,6 +10,8 @@ module Kino
       bind: "127.0.0.1",
       port: 0,
       workers: nil, # resolved to Kino.available_parallelism in #to_h
+      max_workers: nil, # nil = fixed pool; above workers = elastic pool
+      scale_down_after: nil, # resolved to 30 seconds in Server
       threads: nil, # resolved per mode in Server: 1 in :ractor, 3 in :threaded
       mode: :auto,
       queue_depth: 1024,
@@ -44,7 +46,7 @@ module Kino
     SETTINGS = DEFAULTS.keys.freeze
 
     # Source template for {.sample}.
-    SAMPLE_TEMPLATE = File.expand_path("templates/kino.rb.tt", __dir__)
+    SAMPLE_TEMPLATE = File.expand_path("templates/kino.rb.tt", __dir__).freeze
 
     # Where the `kino` CLI and the Rack handler look for a config file when
     # none is named: the project root first, then the Rails-style config/.
@@ -146,6 +148,8 @@ module Kino
     #   bind "0.0.0.0"
     #   port 9292
     #   workers 8           # ractors (or thread groups in :threaded mode)
+    #   max_workers 32      # elastic pool ceiling; unset = fixed pool
+    #   scale_down_after 30 # seconds idle before an extra worker retires
     #   threads 3           # threads per worker
     #   mode :ractor        # :auto | :ractor | :threaded
     #   queue_depth 2048
@@ -172,9 +176,19 @@ module Kino
       # Port to listen on; 0 picks an ephemeral port.
       def port(port) = @config.set(:port, Integer(port))
 
-      # Worker count (ractors in :ractor mode); defaults to CPU cores.
+      # Worker count (ractors in :ractor mode); defaults to CPU cores. The
+      # pool floor when max_workers is set.
       def workers(count) = @config.set(:workers, Integer(count))
 
+      # Pool ceiling: under sustained queue pressure the pool grows past
+      # `workers`, one worker at a time, up to this many. Unset (the
+      # default) keeps the pool fixed at `workers`.
+      def max_workers(count) = @config.set(:max_workers, Integer(count))
+
+      # Seconds a worker above the floor must sit idle before it is
+      # retired (default 30). Only meaningful with max_workers.
+      def scale_down_after(seconds) = @config.set(:scale_down_after, seconds)
+
       # Threads per worker (I/O concurrency inside one ractor); default is
       # mode-dependent: 1 in :ractor mode, 3 in :threaded.
       def threads(count) = @config.set(:threads, Integer(count))

data/lib/kino/monitor.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Kino
+  # @private
+  # A thread on the main ractor that calls `scan` every `tick` seconds
+  # until stopped. Main-ractor so it stays responsive when worker ractors
+  # are wedged. A scan that raises is logged and skipped, never fatal:
+  # monitors keep the server healthy, they must not take it down.
+  class Monitor
+    def initialize(name:, tick:)
+      @name = name
+      @tick = tick
+      @running = false
+      @thread = nil
+    end
+
+    def start
+      @running = true
+      @thread = Thread.new do
+        Thread.current.name = @name
+        run
+      end
+      self
+    end
+
+    def stop
+      @running = false
+      @thread&.join(@tick * 2)
+    end
+
+    private
+
+    def run
+      tick while @running
+    rescue => e
+      Log.error("#{@name} crashed: #{e.class}: #{e.message}")
+    end
+
+    def tick
+      scan
+    rescue => e
+      Log.error("#{@name} tick error: #{e.class}: #{e.message}")
+    ensure
+      sleep @tick
+    end
+
+    # One poll; subclasses define it.
+    def scan
+      raise NotImplementedError, "#{self.class} must define scan"
+    end
+  end
+end

data/lib/kino/pool_scaler.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Kino
+  # @private
+  # Grows and shrinks the worker pool between `floor` and `ceiling`. One
+  # thread on the main ractor polls the native queue and per-slot sensors
+  # every tick and drives a pool (RactorSupervisor or ThreadedPool) through
+  # three methods: `active_count`, `groups` (worker index => slot ids for
+  # every worker that may be retired), `grow`, and `retire(index)`.
+  #
+  # Policy: grow by one worker per tick once requests have been waiting in
+  # the queue on two consecutive ticks (a burst that clears within a tick
+  # is not pressure); retire one worker per tick, the one idle longest,
+  # once it has been idle for `scale_down_after`. "Idle" means no slot of
+  # the worker held a request at two consecutive samples and its served
+  # count did not move between them, so a worker serving short requests
+  # between samples is never mistaken for an idle one.
+  class PoolScaler < Monitor
+    TICK = 0.1
+    # Consecutive ticks with a non-empty queue before the pool grows.
+    PRESSURE_TICKS = 2
+
+    def initialize(server_id:, pool:, floor:, ceiling:, scale_down_after:, tick: TICK)
+      super(name: "pool scaler", tick: tick)
+      @server_id = server_id
+      @pool = pool
+      @floor = floor
+      @ceiling = ceiling
+      @scale_down_after = scale_down_after
+      @pressure = 0
+      @last_served = {}
+      @idle_since = {}
+    end
+
+    # One policy step over one set of observations: the monotonic time,
+    # the queue depth, and worker_stats rows ([slot, served, in_flight,
+    # busy_ms, quarantined, retired]). Public so the policy is testable
+    # without a server.
+    def step(now, queued, rows)
+      @pressure = queued.positive? ? @pressure + 1 : 0
+      if @pressure >= PRESSURE_TICKS
+        grow if @pool.active_count < @ceiling
+        return
+      end
+
+      observe_idle(now, rows)
+      return unless @pool.active_count > @floor
+
+      index, since = @idle_since.min_by { |_index, at| at }
+      return unless index && now - since >= @scale_down_after
+
+      retire(index)
+    end
+
+    private
+
+    def scan
+      queued, _in_flight = Native.queue_stats(@server_id)
+      step(Process.clock_gettime(Process::CLOCK_MONOTONIC), queued, Native.worker_stats(@server_id))
+    end
+
+    def grow
+      index = @pool.grow
+      return unless index
+
+      Native.record_scale_up(@server_id)
+      Log.info("pool grew to #{@pool.active_count} workers (queue pressure)")
+    end
+
+    def retire(index)
+      return unless @pool.retire(index)
+
+      forget(index)
+      Native.record_scale_down(@server_id)
+      Log.info("pool shrank to #{@pool.active_count} workers (worker-#{index} idle)")
+    end
+
+    # Per worker: idle when no slot holds a request at this sample and the
+    # served total did not move since the previous one. First sighting of
+    # a worker is never idle (it takes two samples to know).
+    def observe_idle(now, rows)
+      by_slot = rows.to_h { |row| [row[0], row] }
+      groups = @pool.groups
+      (@idle_since.keys - groups.keys).each { |index| forget(index) }
+      groups.each do |index, slot_ids|
+        served = slot_ids.sum { |id| by_slot.dig(id, 1) || 0 }
+        busy = slot_ids.any? { |id| (by_slot.dig(id, 2) || 0).positive? }
+        idle = !busy && @last_served[index] == served
+        @last_served[index] = served
+        if idle
+          @idle_since[index] ||= now
+        else
+          @idle_since.delete(index)
+        end
+      end
+    end
+
+    def forget(index)
+      @idle_since.delete(index)
+      @last_served.delete(index)
+    end
+  end
+end

data/lib/kino/quarantine_monitor.rb

@@ -3,54 +3,22 @@
 module Kino
   # @private
   # Polls per-slot busy_ms and, past the timeout, quarantines a wedged slot
-  # and asks the replacer to spawn a fresh worker. Runs one thread on the
-  # main ractor (uncontended by wedged worker ractors, so it stays
-  # responsive in :ractor mode). Never interrupts the wedged worker.
-  class QuarantineMonitor
+  # and asks the replacer to spawn a fresh worker. Never interrupts the
+  # wedged worker.
+  class QuarantineMonitor < Monitor
     def initialize(server_id:, timeout_ms:, max:, replacer:, tick: 0.5)
+      super(name: "quarantine monitor", tick: tick)
       @server_id = server_id
       @timeout_ms = timeout_ms
       @max = max
       @replacer = replacer
-      @tick = tick
       @outstanding = 0
       @at_cap_logged = false
-      @running = false
-      @thread = nil
-    end
-
-    def start
-      @running = true
-      @thread = Thread.new do
-        Thread.current.name = "quarantine"
-        run
-      end
-      self
-    end
-
-    def stop
-      @running = false
-      @thread&.join(@tick * 2)
     end
 
     private
 
-    def run
-      tick while @running
-    rescue => e
-      Log.error("quarantine monitor crashed: #{e.class}: #{e.message}")
-    end
-
-    def tick
-      scan_slots
-    rescue => e
-      # A bad tick must never kill the monitor.
-      Log.error("quarantine tick error: #{e.class}: #{e.message}")
-    ensure
-      sleep @tick
-    end
-
-    def scan_slots
+    def scan
       Native.worker_stats(@server_id).each do |index, _served, _in_flight, busy_ms, quarantined|
         next if quarantined || busy_ms <= @timeout_ms