kino 0.2.0 → 0.3.0

data/ext/kino/src/lib.rs

@@ -4,9 +4,11 @@
 #[global_allocator]
 static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
+mod control;
 mod env_strings;
 mod gvl;
 mod logsink;
+mod mono;
 mod pin;
 mod queue;
 mod registry;
@@ -40,6 +42,16 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     native.define_singleton_method("close_queue", function!(server::close_queue, 1))?;
     native.define_singleton_method("queue_stats", function!(server::queue_stats, 1))?;
     native.define_singleton_method("server_stats", function!(server::server_stats, 1))?;
+    native.define_singleton_method("worker_stats", function!(server::worker_stats, 1))?;
+    native.define_singleton_method("queue_time", function!(server::queue_time, 1))?;
+    native.define_singleton_method("quarantine_slot", function!(server::quarantine_slot, 2))?;
+    native.define_singleton_method(
+        "record_quarantine_replacement",
+        function!(server::record_quarantine_replacement, 1),
+    )?;
+    native.define_singleton_method("control_ready", function!(server::control_ready, 1))?;
+    native.define_singleton_method("record_respawn", function!(server::record_respawn, 1))?;
+    native.define_singleton_method("control_stop", function!(control::control_stop, 1))?;
     native.define_singleton_method("abort_inflight", function!(server::abort_inflight, 2))?;
     native.define_singleton_method(
         "abort_all_inflight",

data/ext/kino/src/mono.rs

@@ -0,0 +1,26 @@
+//! Process-monotonic millisecond clock, shared by the request hot path
+//! (recording) and the control thread (reading busy age). Independent of
+//! wall-clock and of Ruby, so it is identical in :ractor and :threaded.
+
+use std::sync::OnceLock;
+use std::time::Instant;
+
+static MONO_EPOCH: OnceLock<Instant> = OnceLock::new();
+
+/// Milliseconds since the first call anywhere in the process. The u128
+/// millis fit u64 for any realistic uptime (u64 ms is ~584 million years).
+pub fn mono_ms() -> u64 {
+    MONO_EPOCH.get_or_init(Instant::now).elapsed().as_millis() as u64
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn mono_ms_is_monotonic_nondecreasing() {
+        let a = mono_ms();
+        let b = mono_ms();
+        assert!(b >= a, "clock went backwards: {a} then {b}");
+    }
+}

data/ext/kino/src/queue.rs

@@ -116,6 +116,12 @@ fn admit(
     mut ctx: BoxedCtx,
 ) -> Result<RHash, Error> {
     server.served.fetch_add(1, Ordering::Relaxed);
+    slot.served.fetch_add(1, Ordering::Relaxed);
+    slot.last_started_ms.store(crate::mono::mono_ms(), Ordering::Relaxed);
+    slot.in_flight.fetch_add(1, Ordering::Relaxed);
+    server
+        .queue_histogram
+        .record(ctx.enqueued_at.elapsed().as_micros() as u64);
     slot.current.lock().push(Arc::downgrade(&ctx.responder));
     // Wire the slot into the request so blocked body reads/writes are
     // interruptible the same way the queue pop is.
@@ -137,6 +143,7 @@ fn checkout(ruby: &Ruby, server_id: u64, worker_id: usize) -> Result<Option<Chec
 
     // The previous batch is fully answered once the worker comes back.
     slot.current.lock().clear();
+    slot.in_flight.store(0, Ordering::Relaxed);
     slot.interrupted.store(false, Ordering::SeqCst);
 
     Ok(block_take(&server, &slot)?.map(|ctx| (server, slot, ctx)))

data/ext/kino/src/registry.rs

@@ -11,6 +11,20 @@ use parking_lot::{Mutex, RwLock};
 use crate::request::RequestCtx;
 use crate::response::Responder;
 
+/// Lifecycle as seen by the control plane's /ready.
+pub const STATE_BOOTING: u8 = 0;
+pub const STATE_READY: u8 = 1;
+pub const STATE_DRAINING: u8 = 2;
+
+/// Boot-time configuration echoed by /stats. Stored resolved: mode is
+/// "ractor" or "threaded", never "auto".
+pub struct Topology {
+    pub mode: String,
+    pub workers: usize,
+    pub threads: usize,
+    pub batch: usize,
+}
+
 /// Requests travel through channels boxed: one heap allocation at accept
 /// time instead of moving ~300 bytes by value through every channel hop.
 pub type BoxedCtx = Box<RequestCtx>;
@@ -45,6 +59,15 @@ pub struct ServerInner {
     /// 413 (truthful Content-Length) or a mid-stream abort (chunked/lying).
     pub max_body_size: usize,
     pub timeouts: AtomicU64,
+    /// Lifecycle for /ready: booting until Ruby reports the workers up,
+    /// draining once stop_accepting runs. Relaxed everywhere (advisory).
+    pub state: std::sync::atomic::AtomicU8,
+    /// Worker respawns, recorded from the Ruby supervisor. Lives here so
+    /// the control plane reads it without touching Ruby.
+    pub respawns: AtomicU64,
+    /// Replacements spawned by the quarantine monitor (Relaxed, advisory).
+    pub quarantine_replacements: AtomicU64,
+    pub topology: Topology,
     pub https: bool,
     /// Native access log sink (None unless log_requests is on).
     pub access_log: Option<crate::logsink::Sink>,
@@ -55,6 +78,9 @@ pub struct ServerInner {
     /// GC roots for zero-copy response buffers (pin.rs). The Ruby Server
     /// object holds the marking PinKeeper for this slab.
     pub pin_slab: Arc<crate::pin::PinSlab>,
+    /// Queue-wait histogram: recorded at admit (queue.rs), emitted by the
+    /// control plane.
+    pub queue_histogram: QueueHistogram,
 }
 
 /// One per worker *thread* (slot count = workers × threads). The interrupt
@@ -72,12 +98,90 @@ pub struct WorkerSlot {
     pub lane_tx: Mutex<Option<flume::Sender<BoxedCtx>>>,
     pub lane_rx: Option<flume::Receiver<BoxedCtx>>,
     pub parked: std::sync::atomic::AtomicBool,
+    /// Per-slot sensors (Relaxed, advisory). served/in_flight mirror the
+    /// global counters at slot granularity; last_started_ms (stamped on
+    /// admit) drives busy-age (wedge) reporting.
+    pub served: AtomicU64,
+    pub in_flight: AtomicUsize,
+    pub last_started_ms: AtomicU64,
+    /// Set by the quarantine monitor when this slot is abandoned as wedged:
+    /// excluded from wedge detection, and its busy_ms is reported as 0.
+    pub quarantined: std::sync::atomic::AtomicBool,
 }
 
 /// Per-lane depth cap: small, so a slow handler can only ever delay this
 /// many queued neighbors (work stealing rescues them anyway).
 pub const LANE_DEPTH: usize = 4;
 
+/// Fixed queue-wait bucket boundaries in microseconds (0.5ms .. 10s),
+/// ascending. Emitted in seconds. Not a knob (YAGNI).
+pub const QUEUE_BOUNDS_US: [u64; 14] = [
+    500, 1_000, 2_500, 5_000, 10_000, 25_000, 50_000, 100_000,
+    250_000, 500_000, 1_000_000, 2_500_000, 5_000_000, 10_000_000,
+];
+
+/// Queue-wait histogram: per-bucket counts plus an overflow (the implicit
+/// +Inf bucket), the sum of waits, and the total count. Relaxed atomics,
+/// advisory like the other counters.
+pub struct QueueHistogram {
+    pub buckets: [AtomicU64; QUEUE_BOUNDS_US.len()],
+    pub overflow: AtomicU64,
+    pub sum_us: AtomicU64,
+    pub count: AtomicU64,
+}
+
+/// A plain (non-atomic) snapshot for the control thread to emit.
+pub struct QueueHistogramSnapshot {
+    pub buckets: [u64; QUEUE_BOUNDS_US.len()],
+    pub overflow: u64,
+    pub sum_us: u64,
+    pub count: u64,
+}
+
+impl QueueHistogram {
+    pub fn new() -> Self {
+        QueueHistogram {
+            buckets: std::array::from_fn(|_| AtomicU64::new(0)),
+            overflow: AtomicU64::new(0),
+            sum_us: AtomicU64::new(0),
+            count: AtomicU64::new(0),
+        }
+    }
+
+    /// Place one wait in its bucket (first bound >= wait, else overflow) and
+    /// update sum and count. A linear scan over 14 bounds is trivial.
+    pub fn record(&self, wait_us: u64) {
+        match QUEUE_BOUNDS_US.iter().position(|&bound| wait_us <= bound) {
+            Some(i) => self.buckets[i].fetch_add(1, Ordering::Relaxed),
+            None => self.overflow.fetch_add(1, Ordering::Relaxed),
+        };
+        self.sum_us.fetch_add(wait_us, Ordering::Relaxed);
+        self.count.fetch_add(1, Ordering::Relaxed);
+    }
+
+    pub fn snapshot(&self) -> QueueHistogramSnapshot {
+        QueueHistogramSnapshot {
+            buckets: std::array::from_fn(|i| self.buckets[i].load(Ordering::Relaxed)),
+            overflow: self.overflow.load(Ordering::Relaxed),
+            sum_us: self.sum_us.load(Ordering::Relaxed),
+            count: self.count.load(Ordering::Relaxed),
+        }
+    }
+}
+
+impl Default for QueueHistogram {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl QueueHistogramSnapshot {
+    /// Summed queue wait, converted from the stored microseconds to seconds.
+    pub fn sum_seconds(&self) -> f64 {
+        self.sum_us as f64 / 1_000_000.0
+    }
+}
+
 impl WorkerSlot {
     fn new(lanes: bool) -> Self {
         let (lane_tx, lane_rx) = if lanes {
@@ -92,6 +196,10 @@ impl WorkerSlot {
             lane_tx: Mutex::new(lane_tx),
             lane_rx,
             parked: std::sync::atomic::AtomicBool::new(false),
+            served: AtomicU64::new(0),
+            in_flight: AtomicUsize::new(0),
+            last_started_ms: AtomicU64::new(0),
+            quarantined: std::sync::atomic::AtomicBool::new(false),
         }
     }
 }
@@ -188,11 +296,16 @@ pub fn test_server(lanes: bool, queue_depth: usize) -> Arc<ServerInner> {
         request_timeout_ms: 0,
         max_body_size: 0,
         timeouts: AtomicU64::new(0),
+        state: std::sync::atomic::AtomicU8::new(STATE_BOOTING),
+        respawns: AtomicU64::new(0),
+        quarantine_replacements: AtomicU64::new(0),
+        topology: Topology { mode: "threaded".to_string(), workers: 0, threads: 0, batch: 1 },
         https: false,
         access_log: None,
         lanes,
         lane_cursor: AtomicUsize::new(0),
         pin_slab: Arc::new(crate::pin::PinSlab::new()),
+        queue_histogram: QueueHistogram::new(),
     })
 }
 
@@ -273,4 +386,46 @@ mod tests {
         let b = next_server_id();
         assert_ne!(a, b);
     }
+
+    #[test]
+    fn servers_boot_in_the_booting_state_with_zero_respawns() {
+        let server = test_server(false, 4);
+        assert_eq!(server.state.load(Ordering::Relaxed), STATE_BOOTING);
+        assert_eq!(server.respawns.load(Ordering::Relaxed), 0);
+        assert_eq!(server.topology.batch, 1);
+    }
+
+    #[test]
+    fn fresh_slot_has_zeroed_per_worker_sensors() {
+        let server = test_server(false, 4);
+        server.register_worker();
+        let slots = server.slots.read();
+        let slot = &slots[0];
+        assert_eq!(slot.served.load(Ordering::Relaxed), 0);
+        assert_eq!(slot.in_flight.load(Ordering::Relaxed), 0);
+        assert_eq!(slot.last_started_ms.load(Ordering::Relaxed), 0);
+    }
+
+    #[test]
+    fn fresh_slot_is_not_quarantined() {
+        let server = test_server(false, 4);
+        server.register_worker();
+        assert!(!server.slots.read()[0].quarantined.load(Ordering::Relaxed));
+        assert_eq!(server.quarantine_replacements.load(Ordering::Relaxed), 0);
+    }
+
+    #[test]
+    fn queue_histogram_buckets_by_wait() {
+        let h = QueueHistogram::new();
+        h.record(400);        // <= 500 -> bucket 0
+        h.record(500);        // == 500 -> bucket 0 (inclusive)
+        h.record(600);        // (500, 1000] -> bucket 1
+        h.record(20_000_000); // > last bound -> overflow
+        let s = h.snapshot();
+        assert_eq!(s.buckets[0], 2);
+        assert_eq!(s.buckets[1], 1);
+        assert_eq!(s.overflow, 1);
+        assert_eq!(s.count, 4);
+        assert_eq!(s.sum_us, 400 + 500 + 600 + 20_000_000);
+    }
 }

data/ext/kino/src/request.rs

@@ -41,6 +41,9 @@ pub struct RequestCtx {
     /// here and ride to hyper without a copy (pin.rs).
     pub pin_slab: Arc<crate::pin::PinSlab>,
     pub responder: Arc<Responder>,
+    /// When this request entered the queue, for the queue-wait histogram.
+    /// Stamped at ctx creation; read once at admit (queue.rs).
+    pub enqueued_at: std::time::Instant,
 }
 
 impl Drop for RequestCtx {
@@ -425,6 +428,7 @@ pub fn test_ctx() -> crate::registry::BoxedCtx {
         slot: None,
         pin_slab: Arc::new(crate::pin::PinSlab::new()),
         responder: Arc::new(Responder::new(head_tx)),
+        enqueued_at: std::time::Instant::now(),
     })
 }
 

data/ext/kino/src/server.rs

@@ -44,8 +44,10 @@ fn cfg_opt<T: magnus::TryConvert>(
 /// at boot, so Hash-lookup cost is irrelevant and the interface stays
 /// extensible. Binding is synchronous so address errors raise in Ruby at
 /// `start` time; returns the actual port for `port: 0`. TLS config errors
-/// (bad cert/key) also raise here, before any traffic.
-pub fn server_start(ruby: &Ruby, config: magnus::RHash) -> Result<(u64, u16), Error> {
+/// (bad cert/key) also raise here, before any traffic. The third element
+/// of the return tuple is the control-plane port (nil unless control_bind
+/// is configured).
+pub fn server_start(ruby: &Ruby, config: magnus::RHash) -> Result<(u64, u16, Option<u16>), Error> {
     let bind: String = cfg(ruby, config, "bind")?;
     let port: u16 = cfg(ruby, config, "port")?;
     let queue_depth: usize = cfg(ruby, config, "queue_depth")?;
@@ -58,6 +60,10 @@ pub fn server_start(ruby: &Ruby, config: magnus::RHash) -> Result<(u64, u16), Er
     let tls_key: Option<String> = cfg_opt(ruby, config, "tls_key")?;
     let lanes: bool = cfg_opt(ruby, config, "lanes")?.unwrap_or(false);
     let log_requests: bool = cfg_opt(ruby, config, "log_requests")?.unwrap_or(false);
+    let mode: String = cfg_opt(ruby, config, "mode")?.unwrap_or_else(|| "threaded".to_string());
+    let workers: usize = cfg_opt(ruby, config, "workers")?.unwrap_or(0);
+    let threads: usize = cfg_opt(ruby, config, "threads")?.unwrap_or(0);
+    let batch: usize = cfg_opt(ruby, config, "batch")?.unwrap_or(1);
     let acceptor = match (&tls_cert, &tls_key) {
         (Some(cert), Some(key)) => Some(
             crate::tls::build_acceptor(cert, key)
@@ -82,6 +88,15 @@ pub fn server_start(ruby: &Ruby, config: magnus::RHash) -> Result<(u64, u16), Er
         .map_err(|e| io_error(ruby, "listener setup failed", e))?
         .port();
 
+    let control_bind_addr: Option<String> = cfg_opt(ruby, config, "control_bind")?;
+    let control_token: Option<String> = cfg_opt(ruby, config, "control_token")?;
+    let control_bind = control_bind_addr
+        .as_deref()
+        .map(|addr| {
+            crate::control::bind_control(addr).map_err(|e| io_error(ruby, "control bind failed", e))
+        })
+        .transpose()?;
+
     let mut builder = tokio::runtime::Builder::new_multi_thread();
     builder.enable_all().thread_name("kino-tokio");
     if tokio_threads > 0 {
@@ -108,11 +123,16 @@ pub fn server_start(ruby: &Ruby, config: magnus::RHash) -> Result<(u64, u16), Er
         request_timeout_ms,
         max_body_size,
         timeouts: std::sync::atomic::AtomicU64::new(0),
+        state: std::sync::atomic::AtomicU8::new(registry::STATE_BOOTING),
+        respawns: std::sync::atomic::AtomicU64::new(0),
+        quarantine_replacements: std::sync::atomic::AtomicU64::new(0),
+        topology: registry::Topology { mode, workers, threads, batch },
         https: acceptor.is_some(),
         access_log: log_requests.then(|| crate::logsink::Sink::new(std::io::stdout())),
         lanes,
         lane_cursor: std::sync::atomic::AtomicUsize::new(0),
         pin_slab: Arc::new(crate::pin::PinSlab::new()),
+        queue_histogram: registry::QueueHistogram::new(),
     });
 
     let tokio_listener = {
@@ -130,8 +150,28 @@ pub fn server_start(ruby: &Ruby, config: magnus::RHash) -> Result<(u64, u16), Er
     *server.runtime.lock() = Some(runtime);
 
     let id = server.id;
+    let control_port = match control_bind {
+        // Not yet in the registry: on failure Ruby never learns this id, so
+        // nothing could ever reach it to shut it down. The accept loop's
+        // task holds its own Arc back to `server` (stored inside its own
+        // `runtime` field), so just dropping our handle would leak the
+        // runtime forever; take it out and stop it explicitly instead.
+        // Safe to block here: this is the plain Ruby thread, no async
+        // context above it.
+        Some(bind) => match crate::control::start(bind, server.clone(), control_token) {
+            Ok(port) => port,
+            Err(e) => {
+                // A plain drop blocks until the accept loop's task (its
+                // only task, idling on accept/shutdown) is torn down; the
+                // runtime only ever had this one thing to cancel.
+                drop(server.runtime.lock().take());
+                return Err(io_error(ruby, "control start failed", e));
+            }
+        },
+        None => None,
+    };
     registry::insert(server);
-    Ok((id, local_port))
+    Ok((id, local_port, control_port))
 }
 
 /// Slowloris guard for TLS: a client that completes the TCP connect but then
@@ -353,6 +393,7 @@ async fn handle_request(
         slot: None,
         pin_slab: server.pin_slab.clone(),
         responder,
+        enqueued_at: std::time::Instant::now(),
     });
 
     // Drop guard, not manual decrement: when a client aborts mid-request,
@@ -518,11 +559,28 @@ pub fn register_worker(ruby: &Ruby, server_id: u64) -> Result<usize, Error> {
 
 pub fn stop_accepting(_ruby: &Ruby, server_id: u64) -> Result<(), Error> {
     if let Some(server) = registry::try_get(server_id) {
+        server.state.store(registry::STATE_DRAINING, Ordering::Relaxed);
         let _ = server.shutdown_tx.send(true);
     }
     Ok(())
 }
 
+/// Ruby reports the worker pool up; /ready starts answering 200.
+pub fn control_ready(_ruby: &Ruby, server_id: u64) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        server.state.store(registry::STATE_READY, Ordering::Relaxed);
+    }
+    Ok(())
+}
+
+/// One worker respawn, recorded by the Ruby supervisor.
+pub fn record_respawn(_ruby: &Ruby, server_id: u64) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        server.respawns.fetch_add(1, Ordering::Relaxed);
+    }
+    Ok(())
+}
+
 pub fn close_queue(_ruby: &Ruby, server_id: u64) -> Result<(), Error> {
     if let Some(server) = registry::try_get(server_id) {
         server.req_tx.lock().take();
@@ -546,6 +604,12 @@ fn abort_slot(slot: &WorkerSlot) {
             responder.respond_500_if_unsent();
         }
     }
+    // A dead worker holds nothing: every request it had is answered above
+    // (or already was), so the slot is quiescent from here on. Without
+    // this, a crashed worker's slot reports in_flight>=1 forever (the
+    // supervisor never reuses a slot after a crash), wedging /stats,
+    // /metrics and server.stats with a phantom busy worker.
+    slot.in_flight.store(0, Ordering::Relaxed);
     // Lane mode: this worker is dead. Close its lane so the dispatcher
     // skips it, and drain anything queued; dropping each ctx fires the
     // Drop-500 backstop so those clients aren't left hanging.
@@ -608,14 +672,14 @@ pub fn log_error(message: String) {
 }
 
 /// Full stats snapshot: [queued, in_flight, served, rejected, timeouts,
-/// lane_depths]. lane_depths is nil unless lane dispatch is on.
+/// respawns, lane_depths]. lane_depths is nil unless lane dispatch is on.
 #[allow(clippy::type_complexity)]
 pub fn server_stats(
     _ruby: &Ruby,
     server_id: u64,
-) -> Result<(usize, usize, u64, u64, u64, Option<Vec<usize>>), Error> {
+) -> Result<(usize, usize, u64, u64, u64, u64, Option<Vec<usize>>), Error> {
     let Some(server) = registry::try_get(server_id) else {
-        return Ok((0, 0, 0, 0, 0, None));
+        return Ok((0, 0, 0, 0, 0, 0, None));
     };
     let lane_depths = server.lane_depths();
     let queued = server.req_rx.len() + lane_depths.as_ref().map_or(0, |d| d.iter().sum::<usize>());
@@ -625,10 +689,56 @@ pub fn server_stats(
         server.served.load(Ordering::Relaxed),
         server.rejected.load(Ordering::Relaxed),
         server.timeouts.load(Ordering::Relaxed),
+        server.respawns.load(Ordering::Relaxed),
         lane_depths,
     ))
 }
 
+/// Queue-wait count and summed seconds for Server#stats parity. Zeros when
+/// the server is gone.
+pub fn queue_time(_ruby: &Ruby, server_id: u64) -> Result<(u64, f64), Error> {
+    let Some(server) = registry::try_get(server_id) else {
+        return Ok((0, 0.0));
+    };
+    let h = server.queue_histogram.snapshot();
+    Ok((h.count, h.sum_seconds()))
+}
+
+/// One worker slot's [index, served, in_flight, busy_ms, quarantined] row.
+pub type WorkerStatRow = (usize, u64, usize, u64, bool);
+
+/// Per-slot rows for Server#stats parity: [index, served, in_flight,
+/// busy_ms, quarantined] each. Empty when the server is gone.
+pub fn worker_stats(
+    _ruby: &Ruby,
+    server_id: u64,
+) -> Result<Vec<WorkerStatRow>, Error> {
+    let Some(server) = registry::try_get(server_id) else {
+        return Ok(Vec::new());
+    };
+    Ok(crate::control::collect_worker_status(&server)
+        .into_iter()
+        .map(|w| (w.index, w.served, w.in_flight, w.busy_ms, w.quarantined))
+        .collect())
+}
+
+/// Mark a slot quarantined (the monitor has abandoned it as wedged).
+pub fn quarantine_slot(ruby: &Ruby, server_id: u64, worker_id: usize) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        let slot = server.slot(ruby, worker_id)?;
+        slot.quarantined.store(true, Ordering::Relaxed);
+    }
+    Ok(())
+}
+
+/// One replacement spawned by the quarantine monitor.
+pub fn record_quarantine_replacement(_ruby: &Ruby, server_id: u64) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        server.quarantine_replacements.fetch_add(1, Ordering::Relaxed);
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

data/lib/kino/cli.rb

@@ -100,11 +100,14 @@ module Kino
       end.join
     end
 
-    # One-line stats dump (the SIGUSR1 handler's output).
+    # One-line stats dump (the SIGUSR1 handler's output). Excludes
+    # worker_status: it's an array with one entry per execution slot, and
+    # printing it inline would break the one-line contract (see /stats for
+    # per-worker detail).
     # @param stats [Hash{Symbol => Object}] see {Kino::Server#stats}
     # @return [String]
     def stats_line(stats)
-      dim("Kino stats: #{stats.map { |k, v| "#{k}=#{v.inspect}" }.join(" ")}")
+      dim("Kino stats: #{stats.except(:worker_status).map { |k, v| "#{k}=#{v.inspect}" }.join(" ")}")
     end
 
     # The two banner halves around Server#start: credits before, the ready

data/lib/kino/configuration.rb

@@ -23,11 +23,19 @@ module Kino
       lanes: false,
       log_requests: false,
       on_error: nil,
+      after_boot: nil,
+      after_worker_boot: nil,
+      after_request_complete: nil,
+      on_worker_exit: nil,
       shutdown_timeout: 30,
       tokio_threads: nil,
       tls: nil,
       environment: nil,
       pidfile: nil,
+      control_bind: nil,
+      control_token: nil,
+      quarantine_timeout: nil,
+      quarantine_max: nil,
       rackup: nil
     }.freeze
 
@@ -185,6 +193,24 @@ module Kino
       # or a block. Must be Ractor-shareable in :ractor mode.
       def on_error(handler = nil, &block) = @config.set(:on_error, handler || block)
 
+      # Called once on the main thread after the worker pool is up. The
+      # readiness seam (wire sd_notify or a "server ready" metric here).
+      def after_boot(handler = nil, &block) = @config.set(:after_boot, handler || block)
+
+      # Called once inside each worker (a ractor in :ractor mode) before it
+      # serves, with the worker's slot id. Must be Ractor-shareable in
+      # :ractor mode (build it with Ractor.shareable_proc).
+      def after_worker_boot(handler = nil, &block) = @config.set(:after_worker_boot, handler || block)
+
+      # Called inside the worker after each successful response with
+      # (env, status). Hot path: leave unset for zero cost. Must be
+      # Ractor-shareable in :ractor mode.
+      def after_request_complete(handler = nil, &block) = @config.set(:after_request_complete, handler || block)
+
+      # Called on the main thread when a worker exits, with (worker_index,
+      # error_or_nil). error is the crash cause, or nil on a clean exit.
+      def on_worker_exit(handler = nil, &block) = @config.set(:on_worker_exit, handler || block)
+
       # Graceful-shutdown drain deadline in seconds.
       def shutdown_timeout(seconds) = @config.set(:shutdown_timeout, seconds)
 
@@ -200,6 +226,37 @@ module Kino
       # Write the master PID here on start.
       def pidfile(path) = @config.set(:pidfile, path.to_s)
 
+      # Serve the read-only control plane (live stats as JSON at /stats,
+      # Prometheus text at /metrics, /ready and /live probes) on this
+      # address: "host:port" or "unix://path". Off unless set.
+      def control_bind(addr) = @config.set(:control_bind, addr.to_s)
+
+      # When set, /stats and /metrics require "Authorization: Bearer <token>".
+      # The probes stay open; they carry no data.
+      def control_token(token) = @config.set(:control_token, token.to_s)
+
+      # Quarantine a dispatch slot whose current request has run longer
+      # than this many seconds, spawning a replacement to restore capacity.
+      # Off unless set. Set it above your slowest legitimate endpoint (and
+      # typically above request_timeout).
+      def quarantine_timeout(seconds)
+        seconds &&= Float(seconds)
+        if seconds && seconds <= 0
+          raise ArgumentError, "quarantine_timeout must be greater than 0 (got #{seconds})"
+        end
+        @config.set(:quarantine_timeout, seconds)
+      end
+
+      # Cap on the total number of replacement events over the process
+      # lifetime. Past the cap the monitor stops replacing and the server
+      # runs at reduced capacity. Default: the worker count in :ractor
+      # mode, workers x threads in :threaded.
+      def quarantine_max(count)
+        count = Integer(count)
+        raise ArgumentError, "quarantine_max must be >= 1 (got #{count})" if count < 1
+        @config.set(:quarantine_max, count)
+      end
+
       # Rackup file the `kino` CLI loads (positional argument wins).
       def rackup(path) = @config.set(:rackup, path.to_s)
     end

data/lib/kino/hook_fire.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Kino
+  # @private
+  # Fires a lifecycle hook and turns a raise into a logged line instead of
+  # letting it escape. Stateless and touches only its arguments plus
+  # Native.log_error (already called from inside worker ractors today), so
+  # it is safe to call from worker context: no main-ractor state is
+  # captured.
+  module HookFire
+    module_function
+
+    def fire(hook, name, *args)
+      return unless hook
+
+      begin
+        hook.call(*args)
+      rescue => e
+        Native.log_error("#{name} hook raised #{e.class}: #{e.message}")
+      end
+    end
+  end
+end