kino 0.2.1 → 0.4.0

data/ext/kino/src/cpus.rs

@@ -0,0 +1,37 @@
+//! How many CPUs this process may actually use: the default worker count.
+//!
+//! `Etc.nprocessors` honours the affinity mask but not a cgroup CPU quota,
+//! so a container limited to two CPUs on a 64-core host would spawn 64
+//! workers. The standard library's `available_parallelism` reads both the
+//! mask and the cgroup v1/v2 quota on Linux (rounding a fractional quota
+//! up), which is also what tokio sizes its own pool by.
+
+use magnus::{Error, Ruby};
+
+/// The usable CPU count, never below one (a quota of 0.5 CPU still needs
+/// a worker; an unreadable count falls back to one rather than failing
+/// boot).
+pub fn available_parallelism(_ruby: &Ruby) -> Result<usize, Error> {
+    Ok(count())
+}
+
+fn count() -> usize {
+    std::thread::available_parallelism().map_or(1, |n| n.get())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::count;
+
+    #[test]
+    fn reports_at_least_one_cpu() {
+        assert!(count() >= 1);
+    }
+
+    #[test]
+    fn never_exceeds_what_the_os_reports_as_online() {
+        // A quota can only lower the count below the online CPUs; never raise it.
+        let online = std::thread::available_parallelism().map_or(1, |n| n.get());
+        assert!(count() <= online);
+    }
+}

data/ext/kino/src/lib.rs

@@ -4,9 +4,15 @@
 #[global_allocator]
 static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
+mod access_log;
+mod control;
+mod cpus;
 mod env_strings;
 mod gvl;
+mod listen;
+mod log;
 mod logsink;
+mod mono;
 mod pin;
 mod queue;
 mod registry;
@@ -40,6 +46,16 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     native.define_singleton_method("close_queue", function!(server::close_queue, 1))?;
     native.define_singleton_method("queue_stats", function!(server::queue_stats, 1))?;
     native.define_singleton_method("server_stats", function!(server::server_stats, 1))?;
+    native.define_singleton_method("worker_stats", function!(server::worker_stats, 1))?;
+    native.define_singleton_method("queue_time", function!(server::queue_time, 1))?;
+    native.define_singleton_method("quarantine_slot", function!(server::quarantine_slot, 2))?;
+    native.define_singleton_method(
+        "record_quarantine_replacement",
+        function!(server::record_quarantine_replacement, 1),
+    )?;
+    native.define_singleton_method("control_ready", function!(server::control_ready, 1))?;
+    native.define_singleton_method("record_respawn", function!(server::record_respawn, 1))?;
+    native.define_singleton_method("control_stop", function!(control::control_stop, 1))?;
     native.define_singleton_method("abort_inflight", function!(server::abort_inflight, 2))?;
     native.define_singleton_method(
         "abort_all_inflight",
@@ -50,8 +66,12 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
         function!(server::interrupt_all_workers, 1),
     )?;
     native.define_singleton_method("shutdown_runtime", function!(server::shutdown_runtime, 2))?;
-    native.define_singleton_method("log_error", function!(server::log_error, 1))?;
+    native.define_singleton_method("log_line", function!(log::log_line, 3))?;
     native.define_singleton_method("sleep_chunk", function!(timer::sleep_chunk, 1))?;
+    native.define_singleton_method(
+        "available_parallelism",
+        function!(cpus::available_parallelism, 0),
+    )?;
     native.define_singleton_method("log_device_open", function!(logsink::device_open, 1))?;
     native.define_singleton_method("log_device_write", function!(logsink::device_write, 2))?;
     native.define_singleton_method("log_device_close", function!(logsink::device_close, 1))?;
@@ -75,6 +95,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     request.define_method("write_chunk", method!(Request::write_chunk, 1))?;
     request.define_method("finish", method!(Request::finish, 0))?;
     request.define_method("abort", method!(Request::abort, 0))?;
+    request.define_method("timing", method!(Request::set_timing, 2))?;
 
     // Force-resolve the TypedData class cache on the main ractor: magnus
     // resolves it lazily on first wrap, and a racy first resolution from two

data/ext/kino/src/listen.rs

@@ -0,0 +1,134 @@
+//! Listening sockets for the main server and the control plane: a TCP
+//! `host:port`, or a `unix://path` domain socket (the usual shape behind
+//! nginx). Binding is synchronous so an address conflict surfaces at boot.
+
+use std::io;
+use std::os::unix::net::UnixListener;
+use std::path::{Path, PathBuf};
+
+/// The bind scheme that selects a unix domain socket; anything else is a
+/// TCP host.
+pub const UNIX_SCHEME: &str = "unix://";
+
+/// The socket path of a `unix://` bind, or None for a TCP host.
+pub fn unix_path(bind: &str) -> Option<&Path> {
+    bind.strip_prefix(UNIX_SCHEME).map(Path::new)
+}
+
+/// A bound, non-blocking listener of either kind.
+pub enum Listener {
+    Tcp(std::net::TcpListener),
+    Unix(UnixListener, PathBuf),
+}
+
+impl Listener {
+    /// Bind `bind:port` (TCP; a hostname resolves to its addresses and the
+    /// first that binds wins) or `unix://path`.
+    pub fn bind(bind: &str, port: u16) -> io::Result<Listener> {
+        match unix_path(bind) {
+            Some(path) => Ok(Listener::Unix(bind_unix(path)?, path.to_path_buf())),
+            None => {
+                let listener = std::net::TcpListener::bind((bind, port))?;
+                listener.set_nonblocking(true)?;
+                Ok(Listener::Tcp(listener))
+            }
+        }
+    }
+
+    /// The bound TCP port; 0 for a unix socket, which has none.
+    pub fn port(&self) -> io::Result<u16> {
+        match self {
+            Listener::Tcp(listener) => Ok(listener.local_addr()?.port()),
+            Listener::Unix(..) => Ok(0),
+        }
+    }
+}
+
+/// Bind a unix domain socket at `path`. A path that already exists is
+/// either a live listener (refuse: never steal it) or a stale file left
+/// behind by a crashed process (unlink and reclaim). A connect probe tells
+/// them apart: a successful connect means someone is accepting right now.
+pub fn bind_unix(path: &Path) -> io::Result<UnixListener> {
+    match std::os::unix::net::UnixStream::connect(path) {
+        Ok(_) => return Err(io::Error::new(io::ErrorKind::AddrInUse, "socket is in use")),
+        Err(e) if e.kind() == io::ErrorKind::ConnectionRefused => {
+            match std::fs::remove_file(path) {
+                Ok(()) => {}
+                Err(e) if e.kind() == io::ErrorKind::NotFound => {}
+                Err(e) => return Err(e),
+            }
+        }
+        Err(e) if e.kind() == io::ErrorKind::NotFound => {}
+        Err(e) => return Err(e),
+    }
+    let listener = UnixListener::bind(path)?;
+    listener.set_nonblocking(true)?;
+    Ok(listener)
+}
+
+/// Remove a socket file at shutdown; one that is already gone is fine.
+pub fn cleanup_unix(path: &Path) {
+    let _ = std::fs::remove_file(path);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{bind_unix, unix_path, Listener};
+    use std::path::PathBuf;
+
+    /// A socket path unique to this process and test. macOS caps sun_path
+    /// at 104 bytes, so the name stays short.
+    fn socket_path(name: &str) -> PathBuf {
+        let path = std::env::temp_dir().join(format!("kino-{}-{name}.sock", std::process::id()));
+        let _ = std::fs::remove_file(&path);
+        path
+    }
+
+    #[test]
+    fn unix_path_recognises_only_the_unix_scheme() {
+        assert_eq!(unix_path("unix:///run/kino.sock").unwrap().to_str(), Some("/run/kino.sock"));
+        assert!(unix_path("127.0.0.1").is_none());
+        assert!(unix_path("unix.example.com").is_none());
+    }
+
+    #[test]
+    fn binds_a_unix_socket_and_reports_no_port() {
+        let path = socket_path("bind");
+        let listener = Listener::bind(&format!("unix://{}", path.display()), 9292).unwrap();
+        assert!(matches!(listener, Listener::Unix(..)));
+        assert_eq!(listener.port().unwrap(), 0);
+        assert!(std::fs::metadata(&path).is_ok());
+        drop(listener);
+        let _ = std::fs::remove_file(&path);
+    }
+
+    #[test]
+    fn reclaims_a_stale_socket_file() {
+        let path = socket_path("stale");
+        // Dropping a listener closes the socket but leaves its file behind,
+        // exactly what a crashed process leaves.
+        drop(bind_unix(&path).unwrap());
+        assert!(std::fs::metadata(&path).is_ok());
+        let reclaimed = bind_unix(&path).unwrap();
+        drop(reclaimed);
+        let _ = std::fs::remove_file(&path);
+    }
+
+    #[test]
+    fn refuses_a_socket_someone_is_listening_on() {
+        let path = socket_path("live");
+        let live = bind_unix(&path).unwrap();
+        let err = bind_unix(&path).unwrap_err();
+        assert_eq!(err.kind(), std::io::ErrorKind::AddrInUse);
+        assert!(err.to_string().contains("in use"));
+        drop(live);
+        let _ = std::fs::remove_file(&path);
+    }
+
+    #[test]
+    fn binds_tcp_on_an_ephemeral_port() {
+        let listener = Listener::bind("127.0.0.1", 0).unwrap();
+        assert!(matches!(listener, Listener::Tcp(_)));
+        assert_ne!(listener.port().unwrap(), 0);
+    }
+}

data/ext/kino/src/log.rs

@@ -0,0 +1,144 @@
+//! Server log lines: lifecycle notices, crashes and respawns, hook
+//! failures, the failed-request report, and whatever apps write to
+//! rack.errors, all in one shape:
+//!
+//! ```text
+//! kino[4213] worker-3: after_worker_boot hook raised RuntimeError: boom
+//! ```
+//!
+//! The label is syslog's `ident[pid]` tag plus the source that spoke (the
+//! ractor and/or thread name, `main` for neither), styled by level; the
+//! message stays plain. Ruby builds the source, since only Ruby knows its
+//! ractor and thread names, and hands the rest over; this side decides
+//! color per stream and writes, so worker ractors never touch $stdout or
+//! $stderr themselves. A multi-line message is labelled on its first line.
+
+use std::io::Write;
+
+use magnus::{Error, Ruby};
+
+use crate::style::{self, Stream};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Level {
+    Info,
+    Warn,
+    Error,
+}
+
+impl Level {
+    /// The level named by Ruby ("info", "warn", "error").
+    pub fn parse(name: &str) -> Option<Level> {
+        match name {
+            "info" => Some(Level::Info),
+            "warn" => Some(Level::Warn),
+            "error" => Some(Level::Error),
+            _ => None,
+        }
+    }
+
+    /// Notes go to stdout; warnings and errors to stderr.
+    fn stream(self) -> Stream {
+        match self {
+            Level::Info => Stream::Stdout,
+            Level::Warn | Level::Error => Stream::Stderr,
+        }
+    }
+
+    fn sgr(self) -> &'static str {
+        match self {
+            Level::Info => style::DIM,
+            Level::Warn => style::WARN,
+            Level::Error => style::ERROR,
+        }
+    }
+}
+
+/// Write one line from `source` at `level`.
+pub fn emit(level: Level, source: &str, message: &str) {
+    let stream = level.stream();
+    let label = label(std::process::id(), source);
+    let line = format_line(level, &label, message, style::enabled(stream));
+    match stream {
+        Stream::Stdout => {
+            let _ = writeln!(std::io::stdout().lock(), "{line}");
+        }
+        Stream::Stderr => {
+            let _ = writeln!(std::io::stderr().lock(), "{line}");
+        }
+    }
+}
+
+/// The `kino[<pid>] <source>:` tag.
+pub fn label(pid: u32, source: &str) -> String {
+    format!("kino[{pid}] {source}:")
+}
+
+/// The label styled by level, then the message as given.
+pub fn format_line(level: Level, label: &str, message: &str, color: bool) -> String {
+    format!("{} {message}", style::sgr(level.sgr(), label, color))
+}
+
+/// The Ruby entry point (Kino::Log): Ruby knows its ractor and thread,
+/// the native side knows the terminal.
+pub fn log_line(ruby: &Ruby, level: String, source: String, message: String) -> Result<(), Error> {
+    let level = Level::parse(&level).ok_or_else(|| {
+        Error::new(
+            ruby.exception_arg_error(),
+            format!("unknown log level {level:?}"),
+        )
+    })?;
+    emit(level, &source, &message);
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{format_line, label, Level};
+
+    #[test]
+    fn label_is_a_syslog_tag_plus_the_source() {
+        assert_eq!(label(4213, "main"), "kino[4213] main:");
+        assert_eq!(label(4213, "worker-3/thread-2"), "kino[4213] worker-3/thread-2:");
+    }
+
+    #[test]
+    fn plain_line_is_label_then_message() {
+        assert_eq!(
+            format_line(Level::Info, "kino[1] main:", "hello", false),
+            "kino[1] main: hello"
+        );
+    }
+
+    #[test]
+    fn color_styles_only_the_label_by_level() {
+        assert_eq!(
+            format_line(Level::Info, "kino[1] main:", "hello", true),
+            "\x1b[90mkino[1] main:\x1b[0m hello"
+        );
+        assert_eq!(
+            format_line(Level::Warn, "kino[1] main:", "careful", true),
+            "\x1b[33mkino[1] main:\x1b[0m careful"
+        );
+        assert_eq!(
+            format_line(Level::Error, "kino[1] main:", "broke", true),
+            "\x1b[91mkino[1] main:\x1b[0m broke"
+        );
+    }
+
+    #[test]
+    fn a_report_is_labelled_on_its_first_line_only() {
+        assert_eq!(
+            format_line(Level::Error, "kino[1] main:", "500 GET / · X: y\n    a.rb:1", false),
+            "kino[1] main: 500 GET / · X: y\n    a.rb:1"
+        );
+    }
+
+    #[test]
+    fn levels_parse_from_their_ruby_names() {
+        assert_eq!(Level::parse("info"), Some(Level::Info));
+        assert_eq!(Level::parse("warn"), Some(Level::Warn));
+        assert_eq!(Level::parse("error"), Some(Level::Error));
+        assert_eq!(Level::parse("debug"), None);
+    }
+}

data/ext/kino/src/mono.rs

@@ -0,0 +1,26 @@
+//! Process-monotonic millisecond clock, shared by the request hot path
+//! (recording) and the control thread (reading busy age). Independent of
+//! wall-clock and of Ruby, so it is identical in :ractor and :threaded.
+
+use std::sync::OnceLock;
+use std::time::Instant;
+
+static MONO_EPOCH: OnceLock<Instant> = OnceLock::new();
+
+/// Milliseconds since the first call anywhere in the process. The u128
+/// millis fit u64 for any realistic uptime (u64 ms is ~584 million years).
+pub fn mono_ms() -> u64 {
+    MONO_EPOCH.get_or_init(Instant::now).elapsed().as_millis() as u64
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn mono_ms_is_monotonic_nondecreasing() {
+        let a = mono_ms();
+        let b = mono_ms();
+        assert!(b >= a, "clock went backwards: {a} then {b}");
+    }
+}

data/ext/kino/src/queue.rs

@@ -116,6 +116,16 @@ fn admit(
     mut ctx: BoxedCtx,
 ) -> Result<RHash, Error> {
     server.served.fetch_add(1, Ordering::Relaxed);
+    slot.served.fetch_add(1, Ordering::Relaxed);
+    slot.last_started_ms.store(crate::mono::mono_ms(), Ordering::Relaxed);
+    slot.in_flight.fetch_add(1, Ordering::Relaxed);
+    // One clock read serves the histogram and, for the access log, the
+    // request's queue wait and the start of its time in Ruby.
+    let now = std::time::Instant::now();
+    let wait = now.duration_since(ctx.enqueued_at);
+    server.queue_histogram.record(wait.as_micros() as u64);
+    ctx.wait = wait;
+    ctx.admitted_at = now;
     slot.current.lock().push(Arc::downgrade(&ctx.responder));
     // Wire the slot into the request so blocked body reads/writes are
     // interruptible the same way the queue pop is.
@@ -137,6 +147,7 @@ fn checkout(ruby: &Ruby, server_id: u64, worker_id: usize) -> Result<Option<Chec
 
     // The previous batch is fully answered once the worker comes back.
     slot.current.lock().clear();
+    slot.in_flight.store(0, Ordering::Relaxed);
     slot.interrupted.store(false, Ordering::SeqCst);
 
     Ok(block_take(&server, &slot)?.map(|ctx| (server, slot, ctx)))

data/ext/kino/src/registry.rs

@@ -11,6 +11,20 @@ use parking_lot::{Mutex, RwLock};
 use crate::request::RequestCtx;
 use crate::response::Responder;
 
+/// Lifecycle as seen by the control plane's /ready.
+pub const STATE_BOOTING: u8 = 0;
+pub const STATE_READY: u8 = 1;
+pub const STATE_DRAINING: u8 = 2;
+
+/// Boot-time configuration echoed by /stats. Stored resolved: mode is
+/// "ractor" or "threaded", never "auto".
+pub struct Topology {
+    pub mode: String,
+    pub workers: usize,
+    pub threads: usize,
+    pub batch: usize,
+}
+
 /// Requests travel through channels boxed: one heap allocation at accept
 /// time instead of moving ~300 bytes by value through every channel hop.
 pub type BoxedCtx = Box<RequestCtx>;
@@ -45,7 +59,18 @@ pub struct ServerInner {
     /// 413 (truthful Content-Length) or a mid-stream abort (chunked/lying).
     pub max_body_size: usize,
     pub timeouts: AtomicU64,
+    /// Lifecycle for /ready: booting until Ruby reports the workers up,
+    /// draining once stop_accepting runs. Relaxed everywhere (advisory).
+    pub state: std::sync::atomic::AtomicU8,
+    /// Worker respawns, recorded from the Ruby supervisor. Lives here so
+    /// the control plane reads it without touching Ruby.
+    pub respawns: AtomicU64,
+    /// Replacements spawned by the quarantine monitor (Relaxed, advisory).
+    pub quarantine_replacements: AtomicU64,
+    pub topology: Topology,
     pub https: bool,
+    /// The socket file of a `unix://` bind, removed at shutdown.
+    pub unix_path: Option<std::path::PathBuf>,
     /// Native access log sink (None unless log_requests is on).
     pub access_log: Option<crate::logsink::Sink>,
     /// Lane-dispatch mode: per-worker queues, awake-preferring dispatch.
@@ -55,6 +80,9 @@ pub struct ServerInner {
     /// GC roots for zero-copy response buffers (pin.rs). The Ruby Server
     /// object holds the marking PinKeeper for this slab.
     pub pin_slab: Arc<crate::pin::PinSlab>,
+    /// Queue-wait histogram: recorded at admit (queue.rs), emitted by the
+    /// control plane.
+    pub queue_histogram: QueueHistogram,
 }
 
 /// One per worker *thread* (slot count = workers × threads). The interrupt
@@ -72,12 +100,90 @@ pub struct WorkerSlot {
     pub lane_tx: Mutex<Option<flume::Sender<BoxedCtx>>>,
     pub lane_rx: Option<flume::Receiver<BoxedCtx>>,
     pub parked: std::sync::atomic::AtomicBool,
+    /// Per-slot sensors (Relaxed, advisory). served/in_flight mirror the
+    /// global counters at slot granularity; last_started_ms (stamped on
+    /// admit) drives busy-age (wedge) reporting.
+    pub served: AtomicU64,
+    pub in_flight: AtomicUsize,
+    pub last_started_ms: AtomicU64,
+    /// Set by the quarantine monitor when this slot is abandoned as wedged:
+    /// excluded from wedge detection, and its busy_ms is reported as 0.
+    pub quarantined: std::sync::atomic::AtomicBool,
 }
 
 /// Per-lane depth cap: small, so a slow handler can only ever delay this
 /// many queued neighbors (work stealing rescues them anyway).
 pub const LANE_DEPTH: usize = 4;
 
+/// Fixed queue-wait bucket boundaries in microseconds (0.5ms .. 10s),
+/// ascending. Emitted in seconds. Not a knob (YAGNI).
+pub const QUEUE_BOUNDS_US: [u64; 14] = [
+    500, 1_000, 2_500, 5_000, 10_000, 25_000, 50_000, 100_000,
+    250_000, 500_000, 1_000_000, 2_500_000, 5_000_000, 10_000_000,
+];
+
+/// Queue-wait histogram: per-bucket counts plus an overflow (the implicit
+/// +Inf bucket), the sum of waits, and the total count. Relaxed atomics,
+/// advisory like the other counters.
+pub struct QueueHistogram {
+    pub buckets: [AtomicU64; QUEUE_BOUNDS_US.len()],
+    pub overflow: AtomicU64,
+    pub sum_us: AtomicU64,
+    pub count: AtomicU64,
+}
+
+/// A plain (non-atomic) snapshot for the control thread to emit.
+pub struct QueueHistogramSnapshot {
+    pub buckets: [u64; QUEUE_BOUNDS_US.len()],
+    pub overflow: u64,
+    pub sum_us: u64,
+    pub count: u64,
+}
+
+impl QueueHistogram {
+    pub fn new() -> Self {
+        QueueHistogram {
+            buckets: std::array::from_fn(|_| AtomicU64::new(0)),
+            overflow: AtomicU64::new(0),
+            sum_us: AtomicU64::new(0),
+            count: AtomicU64::new(0),
+        }
+    }
+
+    /// Place one wait in its bucket (first bound >= wait, else overflow) and
+    /// update sum and count. A linear scan over 14 bounds is trivial.
+    pub fn record(&self, wait_us: u64) {
+        match QUEUE_BOUNDS_US.iter().position(|&bound| wait_us <= bound) {
+            Some(i) => self.buckets[i].fetch_add(1, Ordering::Relaxed),
+            None => self.overflow.fetch_add(1, Ordering::Relaxed),
+        };
+        self.sum_us.fetch_add(wait_us, Ordering::Relaxed);
+        self.count.fetch_add(1, Ordering::Relaxed);
+    }
+
+    pub fn snapshot(&self) -> QueueHistogramSnapshot {
+        QueueHistogramSnapshot {
+            buckets: std::array::from_fn(|i| self.buckets[i].load(Ordering::Relaxed)),
+            overflow: self.overflow.load(Ordering::Relaxed),
+            sum_us: self.sum_us.load(Ordering::Relaxed),
+            count: self.count.load(Ordering::Relaxed),
+        }
+    }
+}
+
+impl Default for QueueHistogram {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl QueueHistogramSnapshot {
+    /// Summed queue wait, converted from the stored microseconds to seconds.
+    pub fn sum_seconds(&self) -> f64 {
+        self.sum_us as f64 / 1_000_000.0
+    }
+}
+
 impl WorkerSlot {
     fn new(lanes: bool) -> Self {
         let (lane_tx, lane_rx) = if lanes {
@@ -92,6 +198,10 @@ impl WorkerSlot {
             lane_tx: Mutex::new(lane_tx),
             lane_rx,
             parked: std::sync::atomic::AtomicBool::new(false),
+            served: AtomicU64::new(0),
+            in_flight: AtomicUsize::new(0),
+            last_started_ms: AtomicU64::new(0),
+            quarantined: std::sync::atomic::AtomicBool::new(false),
         }
     }
 }
@@ -188,11 +298,17 @@ pub fn test_server(lanes: bool, queue_depth: usize) -> Arc<ServerInner> {
         request_timeout_ms: 0,
         max_body_size: 0,
         timeouts: AtomicU64::new(0),
+        state: std::sync::atomic::AtomicU8::new(STATE_BOOTING),
+        respawns: AtomicU64::new(0),
+        quarantine_replacements: AtomicU64::new(0),
+        topology: Topology { mode: "threaded".to_string(), workers: 0, threads: 0, batch: 1 },
         https: false,
+        unix_path: None,
         access_log: None,
         lanes,
         lane_cursor: AtomicUsize::new(0),
         pin_slab: Arc::new(crate::pin::PinSlab::new()),
+        queue_histogram: QueueHistogram::new(),
     })
 }
 
@@ -273,4 +389,46 @@ mod tests {
         let b = next_server_id();
         assert_ne!(a, b);
     }
+
+    #[test]
+    fn servers_boot_in_the_booting_state_with_zero_respawns() {
+        let server = test_server(false, 4);
+        assert_eq!(server.state.load(Ordering::Relaxed), STATE_BOOTING);
+        assert_eq!(server.respawns.load(Ordering::Relaxed), 0);
+        assert_eq!(server.topology.batch, 1);
+    }
+
+    #[test]
+    fn fresh_slot_has_zeroed_per_worker_sensors() {
+        let server = test_server(false, 4);
+        server.register_worker();
+        let slots = server.slots.read();
+        let slot = &slots[0];
+        assert_eq!(slot.served.load(Ordering::Relaxed), 0);
+        assert_eq!(slot.in_flight.load(Ordering::Relaxed), 0);
+        assert_eq!(slot.last_started_ms.load(Ordering::Relaxed), 0);
+    }
+
+    #[test]
+    fn fresh_slot_is_not_quarantined() {
+        let server = test_server(false, 4);
+        server.register_worker();
+        assert!(!server.slots.read()[0].quarantined.load(Ordering::Relaxed));
+        assert_eq!(server.quarantine_replacements.load(Ordering::Relaxed), 0);
+    }
+
+    #[test]
+    fn queue_histogram_buckets_by_wait() {
+        let h = QueueHistogram::new();
+        h.record(400);        // <= 500 -> bucket 0
+        h.record(500);        // == 500 -> bucket 0 (inclusive)
+        h.record(600);        // (500, 1000] -> bucket 1
+        h.record(20_000_000); // > last bound -> overflow
+        let s = h.snapshot();
+        assert_eq!(s.buckets[0], 2);
+        assert_eq!(s.buckets[1], 1);
+        assert_eq!(s.overflow, 1);
+        assert_eq!(s.count, 4);
+        assert_eq!(s.sum_us, 400 + 500 + 600 + 20_000_000);
+    }
 }