kino 0.6.0 → 0.7.1

data/ext/kino/src/pin.rs

@@ -22,6 +22,22 @@
 //! GVL held), release is a single atomic store from whatever tokio
 //! thread drops the buffer, and the mark hook only loads atomics, so
 //! no path takes a lock. A full slab degrades to the copy path.
+//!
+//! The slab is also how env_strings.rs roots its cached env strings (a
+//! second slab, marked through its own keeper), for the same reason: no
+//! GC registration API may be called from parallel ractors.
+//!
+//! Per-ractor GC (Ruby 4.1): each ractor collects its own heap, and a
+//! local collection marks only that ractor's roots. A keeper on the main
+//! ractor is invisible to a worker ractor's local GC, so a string the
+//! worker allocated and rooted only here would be swept while hyper
+//! still reads its bytes. Shareable objects are exempt: only a global
+//! GC, which marks every root including the keeper, may free them. A
+//! slab serving worker ractors on such a Ruby therefore pins only
+//! strings already flagged shareable (typically an app's frozen
+//! constants) and copies the rest; making each per-request body
+//! shareable instead would push it into the population only a
+//! stop-the-world collection reclaims.
 
 use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
 use std::sync::Arc;
@@ -43,6 +59,10 @@ pub const ZERO_COPY_MIN: usize = 4096;
 /// throughput heuristic, not a limit on concurrency.
 const SLAB_CAPACITY: usize = 4096;
 
+/// Whether this Ruby collects each ractor's heap separately (see the
+/// module docs). Ruby 4.0 has one heap for every ractor.
+const PER_RACTOR_GC: bool = cfg!(ruby_gte_4_1);
+
 extern "C" {
     // Exported by libruby but absent from the public headers: io.c's
     // buffer-stabilizing primitive (see module docs). Signature per
@@ -50,24 +70,41 @@ extern "C" {
     fn rb_str_tmp_frozen_acquire(orig: rb_sys::VALUE) -> rb_sys::VALUE;
 }
 
-/// GC roots for in-flight response buffers. Slot value 0 = empty (a
-/// VALUE of 0 is Qfalse, which can never be a pinned string).
+/// A fixed slab of atomic VALUE slots, each a pinning GC root while it
+/// is non-zero. Slot value 0 = empty (a VALUE of 0 is Qfalse, which can
+/// never be a rooted string).
 pub struct PinSlab {
     slots: Box<[AtomicU64]>,
     /// Rotating claim cursor: keeps the free-slot scan O(1) amortized.
     cursor: AtomicUsize,
+    /// Pin only strings flagged shareable (pinned_bytes): set for a
+    /// slab that worker ractors fill on a per-ractor-GC Ruby.
+    shareable_only: bool,
 }
 
 impl PinSlab {
-    pub fn new() -> Self {
+    /// A slab for one server's in-flight response buffers, filled by
+    /// worker ractors when `ractor_mode` (else by main-ractor threads).
+    pub fn for_responses(ractor_mode: bool) -> Self {
         PinSlab {
-            slots: (0..SLAB_CAPACITY).map(|_| AtomicU64::new(0)).collect(),
+            shareable_only: ractor_mode && PER_RACTOR_GC,
+            ..Self::with_capacity(SLAB_CAPACITY)
+        }
+    }
+
+    /// A slab for callers that root only shareable values themselves
+    /// (env_strings), or for tests.
+    pub fn with_capacity(capacity: usize) -> Self {
+        PinSlab {
+            slots: (0..capacity).map(|_| AtomicU64::new(0)).collect(),
             cursor: AtomicUsize::new(0),
+            shareable_only: false,
         }
     }
 
-    /// Root `value`; None when the slab is full (caller copies instead).
-    fn insert(&self, value: rb_sys::VALUE) -> Option<usize> {
+    /// Root `value`; None when the slab is full (the caller falls back to
+    /// a copy, or to an uncached string).
+    pub(crate) fn insert(&self, value: rb_sys::VALUE) -> Option<usize> {
         let start = self.cursor.fetch_add(1, Ordering::Relaxed);
         for offset in 0..self.slots.len() {
             let index = (start + offset) % self.slots.len();
@@ -83,7 +120,7 @@ impl PinSlab {
 
     /// Drop the root. Called from tokio threads: a plain atomic store,
     /// no Ruby API. The string stays alive until the next GC sweep.
-    fn release(&self, index: usize) {
+    pub(crate) fn release(&self, index: usize) {
         self.slots[index].store(0, Ordering::SeqCst);
     }
 
@@ -113,9 +150,10 @@ impl PinSlab {
     }
 }
 
-/// The GC-visible face of a server's PinSlab: `Kino::Server` holds one
-/// for its lifetime (surviving worker-ractor crashes), and Ruby's GC
-/// marks every pinned string through it.
+/// The GC-visible face of a PinSlab: Ruby's GC marks every rooted string
+/// through it. `Kino::Server` holds one for its response-buffer slab for
+/// the server's lifetime (surviving worker-ractor crashes); env_strings
+/// registers one for its cache slab as an immortal root at init.
 #[derive(magnus::TypedData)]
 #[magnus(class = "Kino::Native::PinKeeper", free_immediately, mark)]
 pub struct PinKeeper(pub Arc<PinSlab>);
@@ -157,9 +195,10 @@ impl Drop for PinnedBuf {
 }
 
 /// Bytes borrowing `body`'s buffer, with the string rooted until hyper
-/// drops it; None when the body is below ZERO_COPY_MIN or the slab is
-/// full (caller copies). Requires the calling worker's GVL; safe from
-/// any ractor.
+/// drops it; None when the body is below ZERO_COPY_MIN, the slab is
+/// full, or the slab takes only shareable strings and this one is not
+/// (caller copies). Requires the calling worker's GVL; safe from any
+/// ractor.
 pub fn pinned_bytes(slab: &Arc<PinSlab>, body: RString) -> Option<Bytes> {
     if body.len() < ZERO_COPY_MIN {
         return None;
@@ -168,6 +207,9 @@ pub fn pinned_bytes(slab: &Arc<PinSlab>, body: RString) -> Option<Bytes> {
     // by the caller. The acquired tmp is rooted by this thread's machine
     // stack (conservatively scanned) until the slab insert publishes it.
     let tmp = unsafe { rb_str_tmp_frozen_acquire(body.as_raw()) };
+    if slab.shareable_only && !flagged_shareable(tmp) {
+        return None;
+    }
     let index = slab.insert(tmp)?;
     // SAFETY: tmp is frozen, alive, and slab-rooted; len >= ZERO_COPY_MIN
     // rules out an embedded buffer, so ptr is a stable heap allocation.
@@ -185,13 +227,22 @@ pub fn pinned_bytes(slab: &Arc<PinSlab>, body: RString) -> Option<Bytes> {
     }))
 }
 
+/// Whether `value` (a heap object) carries the shareable flag. A plain
+/// flag read: a frozen string that no one has checked yet may be
+/// shareable in principle but unflagged, and the GC goes by the flag.
+fn flagged_shareable(value: rb_sys::VALUE) -> bool {
+    let shareable = rb_sys::ruby_fl_type::RUBY_FL_SHAREABLE as rb_sys::VALUE;
+    // SAFETY: value is a live heap object (an RString) under the GVL.
+    unsafe { (*(value as *const rb_sys::RBasic)).flags & shareable != 0 }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
     #[test]
     fn insert_release_reuse_cycle() {
-        let slab = PinSlab::new();
+        let slab = PinSlab::with_capacity(SLAB_CAPACITY);
         let a = slab.insert(0x1000).expect("slot free");
         let b = slab.insert(0x2000).expect("slot free");
         assert_ne!(a, b);
@@ -210,7 +261,7 @@ mod tests {
 
     #[test]
     fn full_slab_refuses_instead_of_evicting() {
-        let slab = PinSlab::new();
+        let slab = PinSlab::with_capacity(SLAB_CAPACITY);
         let indexes: Vec<usize> = (0..SLAB_CAPACITY)
             .map(|i| slab.insert(0x1000 + i as rb_sys::VALUE).expect("capacity"))
             .collect();

data/ext/kino/src/queue.rs

@@ -25,6 +25,21 @@ pub const TICK: Duration = Duration::from_millis(50);
 /// the difference and doesn't need to.
 type Taken = Option<BoxedCtx>;
 
+/// What a take does when its bounded wait times out: keep waiting (None),
+/// or end the loop because the pool scaler retired this slot. A retired
+/// lane worker first empties its own lane, one item per timeout: the
+/// dispatcher stopped feeding it when the flag went up (see
+/// registry::WorkerSlot::retire for the ordering), so whatever is
+/// still there is the last of it, and nothing already assigned is
+/// orphaned. Only reached with the queue momentarily empty, so the fast
+/// path pays nothing for it.
+fn after_timeout(slot: &WorkerSlot) -> Option<Taken> {
+    if !slot.retired.load(Ordering::SeqCst) {
+        return None;
+    }
+    Some(slot.lane_rx.as_ref().and_then(|rx| rx.try_recv().ok()))
+}
+
 /// Block until one request arrives (GVL released, interruptible).
 /// No busy-poll before parking, deliberately: the wake-per-request futex
 /// cost is real (~20% of cycles at saturation, per perf), but a measured
@@ -46,7 +61,7 @@ fn block_take(server: &ServerInner, slot: &Arc<WorkerSlot>) -> Result<Taken, Err
             let taken =
                 gvl::interruptible(&slot.interrupted, || match req_rx.recv_timeout(TICK) {
                     Ok(ctx) => Some(Some(ctx)),
-                    Err(flume::RecvTimeoutError::Timeout) => None,
+                    Err(flume::RecvTimeoutError::Timeout) => after_timeout(slot),
                     Err(flume::RecvTimeoutError::Disconnected) => Some(None),
                 })?;
             Ok(taken.flatten())
@@ -93,8 +108,10 @@ fn lane_take(server: &ServerInner, slot: &Arc<WorkerSlot>) -> Result<Taken, Erro
         match lane_rx.recv_timeout(TICK) {
             Ok(ctx) => Some(Some(ctx)),
             // Periodic steal so a backlog behind a slow sibling can't
-            // outlive a tick.
-            Err(flume::RecvTimeoutError::Timeout) => steal().map(Some),
+            // outlive a tick; a retired worker leaves instead.
+            Err(flume::RecvTimeoutError::Timeout) => {
+                after_timeout(slot).or_else(|| steal().map(Some))
+            }
             Err(flume::RecvTimeoutError::Disconnected) => Some(None),
         }
     });
@@ -235,3 +252,82 @@ pub fn respond_and_take(
     crate::request::respond_simple(ruby, request, status, headers, body)?;
     Worker::take_batch(ruby, &worker, max)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::registry::test_server;
+    use crate::request::test_ctx;
+
+    #[test]
+    fn a_take_keeps_waiting_after_a_timeout_unless_the_slot_is_retired() {
+        let server = test_server(false, 4);
+        server.register_worker();
+        let slot = server.slots.read()[0].clone();
+
+        assert!(after_timeout(&slot).is_none());
+    }
+
+    #[test]
+    fn a_retired_shared_queue_worker_ends_its_loop_at_the_next_timeout() {
+        let server = test_server(false, 4);
+        server.register_worker();
+        server.slots.read()[0].retire();
+        let slot = server.slots.read()[0].clone();
+
+        assert!(matches!(after_timeout(&slot), Some(None)));
+    }
+
+    #[test]
+    fn a_retired_lane_worker_drains_its_own_lane_before_leaving() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        let slot = server.slots.read()[0].clone();
+        slot.lane_tx
+            .lock()
+            .as_ref()
+            .expect("lane open")
+            .send(test_ctx())
+            .expect("lane has room");
+        server.slots.read()[0].retire();
+
+        // One item still assigned to this lane: serve it, not orphan it.
+        assert!(matches!(after_timeout(&slot), Some(Some(_))));
+        // Lane empty now: leave.
+        assert!(matches!(after_timeout(&slot), Some(None)));
+    }
+
+    #[test]
+    fn a_retired_lane_worker_drains_every_item_one_per_timeout() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        let slot = server.slots.read()[0].clone();
+        for _ in 0..3 {
+            slot.lane_tx
+                .lock()
+                .as_ref()
+                .expect("lane open")
+                .send(test_ctx())
+                .expect("lane has room");
+        }
+        server.slots.read()[0].retire();
+
+        for _ in 0..3 {
+            assert!(matches!(after_timeout(&slot), Some(Some(_))));
+        }
+        assert!(matches!(after_timeout(&slot), Some(None)));
+    }
+
+    #[test]
+    fn a_reset_slot_keeps_waiting_again() {
+        let server = test_server(false, 4);
+        server.register_worker();
+        let slot = server.slots.read()[0].clone();
+        slot.retire();
+        assert!(matches!(after_timeout(&slot), Some(None)));
+
+        slot.reset();
+
+        assert!(after_timeout(&slot).is_none());
+    }
+}

data/ext/kino/src/registry.rs

@@ -21,7 +21,11 @@ pub const STATE_DRAINING: u8 = 2;
 /// "ractor" or "threaded", never "auto".
 pub struct Topology {
     pub mode: String,
+    /// The pool floor: the configured worker count.
     pub workers: usize,
+    /// The pool ceiling: equal to `workers` for a fixed pool. HTTP/2
+    /// admission (SETTINGS_MAX_CONCURRENT_STREAMS) is advertised from it.
+    pub max_workers: usize,
     pub threads: usize,
     pub batch: usize,
 }
@@ -113,6 +117,13 @@ pub struct ServerInner {
     pub respawns: AtomicU64,
     /// Replacements spawned by the quarantine monitor (Relaxed, advisory).
     pub quarantine_replacements: AtomicU64,
+    /// Workers staying in the pool (a worker told to retire no longer
+    /// counts), reported by the Ruby pool as it grows and shrinks; starts
+    /// at the floor. Relaxed, advisory.
+    pub active_workers: AtomicUsize,
+    /// Pool scaler events (Relaxed, advisory).
+    pub scale_ups: AtomicU64,
+    pub scale_downs: AtomicU64,
     pub topology: Topology,
     pub https: bool,
     /// HTTP/2 serving (ALPN over TLS, prior-knowledge h2c on plaintext);
@@ -158,6 +169,11 @@ pub struct WorkerSlot {
     /// Set by the quarantine monitor when this slot is abandoned as wedged:
     /// excluded from wedge detection, and its busy_ms is reported as 0.
     pub quarantined: std::sync::atomic::AtomicBool,
+    /// Set by the pool scaler to send this slot's worker home: the lane
+    /// dispatcher skips it, and the take loop ends at its next idle tick
+    /// (a request already taken finishes first). Cleared by `reset` when
+    /// the slot is handed to a new worker.
+    pub retired: std::sync::atomic::AtomicBool,
 }
 
 /// Per-lane depth cap: small, so a slow handler can only ever delay this
@@ -251,8 +267,35 @@ impl WorkerSlot {
             in_flight: AtomicUsize::new(0),
             last_started_ms: AtomicU64::new(0),
             quarantined: std::sync::atomic::AtomicBool::new(false),
+            retired: std::sync::atomic::AtomicBool::new(false),
         }
     }
+
+    /// Send this slot's worker home once it is idle. The flag is raised
+    /// under the lane lock, the same lock the lane dispatcher holds while
+    /// it checks the flag and sends: any dispatch that saw "not retired"
+    /// has landed in the lane before the worker can see the flag and
+    /// drain, so no request is orphaned.
+    pub fn retire(&self) {
+        let _lane = self.lane_tx.lock();
+        self.retired.store(true, Ordering::SeqCst);
+    }
+
+    /// Return a retired slot to its fresh state so a new worker can take
+    /// it over (slots are never removed; recycling keeps the registry
+    /// from growing with every scale-up). The lane channel stays, the new
+    /// occupant simply starts taking from it; the quarantine mark stays
+    /// too, a quarantined slot is never handed out.
+    pub fn reset(&self) {
+        let _lane = self.lane_tx.lock();
+        self.current.lock().clear();
+        self.retired.store(false, Ordering::SeqCst);
+        self.parked.store(false, Ordering::SeqCst);
+        self.interrupted.store(false, Ordering::SeqCst);
+        self.served.store(0, Ordering::Relaxed);
+        self.in_flight.store(0, Ordering::Relaxed);
+        self.last_started_ms.store(0, Ordering::Relaxed);
+    }
 }
 
 static REGISTRY: OnceLock<RwLock<HashMap<u64, Arc<ServerInner>>>> = OnceLock::new();
@@ -350,9 +393,13 @@ pub fn test_server(lanes: bool, queue_depth: usize) -> Arc<ServerInner> {
         state: std::sync::atomic::AtomicU8::new(STATE_BOOTING),
         respawns: AtomicU64::new(0),
         quarantine_replacements: AtomicU64::new(0),
+        active_workers: AtomicUsize::new(0),
+        scale_ups: AtomicU64::new(0),
+        scale_downs: AtomicU64::new(0),
         topology: Topology {
             mode: "threaded".to_string(),
             workers: 0,
+            max_workers: 0,
             threads: 0,
             batch: 1,
         },
@@ -362,7 +409,7 @@ pub fn test_server(lanes: bool, queue_depth: usize) -> Arc<ServerInner> {
         access_log: None,
         lanes,
         lane_cursor: AtomicUsize::new(0),
-        pin_slab: Arc::new(crate::pin::PinSlab::new()),
+        pin_slab: Arc::new(crate::pin::PinSlab::for_responses(false)),
         queue_histogram: QueueHistogram::new(),
     })
 }
@@ -491,6 +538,18 @@ mod tests {
         assert_eq!(server.topology.batch, 1);
     }
 
+    #[test]
+    fn pool_counters_start_at_the_configured_floor() {
+        let server = test_server(false, 4);
+        assert_eq!(
+            server.active_workers.load(Ordering::Relaxed),
+            server.topology.workers
+        );
+        assert_eq!(server.topology.max_workers, server.topology.workers);
+        assert_eq!(server.scale_ups.load(Ordering::Relaxed), 0);
+        assert_eq!(server.scale_downs.load(Ordering::Relaxed), 0);
+    }
+
     #[test]
     fn fresh_slot_has_zeroed_per_worker_sensors() {
         let server = test_server(false, 4);
@@ -510,6 +569,88 @@ mod tests {
         assert_eq!(server.quarantine_replacements.load(Ordering::Relaxed), 0);
     }
 
+    #[test]
+    fn fresh_slot_is_not_retired() {
+        let server = test_server(false, 4);
+        server.register_worker();
+        assert!(!server.slots.read()[0].retired.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    fn retire_then_reset_returns_a_slot_to_its_fresh_state() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        let slot = server.slots.read()[0].clone();
+        slot.parked.store(true, Ordering::Relaxed);
+        slot.interrupted.store(true, Ordering::Relaxed);
+        slot.served.store(5, Ordering::Relaxed);
+        slot.in_flight.store(1, Ordering::Relaxed);
+        slot.last_started_ms.store(9, Ordering::Relaxed);
+        slot.current.lock().push(std::sync::Weak::new());
+
+        slot.retire();
+        assert!(slot.retired.load(Ordering::Relaxed));
+
+        slot.reset();
+        assert!(!slot.retired.load(Ordering::Relaxed));
+        assert!(!slot.parked.load(Ordering::Relaxed));
+        assert!(!slot.interrupted.load(Ordering::Relaxed));
+        assert_eq!(slot.served.load(Ordering::Relaxed), 0);
+        assert_eq!(slot.in_flight.load(Ordering::Relaxed), 0);
+        assert_eq!(slot.last_started_ms.load(Ordering::Relaxed), 0);
+        assert!(slot.current.lock().is_empty());
+        // The lane survives reuse: the next occupant takes from it.
+        assert!(slot.lane_tx.lock().is_some());
+    }
+
+    #[test]
+    fn reset_keeps_a_quarantine_mark() {
+        let server = test_server(false, 4);
+        server.register_worker();
+        let slot = server.slots.read()[0].clone();
+        slot.quarantined.store(true, Ordering::Relaxed);
+
+        slot.retire();
+        slot.reset();
+
+        // A wedged slot stays flagged even if it ever came back around.
+        assert!(slot.quarantined.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    fn retire_releases_the_lane_lock() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        let slot = server.slots.read()[0].clone();
+
+        slot.retire();
+
+        assert!(slot.lane_tx.try_lock().is_some());
+    }
+
+    #[test]
+    fn retire_waits_for_a_dispatcher_holding_the_lane_lock() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        let slot = server.slots.read()[0].clone();
+
+        // A dispatcher that has already read "not retired" and is sending.
+        let sending = slot.lane_tx.lock();
+        let retiring = std::thread::spawn({
+            let slot = slot.clone();
+            move || slot.retire()
+        });
+        std::thread::sleep(Duration::from_millis(30));
+        assert!(
+            !slot.retired.load(Ordering::SeqCst),
+            "the flag must not go up under a dispatcher's lock"
+        );
+
+        drop(sending);
+        retiring.join().expect("retire thread");
+        assert!(slot.retired.load(Ordering::SeqCst));
+    }
+
     #[test]
     fn queue_histogram_buckets_by_wait() {
         let h = QueueHistogram::new();

data/ext/kino/src/request.rs

@@ -492,7 +492,7 @@ fn coerce_str(value: Value) -> Result<RString, Error> {
     }
 }
 
-fn split_host_port(host: &str, default_port: u16) -> (String, u16) {
+pub(crate) fn split_host_port(host: &str, default_port: u16) -> (String, u16) {
     match host.rsplit_once(':') {
         Some((name, port)) if !name.is_empty() => match port.parse() {
             Ok(p) => (name.to_string(), p),
@@ -518,7 +518,7 @@ pub fn test_ctx() -> crate::registry::BoxedCtx {
         body_rx: None,
         leftover: None,
         slot: None,
-        pin_slab: Arc::new(crate::pin::PinSlab::new()),
+        pin_slab: Arc::new(crate::pin::PinSlab::for_responses(false)),
         responder: Arc::new(Responder::new(head_tx)),
         enqueued_at: std::time::Instant::now(),
         timed: false,