kino 0.5.0 → 0.7.0

data/ext/kino/src/server.rs

@@ -9,7 +9,8 @@ use std::time::Duration;
 
 use http_body_util::BodyExt;
 use hyper::service::service_fn;
-use hyper_util::rt::TokioIo;
+use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer};
+use hyper_util::server::conn::auto;
 use magnus::{Error, Ruby};
 use parking_lot::{Mutex, RwLock};
 
@@ -62,14 +63,20 @@ pub fn server_start(ruby: &Ruby, config: magnus::RHash) -> Result<(u64, u16, Opt
     let tls_cert: Option<String> = cfg_opt(ruby, config, "tls_cert")?;
     let tls_key: Option<String> = cfg_opt(ruby, config, "tls_key")?;
     let lanes: bool = cfg_opt(ruby, config, "lanes")?.unwrap_or(false);
+    // Default true guards embedders calling the native layer directly.
+    let http2: bool = cfg_opt(ruby, config, "http2")?.unwrap_or(true);
     let log_requests: bool = cfg_opt(ruby, config, "log_requests")?.unwrap_or(false);
     let mode: String = cfg_opt(ruby, config, "mode")?.unwrap_or_else(|| "threaded".to_string());
     let workers: usize = cfg_opt(ruby, config, "workers")?.unwrap_or(0);
+    // Absent or below the floor means a fixed pool.
+    let max_workers: usize = cfg_opt::<usize>(ruby, config, "max_workers")?
+        .unwrap_or(workers)
+        .max(workers);
     let threads: usize = cfg_opt(ruby, config, "threads")?.unwrap_or(0);
     let batch: usize = cfg_opt(ruby, config, "batch")?.unwrap_or(1);
     let acceptor = match (&tls_cert, &tls_key) {
         (Some(cert), Some(key)) => Some(
-            crate::tls::build_acceptor(cert, key)
+            crate::tls::build_acceptor(cert, key, http2)
                 .map_err(|e| Error::new(ruby.exception_runtime_error(), format!("TLS: {e}")))?,
         ),
         (None, None) => None,
@@ -127,13 +134,18 @@ pub fn server_start(ruby: &Ruby, config: magnus::RHash) -> Result<(u64, u16, Opt
         state: std::sync::atomic::AtomicU8::new(registry::STATE_BOOTING),
         respawns: std::sync::atomic::AtomicU64::new(0),
         quarantine_replacements: std::sync::atomic::AtomicU64::new(0),
+        active_workers: std::sync::atomic::AtomicUsize::new(workers),
+        scale_ups: std::sync::atomic::AtomicU64::new(0),
+        scale_downs: std::sync::atomic::AtomicU64::new(0),
         topology: registry::Topology {
             mode,
             workers,
+            max_workers,
             threads,
             batch,
         },
         https: acceptor.is_some(),
+        http2,
         unix_path,
         access_log: log_requests.then(|| crate::logsink::Sink::new(std::io::stdout())),
         lanes,
@@ -340,9 +352,77 @@ pub(crate) async fn serve_conn(
 /// headers within this window. Long enough never to trip a real client (even
 /// on a slow mobile link), short enough to reap a stalled one. Deliberately a
 /// constant, not a config knob: fine-tuning intake limits is the fronting
-/// proxy's job; the actual hazard was having no default at all.
+/// proxy's job; the actual hazard was having no default at all. Guards the
+/// HTTP/1 side only: h2 intake is bounded by the TLS-handshake timeout and
+/// the h2 codec's own SETTINGS handling.
 const HEADER_READ_TIMEOUT: Duration = Duration::from_secs(15);
 
+/// The connection builder every data-plane connection is served with,
+/// in both accept topologies. `http2: true` is the protocol-auto shape:
+/// ALPN-negotiated h2 over TLS, prior-knowledge h2c on plaintext (the
+/// builder sniffs the 24-byte preface once per connection), HTTP/1.x
+/// for everything else. `http2: false` pins the HTTP/1 codec: no sniff,
+/// same wire behavior as a server built without h2.
+///
+/// No auto Date header on either protocol: it costs a clock read per
+/// response (together with timer reads, ~7% of tokio-side cycles in the
+/// profile); it's a SHOULD not a MUST, and apps that need it can set it
+/// themselves.
+///
+/// The http1 timer is installed so header_read_timeout actually fires:
+/// hyper's slow-header guard is inert without one. It arms only while
+/// the request head is being read, so it adds no per-response cost on
+/// the hot path. The h2 side gets a timer too so its own timed
+/// machinery (keep-alive, shutdown deadlines) can fire if ever enabled.
+/// SETTINGS_MAX_CONCURRENT_STREAMS, derived from worker-slot capacity
+/// (workers × threads) instead of hyper's flat 200. Two jobs: an
+/// h2-aware balancer sees the server's real admission and spreads
+/// streams across upstream connections instead of queueing blind, and a
+/// hostile client can't multiply one connection into 200 queued
+/// requests. The floor keeps tiny topologies browser-friendly (one
+/// page's fetches still parallelize; excess streams just queue), the
+/// cap bounds per-connection bookkeeping, and an unknown topology (an
+/// embedder passing zeros) keeps hyper's default.
+fn advertised_streams(workers: usize, threads: usize) -> u32 {
+    let slots = workers.saturating_mul(threads);
+    if slots == 0 {
+        return 200;
+    }
+    slots.clamp(8, 1024) as u32
+}
+
+/// The pool can grow to its ceiling to meet what an h2 balancer sends, so
+/// the ceiling is the admission to advertise (the floor for a fixed pool).
+fn stream_capacity(topology: &registry::Topology) -> u32 {
+    advertised_streams(topology.max_workers, topology.threads)
+}
+
+fn conn_builder(http2: bool, max_streams: u32) -> auto::Builder<TokioExecutor> {
+    let mut builder = auto::Builder::new(TokioExecutor::new());
+    builder
+        .http1()
+        .timer(TokioTimer::new())
+        .header_read_timeout(HEADER_READ_TIMEOUT)
+        .auto_date_header(false);
+    // Beyond the stream cap, hyper's h2 defaults (16 KB frames, 1 MB
+    // windows) stay: a knob sweep (frame size 64K/256K, adaptive
+    // windows, 4/8 MB windows) moved the upload lane nowhere or slightly
+    // down once read_body coalesced its channel drain: the crossings
+    // were the cost, not the codec. The codec's abuse bounds also ship
+    // as defaults: 16 KB header lists, 20 pending remote resets then
+    // GOAWAY (rapid reset), reset-churn and empty-frame budgets.
+    builder
+        .http2()
+        .timer(TokioTimer::new())
+        .auto_date_header(false)
+        .max_concurrent_streams(max_streams);
+    if http2 {
+        builder
+    } else {
+        builder.http1_only()
+    }
+}
+
 async fn serve_connection<I>(
     io: I,
     server: Arc<ServerInner>,
@@ -351,21 +431,38 @@ async fn serve_connection<I>(
 ) where
     I: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
 {
+    let http2 = server.http2;
+    let max_streams = stream_capacity(&server.topology);
+    let mut drain = server.shutdown_tx.subscribe();
     let service =
         service_fn(move |req| handle_request(server.clone(), remote_addr, local_addr, req));
-    // No auto Date header: it costs a clock read per response (together
-    // with timer reads, ~7% of tokio-side cycles in the profile); it's a
-    // SHOULD not a MUST, and apps that need it can set it themselves.
-    //
-    // The timer is installed so header_read_timeout actually fires: hyper's
-    // slow-header guard is inert without one. It arms only while the request
-    // head is being read, so it adds no per-response cost on the hot path.
-    let _ = hyper::server::conn::http1::Builder::new()
-        .timer(hyper_util::rt::TokioTimer::new())
-        .header_read_timeout(HEADER_READ_TIMEOUT)
-        .auto_date_header(false)
-        .serve_connection(TokioIo::new(io), service)
-        .await;
+    let builder = conn_builder(http2, max_streams);
+    let conn = builder.serve_connection(TokioIo::new(io), service);
+    let mut conn = std::pin::pin!(conn);
+    // Serve until done, or switch to graceful shutdown when the drain
+    // signal fires (stop_accepting): the in-flight request finishes and
+    // the connection then closes (`Connection: close` on HTTP/1,
+    // GOAWAY on h2), so a balancer moves on instead of feeding a
+    // draining server, and teardown never cuts a response mid-stream.
+    // A connection that outlives the drain deadline is still cut by the
+    // runtime teardown, as before.
+    tokio::select! {
+        _ = conn.as_mut() => {}
+        _ = drain_signal(&mut drain) => {
+            conn.as_mut().graceful_shutdown();
+            let _ = conn.as_mut().await;
+        }
+    }
+}
+
+/// Resolve when the server starts draining. A connection accepted just
+/// before the signal may subscribe just after it, and `changed()` alone
+/// would miss that edge, so the current value is checked first; a
+/// dropped sender (teardown) counts as draining too.
+async fn drain_signal(rx: &mut tokio::sync::watch::Receiver<bool>) {
+    if !*rx.borrow_and_update() {
+        let _ = rx.changed().await;
+    }
 }
 
 /// The 503 every rejection path returns; counted for stats. Branding
@@ -437,18 +534,18 @@ async fn handle_request(
     let oversize =
         max_body > 0 && content_length(&parts.headers).is_some_and(|len| len > max_body as u64);
 
+    let (head_tx, head_rx) = tokio::sync::oneshot::channel();
+    let responder = Arc::new(Responder::new(head_tx));
+
     // Stream the request body through a bounded channel: hyper is polled
     // only as fast as the Ruby side consumes (inbound backpressure), and the
     // forwarder dropping the sender is EOF. Bodyless requests (most GETs)
-    // skip the forwarder task entirely: dropping the sender IS the EOF.
-    let (body_tx, body_rx) = flume::bounded::<bytes::Bytes>(8);
-    let body_overflow = Arc::new(std::sync::atomic::AtomicBool::new(false));
-    let body_timeout = Arc::new(std::sync::atomic::AtomicBool::new(false));
-    if oversize || hyper::body::Body::is_end_stream(&body) {
-        drop(body_tx);
+    // skip the channel and the forwarder task entirely: no channel IS the EOF.
+    let body_rx = if oversize || hyper::body::Body::is_end_stream(&body) {
+        None
     } else {
-        let overflow = body_overflow.clone();
-        let timed_out = body_timeout.clone();
+        let (body_tx, body_rx) = flume::bounded::<bytes::Bytes>(8);
+        let responder = responder.clone();
         tokio::spawn(async move {
             let mut body = body;
             let mut total: u64 = 0;
@@ -461,16 +558,19 @@ async fn handle_request(
                     Ok(Some(Ok(frame))) => frame,
                     Ok(Some(Err(_))) | Ok(None) => break, // body error or clean EOF
                     Err(_) => {
-                        timed_out.store(true, Ordering::Relaxed);
+                        responder.abandon_body(crate::response::BodyAbandon::TimedOut);
                         break;
                     }
                 };
+                // Non-data frames (h2 trailers) are dropped by design:
+                // Rack has no trailer surface. DATA frames around them
+                // still forward, and hyper reports EOF right after.
                 if let Ok(data) = frame.into_data() {
                     total += data.len() as u64;
                     if max_body > 0 && total > max_body as u64 {
                         // Past the cap: flag it and stop pulling. Dropping the
                         // sender unblocks read_body, which then raises.
-                        overflow.store(true, Ordering::Relaxed);
+                        responder.abandon_body(crate::response::BodyAbandon::Oversize);
                         break;
                     }
                     if body_tx.send_async(data).await.is_err() {
@@ -479,10 +579,9 @@ async fn handle_request(
                 }
             }
         });
-    }
+        Some(body_rx)
+    };
 
-    let (head_tx, head_rx) = tokio::sync::oneshot::channel();
-    let responder = Arc::new(Responder::new(head_tx));
     let now = std::time::Instant::now();
     let ctx = Box::new(RequestCtx {
         method: parts.method,
@@ -493,8 +592,6 @@ async fn handle_request(
         local_addr,
         https: server.https,
         body_rx,
-        body_overflow,
-        body_timeout,
         leftover: None,
         slot: None,
         pin_slab: server.pin_slab.clone(),
@@ -629,6 +726,11 @@ fn try_dispatch(server: &ServerInner, mut ctx: BoxedCtx) -> Dispatch {
             let guard = slot.lane_tx.lock();
             let Some(tx) = guard.as_ref() else { continue };
             any_open = true;
+            // Checked under the lane lock, which WorkerSlot::retire also
+            // takes: see there for the ordering.
+            if slot.retired.load(Ordering::SeqCst) {
+                continue;
+            }
             match tx.try_send(ctx) {
                 Ok(()) => return Dispatch::Sent,
                 Err(flume::TrySendError::Full(c)) | Err(flume::TrySendError::Disconnected(c)) => {
@@ -817,7 +919,7 @@ pub fn queue_time(_ruby: &Ruby, server_id: u64) -> Result<(u64, f64), Error> {
 }
 
 /// One worker slot's [index, served, in_flight, busy_ms, quarantined] row.
-pub type WorkerStatRow = (usize, u64, usize, u64, bool);
+pub type WorkerStatRow = (usize, u64, usize, u64, bool, bool);
 
 /// Per-slot rows for Server#stats parity: [index, served, in_flight,
 /// busy_ms, quarantined] each. Empty when the server is gone.
@@ -827,7 +929,16 @@ pub fn worker_stats(_ruby: &Ruby, server_id: u64) -> Result<Vec<WorkerStatRow>,
     };
     Ok(crate::control::collect_worker_status(&server)
         .into_iter()
-        .map(|w| (w.index, w.served, w.in_flight, w.busy_ms, w.quarantined))
+        .map(|w| {
+            (
+                w.index,
+                w.served,
+                w.in_flight,
+                w.busy_ms,
+                w.quarantined,
+                w.retired,
+            )
+        })
         .collect())
 }
 
@@ -840,6 +951,61 @@ pub fn quarantine_slot(ruby: &Ruby, server_id: u64, worker_id: usize) -> Result<
     Ok(())
 }
 
+/// Send a slot's worker home once idle (pool scale-down); see
+/// registry::WorkerSlot::retire. An unknown id is a caller bug: raise.
+pub fn retire_slot(ruby: &Ruby, server_id: u64, worker_id: usize) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        server.slot(ruby, worker_id)?.retire();
+    }
+    Ok(())
+}
+
+/// Hand a retired slot to a new worker (pool scale-up reusing a slot).
+pub fn reset_slot(ruby: &Ruby, server_id: u64, worker_id: usize) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        server.slot(ruby, worker_id)?.reset();
+    }
+    Ok(())
+}
+
+/// The Ruby pool reports how many workers are alive and serving.
+pub fn set_active_workers(_ruby: &Ruby, server_id: u64, count: usize) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        server.active_workers.store(count, Ordering::Relaxed);
+    }
+    Ok(())
+}
+
+/// One worker added by the pool scaler.
+pub fn record_scale_up(_ruby: &Ruby, server_id: u64) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        server.scale_ups.fetch_add(1, Ordering::Relaxed);
+    }
+    Ok(())
+}
+
+/// One worker retired by the pool scaler.
+pub fn record_scale_down(_ruby: &Ruby, server_id: u64) -> Result<(), Error> {
+    if let Some(server) = registry::try_get(server_id) {
+        server.scale_downs.fetch_add(1, Ordering::Relaxed);
+    }
+    Ok(())
+}
+
+/// [active_workers, max_workers, scale_ups, scale_downs] for Server#stats.
+/// Zeros when the server is gone.
+pub fn pool_stats(_ruby: &Ruby, server_id: u64) -> Result<(usize, usize, u64, u64), Error> {
+    let Some(server) = registry::try_get(server_id) else {
+        return Ok((0, 0, 0, 0));
+    };
+    Ok((
+        server.active_workers.load(Ordering::Relaxed),
+        server.topology.max_workers,
+        server.scale_ups.load(Ordering::Relaxed),
+        server.scale_downs.load(Ordering::Relaxed),
+    ))
+}
+
 /// One replacement spawned by the quarantine monitor.
 pub fn record_quarantine_replacement(_ruby: &Ruby, server_id: u64) -> Result<(), Error> {
     if let Some(server) = registry::try_get(server_id) {
@@ -856,6 +1022,565 @@ mod tests {
     use crate::registry::{test_server, LANE_DEPTH};
     use crate::request::test_ctx;
 
+    /// Serve one real connection through the production `serve_connection`
+    /// over an in-memory pipe, against a queue the test consumes itself (a
+    /// stand-in for the Ruby worker). Returns the client end.
+    fn spawn_conn(server: Arc<ServerInner>) -> tokio::io::DuplexStream {
+        let (client_io, server_io) = tokio::io::duplex(64 * 1024);
+        tokio::spawn(serve_connection(
+            server_io,
+            server,
+            "127.0.0.1:40000".parse().expect("static addr"),
+            "127.0.0.1:9292".parse().expect("static addr"),
+        ));
+        client_io
+    }
+
+    async fn take_ctx(server: &Arc<ServerInner>) -> BoxedCtx {
+        server.req_rx.recv_async().await.expect("a queued request")
+    }
+
+    /// Read the request body the way a worker does: leftover first, then
+    /// the forwarder channel until the sender drops (EOF).
+    async fn drain_body(ctx: &mut RequestCtx) -> Vec<u8> {
+        let mut out = Vec::new();
+        if let Some(leftover) = ctx.leftover.take() {
+            out.extend_from_slice(&leftover);
+        }
+        if let Some(rx) = &ctx.body_rx {
+            while let Ok(chunk) = rx.recv_async().await {
+                out.extend_from_slice(&chunk);
+            }
+        }
+        out
+    }
+
+    #[tokio::test]
+    async fn h2c_prior_knowledge_reaches_the_queue_as_http2() {
+        use http_body_util::{BodyExt, Full};
+
+        let server = test_server(false, 4);
+        let client_io = spawn_conn(server.clone());
+
+        let worker = tokio::spawn(async move {
+            let ctx = take_ctx(&server).await;
+            assert_eq!(ctx.version, http::Version::HTTP_2);
+            // The :authority pseudo-header arrives in the URI, where
+            // build_env picks it up; no Host header exists on h2.
+            assert_eq!(
+                ctx.uri.authority().map(|a| a.as_str()),
+                Some("kino.test:8443")
+            );
+            assert!(ctx.headers.get(http::header::HOST).is_none());
+            assert!(ctx.responder.send_response(plain_response(200, "ok\n")));
+        });
+
+        let (mut sender, conn) = hyper::client::conn::http2::handshake(
+            hyper_util::rt::TokioExecutor::new(),
+            TokioIo::new(client_io),
+        )
+        .await
+        .expect("h2c prior-knowledge handshake");
+        tokio::spawn(conn);
+        let request = hyper::Request::builder()
+            .uri("http://kino.test:8443/")
+            .body(Full::new(bytes::Bytes::new()))
+            .expect("request");
+        let response = sender.send_request(request).await.expect("h2 response");
+        assert_eq!(response.status(), 200);
+        assert_eq!(
+            response.headers().get("server").expect("branded"),
+            "Kino",
+            "responses stay branded over h2"
+        );
+        let body = response.into_body().collect().await.expect("body");
+        assert_eq!(&body.to_bytes()[..], b"ok\n");
+        worker.await.expect("worker assertions");
+    }
+
+    #[tokio::test]
+    async fn h1_is_still_served_by_the_auto_builder() {
+        use http_body_util::{BodyExt, Full};
+
+        let server = test_server(false, 4);
+        let client_io = spawn_conn(server.clone());
+
+        let worker = tokio::spawn(async move {
+            let ctx = take_ctx(&server).await;
+            assert_eq!(ctx.version, http::Version::HTTP_11);
+            assert!(ctx.responder.send_response(plain_response(200, "h1\n")));
+        });
+
+        let (mut sender, conn) = hyper::client::conn::http1::handshake(TokioIo::new(client_io))
+            .await
+            .expect("h1 handshake");
+        tokio::spawn(conn);
+        let request = hyper::Request::builder()
+            .uri("/")
+            .header("host", "kino.test")
+            .body(Full::new(bytes::Bytes::new()))
+            .expect("request");
+        let response = sender.send_request(request).await.expect("h1 response");
+        assert_eq!(response.status(), 200);
+        let body = response.into_body().collect().await.expect("body");
+        assert_eq!(&body.to_bytes()[..], b"h1\n");
+        worker.await.expect("worker assertions");
+    }
+
+    #[tokio::test]
+    async fn http2_off_refuses_the_preface_but_serves_h1() {
+        use http_body_util::{BodyExt, Full};
+        use hyper::service::service_fn;
+
+        // Driven at the builder seam: conn_builder(false) is what a
+        // ServerInner with http2 off serves every connection with.
+        let stub = || {
+            service_fn(|_req: hyper::Request<hyper::body::Incoming>| async {
+                Ok::<_, std::convert::Infallible>(plain_response(200, "pinned\n"))
+            })
+        };
+
+        // An h2 prior-knowledge client must fail: the pinned h1 codec
+        // reads the preface as a malformed request line.
+        let (client_io, server_io) = tokio::io::duplex(64 * 1024);
+        tokio::spawn(async move {
+            let _ = conn_builder(false, 200)
+                .serve_connection(TokioIo::new(server_io), stub())
+                .await;
+        });
+        let refused = async {
+            let (mut sender, conn) = hyper::client::conn::http2::handshake(
+                hyper_util::rt::TokioExecutor::new(),
+                TokioIo::new(client_io),
+            )
+            .await?;
+            tokio::spawn(conn);
+            let request = hyper::Request::builder()
+                .uri("http://kino.test/")
+                .body(Full::new(bytes::Bytes::new()))?;
+            sender.send_request(request).await?;
+            Ok::<_, Box<dyn std::error::Error>>(())
+        }
+        .await;
+        assert!(refused.is_err(), "h2 must not be served when pinned to h1");
+
+        // The same pinned builder serves a plain h1 client.
+        let (client_io, server_io) = tokio::io::duplex(64 * 1024);
+        tokio::spawn(async move {
+            let _ = conn_builder(false, 200)
+                .serve_connection(TokioIo::new(server_io), stub())
+                .await;
+        });
+        let (mut sender, conn) = hyper::client::conn::http1::handshake(TokioIo::new(client_io))
+            .await
+            .expect("h1 handshake");
+        tokio::spawn(conn);
+        let request = hyper::Request::builder()
+            .uri("/")
+            .header("host", "kino.test")
+            .body(Full::new(bytes::Bytes::new()))
+            .expect("request");
+        let response = sender.send_request(request).await.expect("h1 response");
+        assert_eq!(response.status(), 200);
+        let body = response.into_body().collect().await.expect("body");
+        assert_eq!(&body.to_bytes()[..], b"pinned\n");
+    }
+
+    #[tokio::test]
+    async fn h2_upload_forwards_data_and_drops_trailers() {
+        use http_body_util::{BodyExt, StreamBody};
+        use hyper::body::Frame;
+
+        let server = test_server(false, 4);
+        let client_io = spawn_conn(server.clone());
+
+        let worker = tokio::spawn(async move {
+            let mut ctx = take_ctx(&server).await;
+            assert_eq!(ctx.version, http::Version::HTTP_2);
+            let body = drain_body(&mut ctx).await;
+            assert_eq!(&body[..], b"hello world");
+            assert!(
+                ctx.responder.body_abandoned().is_none(),
+                "a trailer frame must not abort the body read"
+            );
+            let response = hyper::Response::builder()
+                .status(200)
+                .body(crate::response::full_body(bytes::Bytes::from(
+                    body.len().to_string(),
+                )))
+                .expect("response");
+            assert!(ctx.responder.send_response(response));
+        });
+
+        let (frames_tx, frames_rx) =
+            flume::bounded::<Result<Frame<bytes::Bytes>, std::io::Error>>(4);
+        frames_tx
+            .send(Ok(Frame::data(bytes::Bytes::from_static(b"hello "))))
+            .expect("frame");
+        frames_tx
+            .send(Ok(Frame::data(bytes::Bytes::from_static(b"world"))))
+            .expect("frame");
+        let mut trailers = http::HeaderMap::new();
+        trailers.insert("x-checksum", "ignored".parse().expect("value"));
+        frames_tx
+            .send(Ok(Frame::trailers(trailers)))
+            .expect("frame");
+        drop(frames_tx); // EOS
+
+        let (mut sender, conn) = hyper::client::conn::http2::handshake(
+            hyper_util::rt::TokioExecutor::new(),
+            TokioIo::new(client_io),
+        )
+        .await
+        .expect("h2 handshake");
+        tokio::spawn(conn);
+        let request = hyper::Request::builder()
+            .method("POST")
+            .uri("http://kino.test/upload")
+            .body(StreamBody::new(frames_rx.into_stream()))
+            .expect("request");
+        let response = sender.send_request(request).await.expect("h2 response");
+        assert_eq!(response.status(), 200);
+        let body = response.into_body().collect().await.expect("body");
+        assert_eq!(&body.to_bytes()[..], b"11");
+        worker.await.expect("worker assertions");
+    }
+
+    #[tokio::test]
+    async fn h2_streaming_response_arrives_chunked() {
+        use http_body_util::{BodyExt, Full};
+
+        let server = test_server(false, 4);
+        let client_io = spawn_conn(server.clone());
+
+        let worker = tokio::spawn(async move {
+            let ctx = take_ctx(&server).await;
+            let started = ctx
+                .responder
+                .send_stream_head(hyper::Response::builder().status(200))
+                .expect("valid head");
+            assert!(started);
+            let frames = ctx.responder.body_sender().expect("open stream");
+            for chunk in [&b"alpha "[..], &b"beta "[..], &b"gamma"[..]] {
+                frames
+                    .send_async(Ok(hyper::body::Frame::data(bytes::Bytes::from_static(
+                        chunk,
+                    ))))
+                    .await
+                    .expect("chunk accepted");
+            }
+            ctx.responder.finish_stream();
+        });
+
+        let (mut sender, conn) = hyper::client::conn::http2::handshake(
+            hyper_util::rt::TokioExecutor::new(),
+            TokioIo::new(client_io),
+        )
+        .await
+        .expect("h2 handshake");
+        tokio::spawn(conn);
+        let request = hyper::Request::builder()
+            .uri("http://kino.test/stream")
+            .body(Full::new(bytes::Bytes::new()))
+            .expect("request");
+        let response = sender.send_request(request).await.expect("h2 response");
+        assert_eq!(response.status(), 200);
+        let body = response.into_body().collect().await.expect("body");
+        assert_eq!(&body.to_bytes()[..], b"alpha beta gamma");
+        worker.await.expect("worker assertions");
+    }
+
+    #[tokio::test]
+    async fn h2_multiplexes_concurrent_streams_into_the_queue() {
+        use http_body_util::{BodyExt, Full};
+
+        let server = test_server(false, 4);
+        let client_io = spawn_conn(server.clone());
+
+        // Both streams must be queued before either is answered: that is
+        // multiplexing observable at the worker boundary; and answering
+        // them in reverse order proves stream completion is not FIFO.
+        let worker = tokio::spawn(async move {
+            let first = take_ctx(&server).await;
+            let second = take_ctx(&server).await;
+            let order = [second.uri.path().to_string(), first.uri.path().to_string()];
+            assert!(second.responder.send_response(plain_response(200, "two\n")));
+            assert!(first.responder.send_response(plain_response(200, "one\n")));
+            order
+        });
+
+        let (mut sender, conn) = hyper::client::conn::http2::handshake(
+            hyper_util::rt::TokioExecutor::new(),
+            TokioIo::new(client_io),
+        )
+        .await
+        .expect("h2 handshake");
+        tokio::spawn(conn);
+        let req = |path: &str| {
+            hyper::Request::builder()
+                .uri(format!("http://kino.test{path}"))
+                .body(Full::new(bytes::Bytes::new()))
+                .expect("request")
+        };
+        let (one, two) = tokio::join!(
+            sender.send_request(req("/one")),
+            sender.send_request(req("/two"))
+        );
+        let one = one.expect("first stream");
+        let two = two.expect("second stream");
+        assert_eq!(one.status(), 200);
+        assert_eq!(two.status(), 200);
+        let one = one.into_body().collect().await.expect("body").to_bytes();
+        let two = two.into_body().collect().await.expect("body").to_bytes();
+        assert_eq!(&one[..], b"one\n");
+        assert_eq!(&two[..], b"two\n");
+        let order = worker.await.expect("worker assertions");
+        assert_eq!(order, ["/two".to_string(), "/one".to_string()]);
+    }
+
+    #[tokio::test]
+    async fn drain_finishes_the_in_flight_h2_stream_then_goaways() {
+        use http_body_util::{BodyExt, Full};
+
+        let server = test_server(false, 4);
+        let client_io = spawn_conn(server.clone());
+
+        let (sender, conn) = hyper::client::conn::http2::handshake(
+            hyper_util::rt::TokioExecutor::new(),
+            TokioIo::new(client_io),
+        )
+        .await
+        .expect("h2 handshake");
+        tokio::spawn(conn);
+
+        let mut in_flight_sender = sender.clone();
+        let in_flight = tokio::spawn(async move {
+            let request = hyper::Request::builder()
+                .uri("http://kino.test/inflight")
+                .body(Full::new(bytes::Bytes::new()))
+                .expect("request");
+            in_flight_sender.send_request(request).await
+        });
+
+        // The request is with the worker when the drain fires; its
+        // response must still reach the client through the shutdown.
+        let ctx = take_ctx(&server).await;
+        let _ = server.shutdown_tx.send(true);
+        tokio::time::sleep(Duration::from_millis(20)).await;
+        assert!(ctx.responder.send_response(plain_response(200, "late\n")));
+
+        let response = in_flight.await.expect("join").expect("in-flight served");
+        assert_eq!(response.status(), 200);
+        let body = response.into_body().collect().await.expect("body");
+        assert_eq!(&body.to_bytes()[..], b"late\n");
+
+        // The connection is now GOAWAY'd: a new stream must fail.
+        let mut sender = sender;
+        let request = hyper::Request::builder()
+            .uri("http://kino.test/after")
+            .body(Full::new(bytes::Bytes::new()))
+            .expect("request");
+        assert!(
+            sender.send_request(request).await.is_err(),
+            "a drained connection must not accept new streams"
+        );
+    }
+
+    #[tokio::test]
+    async fn drain_closes_an_h1_connection_after_its_response() {
+        use http_body_util::Full;
+
+        let server = test_server(false, 4);
+        let client_io = spawn_conn(server.clone());
+
+        let (mut sender, conn) = hyper::client::conn::http1::handshake(TokioIo::new(client_io))
+            .await
+            .expect("h1 handshake");
+        tokio::spawn(conn);
+
+        // h1's SendRequest is not Clone: one task drives both requests;
+        // the in-flight one, then (once it completed, so the drain has
+        // been seen) the keep-alive follow-up that must be refused.
+        let req = |path: &str| {
+            hyper::Request::builder()
+                .uri(path.to_string())
+                .header("host", "kino.test")
+                .body(Full::new(bytes::Bytes::new()))
+                .expect("request")
+        };
+        let client = tokio::spawn(async move {
+            let first = sender.send_request(req("/inflight")).await;
+            let second_failed = sender.send_request(req("/after")).await.is_err();
+            (first, second_failed)
+        });
+
+        let ctx = take_ctx(&server).await;
+        let _ = server.shutdown_tx.send(true);
+        tokio::time::sleep(Duration::from_millis(20)).await;
+        assert!(ctx.responder.send_response(plain_response(200, "late\n")));
+
+        let (first, second_failed) = client.await.expect("join");
+        assert_eq!(first.expect("in-flight served").status(), 200);
+        assert!(
+            second_failed,
+            "a drained keep-alive connection must close after its response"
+        );
+    }
+
+    #[test]
+    fn advertised_streams_tracks_slot_capacity() {
+        // Slot capacity, floored for tiny topologies and capped.
+        assert_eq!(advertised_streams(8, 3), 24);
+        assert_eq!(advertised_streams(2, 1), 8, "floor");
+        assert_eq!(advertised_streams(64, 32), 1024, "cap");
+        // Unknown topology (embedder passing zeros): hyper's default.
+        assert_eq!(advertised_streams(0, 0), 200);
+        assert_eq!(advertised_streams(8, 0), 200);
+    }
+
+    #[test]
+    fn stream_capacity_is_advertised_from_the_pool_ceiling() {
+        // An elastic pool grows to meet load an h2 balancer sends, so the
+        // ceiling, not the floor, is the admission to advertise.
+        let topology = registry::Topology {
+            mode: "ractor".to_string(),
+            workers: 2,
+            max_workers: 8,
+            threads: 3,
+            batch: 1,
+        };
+        assert_eq!(stream_capacity(&topology), 24);
+    }
+
+    #[test]
+    fn stream_capacity_of_a_fixed_pool_is_its_slot_count() {
+        let topology = registry::Topology {
+            mode: "threaded".to_string(),
+            workers: 4,
+            max_workers: 4,
+            threads: 3,
+            batch: 1,
+        };
+        assert_eq!(stream_capacity(&topology), 12);
+    }
+
+    #[tokio::test]
+    async fn streams_beyond_the_advertised_cap_queue_instead_of_failing() {
+        use http_body_util::Full;
+        use hyper::service::service_fn;
+
+        // Cap of 2: six concurrent requests must all complete; the h2
+        // client holds excess streams locally until the server frees a
+        // slot; nothing is refused or reset.
+        let (client_io, server_io) = tokio::io::duplex(64 * 1024);
+        tokio::spawn(async move {
+            let stub = service_fn(|_req: hyper::Request<hyper::body::Incoming>| async {
+                Ok::<_, std::convert::Infallible>(plain_response(200, "capped\n"))
+            });
+            let _ = conn_builder(true, 2)
+                .serve_connection(TokioIo::new(server_io), stub)
+                .await;
+        });
+        let (mut sender, conn) = hyper::client::conn::http2::handshake(
+            hyper_util::rt::TokioExecutor::new(),
+            TokioIo::new(client_io),
+        )
+        .await
+        .expect("h2 handshake");
+        tokio::spawn(conn);
+        let mut requests = Vec::new();
+        for i in 0..6 {
+            let request = hyper::Request::builder()
+                .uri(format!("http://kino.test/{i}"))
+                .body(Full::new(bytes::Bytes::new()))
+                .expect("request");
+            requests.push(sender.send_request(request));
+        }
+        for request in requests {
+            let response = request.await.expect("queued stream served");
+            assert_eq!(response.status(), 200);
+        }
+    }
+
+    #[tokio::test]
+    async fn rapid_stream_resets_do_not_wedge_the_pipeline() {
+        use http_body_util::{BodyExt, Full};
+
+        let server = test_server(false, 64);
+        let client_io = spawn_conn(server.clone());
+
+        // A worker that answers everything it sees until told to stop;
+        // answers to already-reset streams just vanish, as in production.
+        let worker = tokio::spawn(async move {
+            loop {
+                let Ok(ctx) = server.req_rx.recv_async().await else {
+                    break;
+                };
+                let done = ctx.uri.path() == "/done";
+                ctx.responder.send_response(plain_response(200, "ok\n"));
+                if done {
+                    break;
+                }
+            }
+        });
+
+        let (mut sender, conn) = hyper::client::conn::http2::handshake(
+            hyper_util::rt::TokioExecutor::new(),
+            TokioIo::new(client_io),
+        )
+        .await
+        .expect("h2 handshake");
+        tokio::spawn(conn);
+
+        // Fire-and-cancel: dropping the response future resets the
+        // stream. The codec bounds reset churn; the server must keep
+        // serving afterwards.
+        for i in 0..40 {
+            let request = hyper::Request::builder()
+                .uri(format!("http://kino.test/cancel/{i}"))
+                .body(Full::new(bytes::Bytes::new()))
+                .expect("request");
+            drop(sender.send_request(request));
+        }
+        let request = hyper::Request::builder()
+            .uri("http://kino.test/done")
+            .body(Full::new(bytes::Bytes::new()))
+            .expect("request");
+        let response = sender.send_request(request).await.expect("still served");
+        assert_eq!(response.status(), 200);
+        let body = response.into_body().collect().await.expect("body");
+        assert_eq!(&body.to_bytes()[..], b"ok\n");
+        worker.await.expect("worker loop");
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn header_read_timeout_still_fires_through_the_auto_builder() {
+        use hyper::service::service_fn;
+        use tokio::io::AsyncWriteExt;
+
+        let (mut client_io, server_io) = tokio::io::duplex(64 * 1024);
+        let conn = tokio::spawn(async move {
+            let _ = conn_builder(true, 200)
+                .serve_connection(
+                    TokioIo::new(server_io),
+                    service_fn(|_req: hyper::Request<hyper::body::Incoming>| async {
+                        Ok::<_, std::convert::Infallible>(plain_response(200, "never\n"))
+                    }),
+                )
+                .await;
+        });
+        // A partial h1 request line, then silence: the slow-header guard
+        // must reap the connection (paused clock auto-advances past 15s).
+        client_io
+            .write_all(b"GET / HT")
+            .await
+            .expect("partial write");
+        tokio::time::timeout(Duration::from_secs(60), conn)
+            .await
+            .expect("connection reaped by header_read_timeout")
+            .expect("serve task join");
+    }
+
     #[test]
     fn dispatch_with_no_slots_reports_full() {
         let server = test_server(true, 4);
@@ -950,4 +1675,110 @@ mod tests {
         assert!(matches!(try_dispatch(&server, test_ctx()), Dispatch::Sent));
         assert_eq!(server.lane_depths(), Some(vec![0, 2]));
     }
+
+    #[test]
+    fn dispatch_skips_retired_lanes() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        server.register_worker();
+        server.slots.read()[0].retire();
+
+        // A retiring worker gets nothing new: both land on slot 1.
+        assert!(matches!(try_dispatch(&server, test_ctx()), Dispatch::Sent));
+        assert!(matches!(try_dispatch(&server, test_ctx()), Dispatch::Sent));
+        assert_eq!(server.lane_depths(), Some(vec![0, 2]));
+    }
+
+    #[test]
+    fn dispatch_retries_rather_than_closing_when_only_retired_lanes_remain() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        server.slots.read()[0].retire();
+
+        // Retired is not draining: the caller keeps retrying until the
+        // queue timeout, so a replacement worker can still pick this up.
+        assert!(matches!(
+            try_dispatch(&server, test_ctx()),
+            Dispatch::Full(_)
+        ));
+    }
+
+    #[test]
+    fn dispatch_skips_a_retired_lane_even_when_it_is_the_only_awake_one() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        server.register_worker();
+        server.slots.read()[0].retire();
+        server.slots.read()[1].parked.store(true, Ordering::Relaxed);
+
+        // The second pass (parked lanes allowed) still skips the retired one.
+        assert!(matches!(try_dispatch(&server, test_ctx()), Dispatch::Sent));
+        assert_eq!(server.lane_depths(), Some(vec![0, 1]));
+    }
+
+    #[test]
+    fn dispatch_keeps_retrying_when_the_other_lanes_are_closed() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        server.register_worker();
+        server.slots.read()[0].lane_tx.lock().take(); // a crashed worker's lane
+        server.slots.read()[1].retire();
+
+        assert!(matches!(
+            try_dispatch(&server, test_ctx()),
+            Dispatch::Full(_)
+        ));
+    }
+
+    #[test]
+    fn a_reset_lane_receives_dispatches_again() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        server.register_worker();
+        server.slots.read()[0].retire();
+        assert!(matches!(try_dispatch(&server, test_ctx()), Dispatch::Sent));
+        assert_eq!(server.lane_depths(), Some(vec![0, 1]));
+
+        server.slots.read()[0].reset();
+        for _ in 0..2 {
+            assert!(matches!(try_dispatch(&server, test_ctx()), Dispatch::Sent));
+        }
+        // Back in the round-robin.
+        assert_eq!(server.lane_depths(), Some(vec![1, 2]));
+    }
+
+    #[test]
+    fn nothing_lands_in_a_lane_after_its_retirement_and_final_drain() {
+        let server = test_server(true, 4);
+        server.register_worker();
+        server.register_worker();
+        let stop = Arc::new(std::sync::atomic::AtomicBool::new(false));
+        let dispatcher = std::thread::spawn({
+            let server = server.clone();
+            let stop = stop.clone();
+            move || {
+                while !stop.load(Ordering::Relaxed) {
+                    let _ = try_dispatch(&server, test_ctx());
+                    // Lane 1 keeps draining, so dispatch never stalls on Full.
+                    let slots = server.slots.read();
+                    if let Some(rx) = slots[1].lane_rx.as_ref() {
+                        while rx.try_recv().is_ok() {}
+                    }
+                }
+            }
+        });
+        std::thread::sleep(Duration::from_millis(20));
+
+        // The scaler retires slot 0; its worker then drains what it holds.
+        let slot = server.slots.read()[0].clone();
+        slot.retire();
+        if let Some(rx) = slot.lane_rx.as_ref() {
+            while rx.try_recv().is_ok() {}
+        }
+
+        std::thread::sleep(Duration::from_millis(50));
+        stop.store(true, Ordering::Relaxed);
+        dispatcher.join().expect("dispatcher thread");
+        assert_eq!(server.lane_depths(), Some(vec![0, 0]));
+    }
 }