kino 0.4.0 → 0.5.0

data/ext/kino/src/io_shards.rs

@@ -0,0 +1,437 @@
+//! Current-thread Tokio runtimes for HTTP I/O.
+//!
+//! One accept thread owns the listener and assigns accepted connections to
+//! the least-loaded shard. Each shard then owns that connection for its
+//! lifetime, avoiding the shared Tokio worker pool on hot HTTP paths.
+
+use std::net::SocketAddr;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::thread::JoinHandle;
+
+use crate::listen::Listener;
+use crate::log::{self, Level};
+use crate::registry::{ServerInner, STATE_DRAINING};
+use crate::server::{serve_conn, AsyncListener, Conn};
+
+/// A connection in transit from the acceptor to its shard. Tokio streams
+/// are bound to the runtime that registered them, so the handoff carries
+/// the std stream and the shard re-registers it on arrival.
+enum StdConn {
+    Tcp(std::net::TcpStream),
+    Unix(std::os::unix::net::UnixStream),
+}
+
+impl StdConn {
+    /// Register with the calling (shard) runtime.
+    fn into_tokio(self) -> std::io::Result<Conn> {
+        Ok(match self {
+            StdConn::Tcp(stream) => Conn::Tcp(tokio::net::TcpStream::from_std(stream)?),
+            StdConn::Unix(stream) => Conn::Unix(tokio::net::UnixStream::from_std(stream)?),
+        })
+    }
+}
+
+/// Detach an accepted stream from the acceptor's runtime for the handoff.
+fn into_std(conn: Conn) -> std::io::Result<StdConn> {
+    Ok(match conn {
+        Conn::Tcp(stream) => StdConn::Tcp(stream.into_std()?),
+        Conn::Unix(stream) => StdConn::Unix(stream.into_std()?),
+    })
+}
+
+/// One accepted connection en route to a shard: the detached stream, the
+/// addresses hyper reports, and the slot it holds against max_connections.
+struct Accepted {
+    conn: StdConn,
+    remote_addr: SocketAddr,
+    local_addr: SocketAddr,
+    permit: tokio::sync::OwnedSemaphorePermit,
+}
+
+/// Shard count: an explicit `io_threads` wins; the default is half the
+/// available CPUs. Framing requests is cheap next to running the app, so
+/// the I/O plane gets the smaller share and Ruby workers keep the rest.
+pub(crate) fn thread_count(io_threads: usize) -> usize {
+    if io_threads > 0 {
+        return io_threads;
+    }
+    default_thread_count(std::thread::available_parallelism().map_or(1, usize::from))
+}
+
+fn default_thread_count(cpus: usize) -> usize {
+    cpus.div_ceil(2)
+}
+
+/// Boot the shard threads, then the acceptor. Any thread that fails to
+/// come up fails the whole boot: the already started threads are drained
+/// (their senders drop) and joined before the error reaches Ruby.
+pub(crate) fn spawn(
+    listener: Listener,
+    acceptor: Option<tokio_rustls::TlsAcceptor>,
+    server: Arc<ServerInner>,
+    max_connections: usize,
+    accept_shutdown_rx: tokio::sync::watch::Receiver<bool>,
+    runtime_shutdown_rx: tokio::sync::watch::Receiver<bool>,
+    shard_count: usize,
+) -> std::io::Result<Vec<JoinHandle<()>>> {
+    let shard_count = shard_count.max(1);
+    let mut handles = Vec::with_capacity(shard_count + 1);
+    let mut shard_txs = Vec::with_capacity(shard_count);
+    let mut loads = Vec::with_capacity(shard_count);
+
+    for i in 0..shard_count {
+        let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
+        let load = Arc::new(AtomicUsize::new(0));
+        let spawned = spawn_shard(
+            i,
+            rx,
+            acceptor.clone(),
+            server.clone(),
+            load.clone(),
+            runtime_shutdown_rx.clone(),
+        );
+        match spawned {
+            Ok(handle) => {
+                shard_txs.push(tx);
+                loads.push(load);
+                handles.push(handle);
+            }
+            Err(error) => {
+                drop(tx);
+                drop(shard_txs);
+                join_all(handles);
+                return Err(error);
+            }
+        }
+    }
+
+    match spawn_acceptor(
+        listener,
+        server,
+        max_connections,
+        accept_shutdown_rx,
+        shard_txs,
+        loads,
+    ) {
+        Ok(handle) => handles.push(handle),
+        Err(error) => {
+            join_all(handles);
+            return Err(error);
+        }
+    }
+    Ok(handles)
+}
+
+fn join_all(handles: Vec<JoinHandle<()>>) {
+    for handle in handles {
+        let _ = handle.join();
+    }
+}
+
+fn current_thread_runtime() -> std::io::Result<tokio::runtime::Runtime> {
+    tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+}
+
+/// Wait for a just spawned I/O thread to report its runtime up, so a
+/// startup failure becomes the boot error Ruby sees instead of a silently
+/// dead thread.
+fn await_ready(
+    handle: JoinHandle<()>,
+    ready_rx: std::sync::mpsc::Receiver<std::io::Result<()>>,
+    what: &str,
+) -> std::io::Result<JoinHandle<()>> {
+    match ready_rx.recv() {
+        Ok(Ok(())) => Ok(handle),
+        Ok(Err(error)) => {
+            let _ = handle.join();
+            Err(error)
+        }
+        Err(_) => Err(std::io::Error::other(format!(
+            "{what} thread exited during startup"
+        ))),
+    }
+}
+
+fn spawn_shard(
+    index: usize,
+    rx: tokio::sync::mpsc::UnboundedReceiver<Accepted>,
+    acceptor: Option<tokio_rustls::TlsAcceptor>,
+    server: Arc<ServerInner>,
+    load: Arc<AtomicUsize>,
+    mut shutdown_rx: tokio::sync::watch::Receiver<bool>,
+) -> std::io::Result<JoinHandle<()>> {
+    let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel(1);
+    let handle = std::thread::Builder::new()
+        .name(format!("kino-io-{index}"))
+        .spawn(move || {
+            let runtime = match current_thread_runtime() {
+                Ok(runtime) => runtime,
+                Err(error) => {
+                    let _ = ready_tx.send(Err(error));
+                    return;
+                }
+            };
+            let _ = ready_tx.send(Ok(()));
+            runtime.block_on(shard_loop(rx, acceptor, server, load, &mut shutdown_rx));
+        })?;
+    await_ready(handle, ready_rx, "shard")
+}
+
+fn spawn_acceptor(
+    listener: Listener,
+    server: Arc<ServerInner>,
+    max_connections: usize,
+    shutdown_rx: tokio::sync::watch::Receiver<bool>,
+    shard_txs: Vec<tokio::sync::mpsc::UnboundedSender<Accepted>>,
+    loads: Vec<Arc<AtomicUsize>>,
+) -> std::io::Result<JoinHandle<()>> {
+    let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel(1);
+    let handle = std::thread::Builder::new()
+        .name("kino-accept".to_string())
+        .spawn(move || {
+            let runtime = match current_thread_runtime() {
+                Ok(runtime) => runtime,
+                Err(error) => {
+                    let _ = ready_tx.send(Err(error));
+                    return;
+                }
+            };
+            runtime.block_on(async move {
+                // Registration must happen on this runtime; a failure is
+                // routed through the same ready channel as a build error.
+                let listener = match AsyncListener::from_std(listener) {
+                    Ok(listener) => listener,
+                    Err(error) => {
+                        let _ = ready_tx.send(Err(error));
+                        return;
+                    }
+                };
+                accept_loop(
+                    listener,
+                    server,
+                    max_connections,
+                    shutdown_rx,
+                    shard_txs,
+                    loads,
+                    ready_tx,
+                )
+                .await;
+            });
+        })?;
+    await_ready(handle, ready_rx, "accept")
+}
+
+/// Serve handed-over connections until the acceptor is gone, then let the
+/// remaining ones finish. The final teardown signal cuts either phase
+/// short: dropping the runtime cancels connection tasks at their next
+/// await point.
+async fn shard_loop(
+    mut rx: tokio::sync::mpsc::UnboundedReceiver<Accepted>,
+    acceptor: Option<tokio_rustls::TlsAcceptor>,
+    server: Arc<ServerInner>,
+    load: Arc<AtomicUsize>,
+    shutdown_rx: &mut tokio::sync::watch::Receiver<bool>,
+) {
+    let mut connections = tokio::task::JoinSet::new();
+    loop {
+        tokio::select! {
+            _ = shutdown_rx.changed() => return,
+            accepted = rx.recv() => {
+                let Some(accepted) = accepted else { break };
+                let acceptor = acceptor.clone();
+                let server = server.clone();
+                let guard = LoadGuard(load.clone());
+                connections.spawn(async move {
+                    let _guard = guard;
+                    serve_accepted(accepted, acceptor, server).await;
+                });
+            }
+            // Reap closed connections so the set stays small.
+            Some(_) = connections.join_next(), if !connections.is_empty() => {}
+        }
+    }
+    // The acceptor is gone: drain. Every join is one connection closing.
+    while !connections.is_empty() {
+        tokio::select! {
+            _ = shutdown_rx.changed() => return,
+            _ = connections.join_next() => {}
+        }
+    }
+}
+
+/// Keeps the shard's connection count honest whichever way the task ends:
+/// return, panic, or cancellation at teardown. A plain decrement after the
+/// await would never run on the last two.
+struct LoadGuard(Arc<AtomicUsize>);
+
+impl Drop for LoadGuard {
+    fn drop(&mut self) {
+        self.0.fetch_sub(1, Ordering::Relaxed);
+    }
+}
+
+/// The shard's half of the handoff: re-register the stream on this
+/// runtime, then run the shared connection pipeline (TLS handshake,
+/// protocol layer) exactly as the default runtime would.
+async fn serve_accepted(
+    accepted: Accepted,
+    acceptor: Option<tokio_rustls::TlsAcceptor>,
+    server: Arc<ServerInner>,
+) {
+    // Held for the connection's lifetime; dropping it frees a slot.
+    let _permit = accepted.permit;
+    let conn = match accepted.conn.into_tokio() {
+        Ok(conn) => conn,
+        Err(_) => {
+            log::emit(
+                Level::Warn,
+                "tokio",
+                "failed to register a stream on an I/O shard",
+            );
+            return;
+        }
+    };
+    serve_conn(
+        conn,
+        acceptor,
+        server,
+        accepted.remote_addr,
+        accepted.local_addr,
+    )
+    .await;
+}
+
+/// The sharded accept loop. Same backpressure as the default loop: the
+/// permit is acquired before accept, so past max_connections the excess
+/// waits in the kernel backlog instead of being accepted and dropped.
+/// A shard whose channel is gone is marked dead and routed around; with
+/// no shard left the loop stops accepting and flips the server to
+/// draining, so the control plane stops reporting ready.
+async fn accept_loop(
+    listener: AsyncListener,
+    server: Arc<ServerInner>,
+    max_connections: usize,
+    mut shutdown_rx: tokio::sync::watch::Receiver<bool>,
+    shard_txs: Vec<tokio::sync::mpsc::UnboundedSender<Accepted>>,
+    loads: Vec<Arc<AtomicUsize>>,
+    ready_tx: std::sync::mpsc::SyncSender<std::io::Result<()>>,
+) {
+    let conn_limit = Arc::new(tokio::sync::Semaphore::new(max_connections));
+    let mut live = vec![true; shard_txs.len()];
+    let _ = ready_tx.send(Ok(()));
+    'accept: loop {
+        let permit = tokio::select! {
+            _ = shutdown_rx.changed() => break,
+            permit = conn_limit.clone().acquire_owned() => match permit {
+                Ok(permit) => permit,
+                Err(_) => break,
+            },
+        };
+        let (conn, remote_addr, local_addr) = tokio::select! {
+            _ = shutdown_rx.changed() => break,
+            accepted = listener.accept() => match accepted {
+                Ok(accepted) => accepted,
+                Err(_) => continue,
+            },
+        };
+        let conn = match into_std(conn) {
+            Ok(conn) => conn,
+            Err(_) => {
+                log::emit(
+                    Level::Warn,
+                    "tokio",
+                    "failed to detach an accepted stream; connection dropped",
+                );
+                continue;
+            }
+        };
+        let mut accepted = Accepted {
+            conn,
+            remote_addr,
+            local_addr,
+            permit,
+        };
+        loop {
+            let Some(index) = least_loaded(&loads, &live) else {
+                log::emit(
+                    Level::Error,
+                    "tokio",
+                    "all I/O shards are down; not accepting connections",
+                );
+                server.state.store(STATE_DRAINING, Ordering::Relaxed);
+                break 'accept;
+            };
+            loads[index].fetch_add(1, Ordering::Relaxed);
+            match shard_txs[index].send(accepted) {
+                Ok(()) => break,
+                Err(returned) => {
+                    loads[index].fetch_sub(1, Ordering::Relaxed);
+                    live[index] = false;
+                    log::emit(Level::Warn, "tokio", "I/O shard is down; routing around it");
+                    accepted = returned.0;
+                }
+            }
+        }
+    }
+}
+
+/// The live shard with the fewest open connections.
+fn least_loaded(loads: &[Arc<AtomicUsize>], live: &[bool]) -> Option<usize> {
+    loads
+        .iter()
+        .enumerate()
+        .filter(|(index, _)| live[*index])
+        .min_by_key(|(_, load)| load.load(Ordering::Relaxed))
+        .map(|(index, _)| index)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{default_thread_count, least_loaded, thread_count, Arc, AtomicUsize};
+
+    fn loads(counts: &[usize]) -> Vec<Arc<AtomicUsize>> {
+        counts
+            .iter()
+            .map(|&n| Arc::new(AtomicUsize::new(n)))
+            .collect()
+    }
+
+    #[test]
+    fn explicit_io_threads_win() {
+        assert_eq!(thread_count(3), 3);
+    }
+
+    #[test]
+    fn least_loaded_picks_the_emptiest_live_shard() {
+        assert_eq!(
+            least_loaded(&loads(&[3, 0, 1]), &[true, true, true]),
+            Some(1)
+        );
+    }
+
+    #[test]
+    fn least_loaded_routes_around_dead_shards() {
+        // The dead shard's count is frozen at 0; it must not win anyway.
+        assert_eq!(
+            least_loaded(&loads(&[3, 0, 1]), &[true, false, true]),
+            Some(2)
+        );
+    }
+
+    #[test]
+    fn least_loaded_reports_when_no_shard_is_left() {
+        assert_eq!(least_loaded(&loads(&[0, 0]), &[false, false]), None);
+    }
+
+    #[test]
+    fn default_is_half_the_cpus() {
+        assert_eq!(default_thread_count(1), 1);
+        assert_eq!(default_thread_count(2), 1);
+        assert_eq!(default_thread_count(3), 2);
+        assert_eq!(default_thread_count(12), 6);
+        assert_eq!(default_thread_count(128), 64);
+    }
+}

data/ext/kino/src/lib.rs

@@ -9,6 +9,7 @@ mod control;
 mod cpus;
 mod env_strings;
 mod gvl;
+mod io_shards;
 mod listen;
 mod log;
 mod logsink;

data/ext/kino/src/listen.rs

@@ -86,7 +86,10 @@ mod tests {
 
     #[test]
     fn unix_path_recognises_only_the_unix_scheme() {
-        assert_eq!(unix_path("unix:///run/kino.sock").unwrap().to_str(), Some("/run/kino.sock"));
+        assert_eq!(
+            unix_path("unix:///run/kino.sock").unwrap().to_str(),
+            Some("/run/kino.sock")
+        );
         assert!(unix_path("127.0.0.1").is_none());
         assert!(unix_path("unix.example.com").is_none());
     }

data/ext/kino/src/log.rs

@@ -99,7 +99,10 @@ mod tests {
     #[test]
     fn label_is_a_syslog_tag_plus_the_source() {
         assert_eq!(label(4213, "main"), "kino[4213] main:");
-        assert_eq!(label(4213, "worker-3/thread-2"), "kino[4213] worker-3/thread-2:");
+        assert_eq!(
+            label(4213, "worker-3/thread-2"),
+            "kino[4213] worker-3/thread-2:"
+        );
     }
 
     #[test]
@@ -129,7 +132,12 @@ mod tests {
     #[test]
     fn a_report_is_labelled_on_its_first_line_only() {
         assert_eq!(
-            format_line(Level::Error, "kino[1] main:", "500 GET / · X: y\n    a.rb:1", false),
+            format_line(
+                Level::Error,
+                "kino[1] main:",
+                "500 GET / · X: y\n    a.rb:1",
+                false
+            ),
             "kino[1] main: 500 GET / · X: y\n    a.rb:1"
         );
     }

data/ext/kino/src/queue.rs

@@ -117,7 +117,8 @@ fn admit(
 ) -> Result<RHash, Error> {
     server.served.fetch_add(1, Ordering::Relaxed);
     slot.served.fetch_add(1, Ordering::Relaxed);
-    slot.last_started_ms.store(crate::mono::mono_ms(), Ordering::Relaxed);
+    slot.last_started_ms
+        .store(crate::mono::mono_ms(), Ordering::Relaxed);
     slot.in_flight.fetch_add(1, Ordering::Relaxed);
     // One clock read serves the histogram and, for the access log, the
     // request's queue wait and the start of its time in Ruby.

data/ext/kino/src/registry.rs

@@ -5,6 +5,7 @@
 
 use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
 use std::sync::{Arc, OnceLock, Weak};
+use std::time::{Duration, Instant};
 
 use parking_lot::{Mutex, RwLock};
 
@@ -32,6 +33,51 @@ pub type BoxedCtx = Box<RequestCtx>;
 /// Probed on every take; keys are our own ids, so ahash over SipHash.
 type HashMap<K, V> = std::collections::HashMap<K, V, ahash::RandomState>;
 
+/// What runs the server's I/O, taken out at shutdown. The default
+/// multi-thread runtime and sharded I/O carry different teardown state,
+/// so the shape stays explicit instead of parallel optional fields.
+#[derive(Default)]
+pub enum RuntimeHandle {
+    /// Not started yet, or already shut down.
+    #[default]
+    None,
+    MultiThread(tokio::runtime::Runtime),
+    /// The shards' final-teardown signal and every I/O thread (shards plus
+    /// the acceptor).
+    Shards {
+        shutdown_tx: tokio::sync::watch::Sender<bool>,
+        threads: Vec<std::thread::JoinHandle<()>>,
+    },
+}
+
+impl RuntimeHandle {
+    /// Stop the I/O side, giving in-flight work `timeout` to finish. A
+    /// thread that misses the deadline is abandoned rather than joined:
+    /// a wedged shard must not hang `Server#shutdown`, which promises to
+    /// return by its deadline.
+    pub fn shutdown(self, timeout: Duration) {
+        match self {
+            RuntimeHandle::None => {}
+            RuntimeHandle::MultiThread(runtime) => runtime.shutdown_timeout(timeout),
+            RuntimeHandle::Shards {
+                shutdown_tx,
+                threads,
+            } => {
+                let _ = shutdown_tx.send(true);
+                let deadline = Instant::now() + timeout;
+                for thread in threads {
+                    while !thread.is_finished() && Instant::now() < deadline {
+                        std::thread::sleep(Duration::from_millis(1));
+                    }
+                    if thread.is_finished() {
+                        let _ = thread.join();
+                    }
+                }
+            }
+        }
+    }
+}
+
 /// One per `Kino::Server`. Owns the tokio runtime, the request queue and the
 /// worker slots.
 pub struct ServerInner {
@@ -42,9 +88,9 @@ pub struct ServerInner {
     pub req_rx: flume::Receiver<BoxedCtx>,
     /// Signals the accept loop to stop. Watch channel: `true` = draining.
     pub shutdown_tx: tokio::sync::watch::Sender<bool>,
-    /// Runtime is kept so we can shut it down explicitly; in an Option so
-    /// `shutdown_runtime` can take ownership out of the Arc.
-    pub runtime: Mutex<Option<tokio::runtime::Runtime>>,
+    /// Runtime is kept so we can shut it down explicitly; `shutdown_runtime`
+    /// takes ownership out of the Arc.
+    pub runtime: Mutex<RuntimeHandle>,
     pub slots: RwLock<Vec<Arc<WorkerSlot>>>,
     pub in_flight: AtomicUsize,
     /// Requests handed to Ruby workers (admitted), and requests rejected
@@ -118,8 +164,8 @@ pub const LANE_DEPTH: usize = 4;
 /// Fixed queue-wait bucket boundaries in microseconds (0.5ms .. 10s),
 /// ascending. Emitted in seconds. Not a knob (YAGNI).
 pub const QUEUE_BOUNDS_US: [u64; 14] = [
-    500, 1_000, 2_500, 5_000, 10_000, 25_000, 50_000, 100_000,
-    250_000, 500_000, 1_000_000, 2_500_000, 5_000_000, 10_000_000,
+    500, 1_000, 2_500, 5_000, 10_000, 25_000, 50_000, 100_000, 250_000, 500_000, 1_000_000,
+    2_500_000, 5_000_000, 10_000_000,
 ];
 
 /// Queue-wait histogram: per-bucket counts plus an overflow (the implicit
@@ -289,7 +335,7 @@ pub fn test_server(lanes: bool, queue_depth: usize) -> Arc<ServerInner> {
         req_tx: Mutex::new(Some(req_tx)),
         req_rx,
         shutdown_tx,
-        runtime: Mutex::new(None),
+        runtime: Mutex::new(RuntimeHandle::None),
         slots: RwLock::new(Vec::new()),
         in_flight: AtomicUsize::new(0),
         served: AtomicU64::new(0),
@@ -301,7 +347,12 @@ pub fn test_server(lanes: bool, queue_depth: usize) -> Arc<ServerInner> {
         state: std::sync::atomic::AtomicU8::new(STATE_BOOTING),
         respawns: AtomicU64::new(0),
         quarantine_replacements: AtomicU64::new(0),
-        topology: Topology { mode: "threaded".to_string(), workers: 0, threads: 0, batch: 1 },
+        topology: Topology {
+            mode: "threaded".to_string(),
+            workers: 0,
+            threads: 0,
+            batch: 1,
+        },
         https: false,
         unix_path: None,
         access_log: None,
@@ -317,6 +368,44 @@ mod tests {
     use super::*;
     use crate::request::test_ctx;
 
+    #[test]
+    fn shards_shutdown_joins_threads_that_observe_the_signal() {
+        let (shutdown_tx, rx) = tokio::sync::watch::channel(false);
+        let thread = std::thread::spawn(move || {
+            while !*rx.borrow() {
+                std::thread::sleep(Duration::from_millis(1));
+            }
+        });
+
+        let start = Instant::now();
+        RuntimeHandle::Shards {
+            shutdown_tx,
+            threads: vec![thread],
+        }
+        .shutdown(Duration::from_secs(5));
+
+        // The thread exits on the signal, so the join comes nowhere near
+        // the deadline.
+        assert!(start.elapsed() < Duration::from_secs(1));
+    }
+
+    #[test]
+    fn shards_shutdown_abandons_a_wedged_thread_at_the_deadline() {
+        let (shutdown_tx, _rx) = tokio::sync::watch::channel(false);
+        let wedged = std::thread::spawn(|| std::thread::sleep(Duration::from_secs(30)));
+
+        let start = Instant::now();
+        RuntimeHandle::Shards {
+            shutdown_tx,
+            threads: vec![wedged],
+        }
+        .shutdown(Duration::from_millis(50));
+
+        let elapsed = start.elapsed();
+        assert!(elapsed >= Duration::from_millis(50));
+        assert!(elapsed < Duration::from_secs(5));
+    }
+
     #[test]
     fn worker_registration_hands_out_sequential_slot_ids() {
         let server = test_server(false, 4);
@@ -420,9 +509,9 @@ mod tests {
     #[test]
     fn queue_histogram_buckets_by_wait() {
         let h = QueueHistogram::new();
-        h.record(400);        // <= 500 -> bucket 0
-        h.record(500);        // == 500 -> bucket 0 (inclusive)
-        h.record(600);        // (500, 1000] -> bucket 1
+        h.record(400); // <= 500 -> bucket 0
+        h.record(500); // == 500 -> bucket 0 (inclusive)
+        h.record(600); // (500, 1000] -> bucket 1
         h.record(20_000_000); // > last bound -> overflow
         let s = h.snapshot();
         assert_eq!(s.buckets[0], 2);

data/ext/kino/src/response.rs

@@ -115,10 +115,7 @@ pub fn plain_response(status: u16, message: &'static str) -> HyperResponse {
 mod tests {
     use super::*;
 
-    fn pair() -> (
-        Responder,
-        tokio::sync::oneshot::Receiver<HyperResponse>,
-    ) {
+    fn pair() -> (Responder, tokio::sync::oneshot::Receiver<HyperResponse>) {
         let (head_tx, head_rx) = tokio::sync::oneshot::channel();
         (Responder::new(head_tx), head_rx)
     }