kino 0.2.1 → 0.4.0

data/README.md

@@ -196,6 +196,12 @@ bundle exec kino           # picks up config.ru + kino.rb, serves on :9292
 (After a standalone `gem install`, the `kino` command works without
 `bundle exec`.)
 
+Prefer your framework's own command? Kino ships a Rack handler, so
+`rails server -u kino` and `rackup -s kino` boot it too. They read the
+same `kino.rb` (or `config/kino.rb`), the host's `-p`/`-b` flags win over
+the file, and `rackup -s kino -O Workers=4 -O Mode=threaded` reaches the
+rest (`rackup -s kino --help` lists them).
+
 No Rust compiler needed: released versions ship precompiled native gems
 for Linux (x86_64/aarch64, glibc and musl) and macOS (arm64). On other
 platforms the gem compiles at install time; that needs a Rust toolchain,
@@ -218,9 +224,9 @@ Or embedded, with everything spelled out:
 
 ```ruby
 server = Kino::Server.new(app,
-  bind: "127.0.0.1",
+  bind: "127.0.0.1",          # or "unix:///run/kino.sock" behind a proxy
   port: 9292,                 # 0 = ephemeral; read back via server.port
-  workers: Etc.nprocessors,   # ractors (parallelism)
+  workers: Kino.available_parallelism, # ractors (parallelism); the default
   threads: 1,                 # per worker; ractor default 1, threaded default 3
   mode: :auto,                # :auto | :ractor | :threaded
   queue_depth: 1024,          # bounded queue; overflow → 503
@@ -230,6 +236,8 @@ server = Kino::Server.new(app,
   max_body_size: 50 * 1024 * 1024,  # bytes before a 413; nil = let a proxy handle it
   on_error: ->(e, env) { ErrorTracker.capture(e) },  # after the client got its 500
   shutdown_timeout: 30,       # drain deadline
+  control_bind: "127.0.0.1:9293",   # monitoring: /stats /metrics /ready /live; port 0 reads back via server.control_port
+  control_token: ENV["KINO_CONTROL_TOKEN"],  # optional Bearer auth for /stats + /metrics
   tls: { cert: "cert.pem", key: "key.pem" },  # file paths or inline PEM
 )
 server.start
@@ -253,8 +261,10 @@ server.shutdown               # graceful: drain → deadline → abort straggler
 
 ## Config file and CLI
 
-Settings can live in a Puma-style Ruby DSL file. Precedence: explicit
-kwargs and CLI flags > config file > defaults.
+Settings can live in a Puma-style Ruby DSL file: `kino.rb` in the
+working directory, or `config/kino.rb` (the Rails layout), is picked up
+automatically; `-C PATH` names any other. Precedence: explicit kwargs
+and CLI flags > config file > defaults.
 
 ```ruby
 # kino.rb
@@ -320,6 +330,44 @@ after the client got its 500—the only place a tracker sees errors
 raised while the response was being written (in `:ractor` mode, build
 the handler with `Ractor.shareable_proc`).
 
+## Lifecycle hooks
+
+Kino fires four lifecycle hooks alongside `on_error`, split by firing context.
+
+**Worker-context hooks** run inside the worker and are available to all workers:
+- `after_worker_boot { |worker_id| }`: runs once before the worker begins serving, with its slot id. In `:ractor` mode it runs inside the worker ractor and must be `Ractor.shareable_proc`.
+- `after_request_complete { |env, status| }`: fires inside the worker after each successful response. This is the hot path—leave it unset for zero cost. In `:ractor` mode it must be `Ractor.shareable_proc`.
+
+**Main-context hooks** run on the main thread, outside workers, and are plain procs:
+- `after_boot { }`: fires once after the worker pool is up. Wire readiness here—sd_notify, a "server ready" metric, and so on.
+- `on_worker_exit { |worker_index, error| }`: fires when a worker exits, with its index and the crash cause (or nil on a clean exit).
+
+`after_worker_boot`'s argument is the worker's slot id, while in `:ractor` mode `on_worker_exit`'s argument identifies the exited ractor (`0`..`workers - 1`)—a different number space—so don't correlate boot and exit by that number in `:ractor` mode.
+
+A raising hook is logged and never kills a worker.
+
+## Stuck-worker quarantine
+
+`quarantine_timeout: seconds` (or `quarantine_timeout 60` in `kino.rb`)
+quarantines a dispatch slot whose request has run longer than the deadline
+and spawns a replacement worker to restore capacity—distinct from
+`request_timeout`, which gives the client a 504 but leaves the slot
+occupied. `quarantine_max` (default: the worker count in `:ractor` mode,
+workers × threads in `:threaded`) caps the total number of replacement
+events over the process lifetime—past it the monitor stops replacing and
+the server runs at reduced capacity.
+
+The wedged worker is never interrupted or force-killed, and its slot stays
+quarantined for good. In `:threaded` mode, if the blocked thread
+eventually returns, it keeps serving requests on that same slot—but the
+slot itself stays flagged quarantined (busy_ms reported as 0) for the rest
+of the process; in `:ractor` mode the wedged ractor (and its supervisor
+thread) leaks until the process exits, since a wedged ractor cannot be
+safely interrupted. Monitor quarantine activity via `server.stats`
+(top-level `quarantined` count and per-slot `worker_status[].quarantined`
+flag), `GET /stats` (same), and `GET /metrics` (`kino_quarantined_workers`
+gauge and `kino_quarantine_replacements_total` counter).
+
 ## Stats
 
 `server.stats` returns a live snapshot: the configuration plus counters
@@ -330,17 +378,50 @@ cost):
 server.stats
 # => {mode: :ractor, lanes: false, workers: 8, threads: 1, batch: 1,
 #     respawns: 0, queued: 0, in_flight: 2, served: 1041, rejected: 0,
-#     timeouts: 0}
+#     timeouts: 0, worker_status: [...]}
 # plus lane_depths: [...] when lane dispatch is on
 ```
 
-From the outside, `kill -USR1 <pid>` prints the same snapshot as one line
+From the outside, `kill -USR1 <pid>` logs the same snapshot as one line
 (pair it with `pidfile` to find the pid):
 
 ```
-Kino stats: mode=:ractor lanes=false workers=8 threads=1 batch=1 respawns=0 queued=0 in_flight=2 served=1041 rejected=0 timeouts=0
+kino[4213] main: stats mode=:ractor lanes=false workers=8 threads=1 batch=1 respawns=0 queued=0 in_flight=2 served=1041 rejected=0 timeouts=0
 ```
 
+For pull-based monitoring, `control_bind "127.0.0.1:9293"` (or a
+`unix://` path) serves a read-only **control plane** from the native
+layer on its own thread—it keeps answering even while every Ruby worker
+is busy or stuck, and reports `draining` through a graceful shutdown:
+
+- `GET /stats`—the same snapshot as `server.stats`, as JSON (plus
+  `state` and `version`).
+- `GET /metrics`—Prometheus text format (`kino_requests_served_total`,
+  `kino_queue_depth`, `kino_ready`, …).
+
+Both `/stats` and `/metrics` also break the counters down per dispatch
+slot: `/stats` carries a `worker_status` array (`index`, `served`,
+`in_flight`, `busy_ms`) and `/metrics` emits `kino_worker_*{worker="N"}`
+series, one entry per execution slot (`workers × threads`)—a crashed
+worker's slot is never reused, so it stays in the list with its counters
+frozen where they stopped, meaning the array (and its `worker="N"` metric
+series) grows by one across every respawn. `busy_ms` is how long the
+slot's current request has been running (0 when idle), so a single slot
+climbing while the rest sit at 0 is your stuck worker.
+
+The `/stats` response and `server.stats` carry `queue_time` (count and
+summed seconds), and `/metrics` exposes `kino_request_queue_seconds`—a
+Prometheus histogram of queue-wait time, the worker-saturation signal.
+Counts admitted requests only; a 503 after queue wait goes to `rejected`,
+not `queue_time`.
+
+- `GET /ready`—`200` when serving, `503` while booting or draining:
+  wire it to your load balancer or Kubernetes readiness probe.
+- `GET /live`—`200` whenever the process is alive: the liveness probe.
+
+`control_token "..."` puts `/stats` and `/metrics` behind
+`Authorization: Bearer`; the probes stay open.
+
 ## Logging
 
 With one log line per request, `Kino::Logger` sustained **2.4× the
@@ -349,16 +430,28 @@ box). There are two native pieces. Both write through a lock-free
 channel to a Rust flusher thread, so request threads never take a log
 mutex and never make a write syscall:
 
-- **Access log** (`log_requests true`): one line per request to stdout,
-  including the 503s that never reach your app. Recommended in
-  development; cheap enough for production. On color terminals the
-  lines are tinted by status class: 2xx green, 3xx yellow, 4xx maroon,
-  5xx bright red:
+- **Access log** (`log_requests true`): two records per request to
+  stdout, including the 503s that never reach your app. The arrival line
+  is queued before the app runs, so a request that hangs shows as an
+  arrow with no answer; the completion line carries the status, the
+  total, and a timing breakdown: `ruby` is the time the request spent in
+  Ruby (with the GC pause and the objects allocated during it), `kino`
+  the server's own overhead, `wait` the queue time before a worker took
+  it. Recommended in development; cheap enough for production. On color
+  terminals the completion line is tinted by status class: 2xx green,
+  3xx yellow, 4xx maroon, 5xx bright red:
 
   ```
-  127.0.0.1 [Tue, 10 Jun 2026 13:39:56 GMT] "GET / HTTP/1.1" 200 0.1ms
+  2026-08-22 14:03:11 +0300 → GET /users?q=1  from 127.0.0.1
+  2026-08-22 14:03:11 +0300 ← 200 GET /users?q=1  12.4ms (ruby 9.1ms [gc 0.8ms; 1.5k obj]; kino 3.2ms; wait 0.1ms)
   ```
 
+  The GC and allocation figures come from the VM's process-wide
+  counters, so they appear only where one request at a time can own
+  them: in `:threaded` mode, or in `:ractor` mode with `workers 1`.
+  Parallel ractors would bill each other's work, so there the breakdown
+  is `(ruby; kino; wait)` alone.
+
 - **`Kino::Logger`**: a `::Logger` over the same async sink, for your
   app's own logging (`Kino::Logger.new("log/production.log")`, or no
   argument for stdout). The raw IO-like device is `Kino::Logger::Device`,
@@ -400,6 +493,27 @@ lines/s), the sink drops lines instead of blocking request threads.
 These trade-offs are measured in
 [doc/benchmarks.md](doc/benchmarks.md#logging-costs).
 
+**Server lines.** Everything Kino says about itself (draining, a crash
+and its respawn, a hook that raised, quarantine, the stats line,
+`rack.errors`) reads `kino[<pid>] <source>: message`, the source being
+the worker that spoke, `worker-3` (or `worker-3/thread-2` in a
+multi-threaded ractor), or `main`. On color terminals the label is dim
+for notes, yellow for warnings, red for errors; the message stays plain.
+A failed request gets a report instead of a bare backtrace: the request
+line, the error, and where it raised in your code, then the trace with
+your frames first, relative to the working directory, and the rest
+folded:
+
+```
+kino[4213] worker-2: 500 GET /boom · RuntimeError: kaboom (app.rb:12:in 'explode')
+    app.rb:12:in 'explode'
+    /usr/lib/ruby/gems/4.0.0/gems/rack-3.2.7/lib/rack/builder.rb:...
+    … 38 more
+```
+
+Hooks can log through the same channel with `Kino::Log.info`, `.warn`,
+and `.error`; it is safe inside worker ractors.
+
 ## Timer waits
 
 `Kino.sleep(seconds)` is a high-resolution sleep on the OS clock with
@@ -418,8 +532,9 @@ optional in Rack 3.
 
 ## Rails
 
-Rails (edge) runs on Kino today in `:threaded` mode; see
-`examples/rails-hello`. Ractor-mode Rails is blocked upstream. The exact
+Rails (edge) runs on Kino today in `:threaded` mode (`rails server -u
+kino`, or the `kino` CLI); see `examples/rails-hello`. Ractor-mode Rails
+is blocked upstream. The exact
 blockers, the `Ruby::Box` findings, and what would unlock it are written
 up in [doc/rails-on-ractors.md](doc/rails-on-ractors.md). The example
 ships a probe script that re-tests against whatever Rails you bundle.
@@ -442,7 +557,7 @@ For the Rust network stack, thanks to [Sean McArthur](https://github.com/seanmon
 
 ## Assisted by
 
-Claude Code (Mythos, Opus).
+Claude Code (Fable 5, Opus 4.8).
 
 ## Contributing
 

data/exe/kino

@@ -9,10 +9,6 @@
 # process-global bits an executable owns.
 
 Warning[:experimental] = false
-# Startup output must land immediately even when stdout is a pipe or file
-# (process supervisors, `kino > server.log`); block buffering would hold
-# the banner back until exit.
-$stdout.sync = true
 
 # Running from a git checkout (no installed gem, no bundler context):
 # prefer the checkout's own lib so `require "kino"` resolves.

data/ext/kino/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kino"
-version = "0.2.1"
+version = "0.4.0"
 edition = "2021"
 authors = ["Yaroslav Markin <yaroslav@markin.net>"]
 license = "MIT"
@@ -26,7 +26,9 @@ hyper-util = { version = "0.1", features = ["server", "tokio", "http1"] }
 http = "1"
 http-body-util = "0.1"
 bytes = "1"
-httpdate = "1"
+# Local-time stamps for the access log: the system zone only (no bundled
+# tzdb, no serde), which keeps the dependency tree to jiff itself.
+jiff = { version = "0.2", default-features = false, features = ["std", "tz-system", "tzdb-zoneinfo"] }
 # ring (not the default aws-lc-rs) as the rustls crypto provider: it
 # cross-compiles cleanly in rb-sys-dock containers, while aws-lc-sys wants
 # cmake and a per-target C toolchain.

data/ext/kino/src/access_log.rs

@@ -0,0 +1,245 @@
+//! The access log: two records per request on the async sink, so neither
+//! costs the request path a write syscall. An arrival line is queued as
+//! soon as the request head is parsed, before the app sees it, so a hang
+//! shows as an arrow with no answer; a status-tinted completion line
+//! follows the response head with the total and a timing breakdown:
+//!
+//! ```text
+//! 2026-08-22 14:03:11 +0300 → GET /users?q=1  from 127.0.0.1
+//! 2026-08-22 14:03:11 +0300 ← 200 GET /users?q=1  12.4ms (ruby 9.1ms [gc 0.8ms; 1.5k obj]; kino 3.2ms; wait 0.1ms)
+//! ```
+//!
+//! `ruby` is the time the request spent in a Ruby worker (admit to
+//! response head), with the GC pause and objects allocated during the app
+//! call when the worker measured them; `kino` is the server's own
+//! overhead (total minus ruby minus wait); `wait` is the queue time before
+//! a worker took the request. A blank line sets one request apart from
+//! the next.
+
+use std::fmt::Write as _;
+use std::net::IpAddr;
+use std::time::Duration;
+
+use crate::style::{self, Stream};
+
+/// Per-request timing, measured on the way through and riding the
+/// response as an extension so the intake side can log it.
+#[derive(Clone, Copy, Debug)]
+pub struct Timing {
+    /// Queue wait before a worker took the request.
+    pub wait: Duration,
+    /// Admit to response head: the request's time in Ruby.
+    pub ruby: Duration,
+    /// GC pause and objects allocated during the app call, when measured.
+    /// Left out where the VM's process-wide counters cannot be attributed
+    /// to one request (parallel ractors).
+    pub gc: Option<(Duration, u64)>,
+}
+
+/// The arrival record, stamped and styled for stdout.
+pub fn arrival(method: &str, target: &str, ip: IpAddr) -> String {
+    let color = style::enabled(Stream::Stdout);
+    let record = style::sgr(style::BOLD_WHITE, &arrival_line(method, target, ip), color);
+    stamped(&record, color)
+}
+
+/// The completion record, stamped, tinted by status class, followed by the
+/// dimmed breakdown when timing was measured (a 503 or 504 never reached a
+/// worker, so it has none). Ends with an extra newline: with the sink's
+/// own, that leaves the blank line after the record.
+pub fn completion(
+    status: u16,
+    method: &str,
+    target: &str,
+    total: Duration,
+    timing: Option<Timing>,
+) -> String {
+    let color = style::enabled(Stream::Stdout);
+    let record = completion_line(status, method, target, total);
+    let record = match style::status_sgr(status) {
+        Some(code) => style::sgr(code, &record, color),
+        None => record,
+    };
+    let mut line = stamped(&record, color);
+    if let Some(timing) = timing {
+        line.push(' ');
+        if color {
+            let _ = write!(line, "\x1b[{}m", style::DIM);
+        }
+        write_breakdown(&mut line, total, &timing);
+        if color {
+            line.push_str("\x1b[0m");
+        }
+    }
+    line.push('\n');
+    line
+}
+
+/// Prefix a record with the local timestamp, dimmed when coloring so it
+/// recedes behind the arrow and status.
+fn stamped(record: &str, color: bool) -> String {
+    format!("{} {record}", style::sgr(style::DIM, &now_stamp(), color))
+}
+
+/// The local wall clock as `2026-08-22 14:03:11 +0300`: date, time, and
+/// the numeric UTC offset, so a log file is unambiguous across machines.
+fn now_stamp() -> String {
+    jiff::Zoned::now()
+        .strftime("%Y-%m-%d %H:%M:%S %z")
+        .to_string()
+}
+
+/// `→ METHOD target  from IP`.
+fn arrival_line(method: &str, target: &str, ip: IpAddr) -> String {
+    format!("\u{2192} {method} {target}  from {ip}")
+}
+
+/// `← STATUS METHOD target  N.Nms`.
+fn completion_line(status: u16, method: &str, target: &str, total: Duration) -> String {
+    format!(
+        "\u{2190} {status} {method} {target}  {:.1}ms",
+        millis(total)
+    )
+}
+
+/// Append the breakdown straight into the line buffer (no string of its
+/// own): `(ruby N.Nms [gc N.Nms; N obj]; kino N.Nms; wait N.Nms)`, the gc
+/// bracket only when measured. `kino` floors at zero: at sub-millisecond
+/// totals clock granularity can make the parts exceed the whole.
+fn write_breakdown(out: &mut String, total: Duration, timing: &Timing) {
+    let kino = total
+        .saturating_sub(timing.ruby)
+        .saturating_sub(timing.wait);
+    let _ = write!(out, "(ruby {:.1}ms", millis(timing.ruby));
+    if let Some((gc, allocs)) = timing.gc {
+        let _ = write!(out, " [gc {:.1}ms; ", millis(gc));
+        write_count(out, allocs);
+        out.push_str(" obj]");
+    }
+    let _ = write!(
+        out,
+        "; kino {:.1}ms; wait {:.1}ms)",
+        millis(kino),
+        millis(timing.wait)
+    );
+}
+
+fn millis(d: Duration) -> f64 {
+    d.as_secs_f64() * 1000.0
+}
+
+/// A humanized count: `523`, `1.5k`, `52k`.
+fn write_count(out: &mut String, n: u64) {
+    if n < 1000 {
+        let _ = write!(out, "{n}");
+    } else if n < 10_000 {
+        let _ = write!(out, "{:.1}k", n as f64 / 1000.0);
+    } else {
+        let _ = write!(out, "{}k", n / 1000);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{arrival_line, completion_line, now_stamp, write_breakdown, write_count, Timing};
+    use std::net::{IpAddr, Ipv4Addr};
+    use std::time::Duration;
+
+    fn breakdown(total: Duration, timing: &Timing) -> String {
+        let mut out = String::new();
+        write_breakdown(&mut out, total, timing);
+        out
+    }
+
+    fn count(n: u64) -> String {
+        let mut out = String::new();
+        write_count(&mut out, n);
+        out
+    }
+
+    #[test]
+    fn arrival_line_carries_method_target_and_ip() {
+        let ip = IpAddr::V4(Ipv4Addr::new(203, 0, 113, 5));
+        assert_eq!(
+            arrival_line("GET", "/users?q=1", ip),
+            "\u{2192} GET /users?q=1  from 203.0.113.5"
+        );
+    }
+
+    #[test]
+    fn completion_line_carries_status_target_and_total() {
+        assert_eq!(
+            completion_line(200, "GET", "/users", Duration::from_millis(12)),
+            "\u{2190} 200 GET /users  12.0ms"
+        );
+    }
+
+    #[test]
+    fn breakdown_splits_the_total_into_ruby_kino_and_wait() {
+        let timing = Timing {
+            wait: Duration::from_micros(100),
+            ruby: Duration::from_millis(41),
+            gc: Some((Duration::from_micros(9_700), 52_000)),
+        };
+        // kino = total - ruby - wait = 45.2 - 41.0 - 0.1 = 4.1ms.
+        assert_eq!(
+            breakdown(Duration::from_micros(45_200), &timing),
+            "(ruby 41.0ms [gc 9.7ms; 52k obj]; kino 4.1ms; wait 0.1ms)"
+        );
+    }
+
+    #[test]
+    fn breakdown_leaves_out_the_gc_bracket_when_not_measured() {
+        let timing = Timing {
+            wait: Duration::from_millis(1),
+            ruby: Duration::from_millis(2),
+            gc: None,
+        };
+        assert_eq!(
+            breakdown(Duration::from_millis(4), &timing),
+            "(ruby 2.0ms; kino 1.0ms; wait 1.0ms)"
+        );
+    }
+
+    #[test]
+    fn breakdown_floors_kino_at_zero() {
+        let timing = Timing {
+            wait: Duration::from_millis(1),
+            ruby: Duration::from_millis(40),
+            gc: Some((Duration::ZERO, 10)),
+        };
+        assert_eq!(
+            breakdown(Duration::from_millis(40), &timing),
+            "(ruby 40.0ms [gc 0.0ms; 10 obj]; kino 0.0ms; wait 1.0ms)"
+        );
+    }
+
+    #[test]
+    fn count_humanizes_allocation_counts() {
+        assert_eq!(count(0), "0");
+        assert_eq!(count(523), "523");
+        assert_eq!(count(1500), "1.5k");
+        assert_eq!(count(52_000), "52k");
+    }
+
+    #[test]
+    fn timestamp_is_date_time_and_numeric_offset() {
+        // `YYYY-MM-DD HH:MM:SS +HHMM`, e.g. `2026-08-22 14:03:11 +0300`.
+        let ts = now_stamp();
+        let (datetime, offset) = ts.rsplit_once(' ').expect("offset field");
+        let (date, time) = datetime.split_once(' ').expect("date and time");
+        assert_eq!(date.len(), 10, "date {date}");
+        assert_eq!(&date[4..5], "-");
+        assert_eq!(time.len(), 8, "time {time}");
+        assert_eq!(&time[2..3], ":");
+        assert_eq!(offset.len(), 5, "offset {offset}");
+        assert!(matches!(&offset[0..1], "+" | "-"));
+        assert!(offset[1..].bytes().all(|b| b.is_ascii_digit()));
+    }
+
+    #[test]
+    fn completion_ends_with_the_blank_line_newline() {
+        let line = super::completion(200, "GET", "/", Duration::from_millis(1), None);
+        assert!(line.ends_with("\u{2190} 200 GET /  1.0ms\n") || line.ends_with("1.0ms\x1b[0m\n"));
+    }
+}