kino 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec37a5aaacb048ffa4533a89545295c9816a6fa7e8a968e6ffa2e154ab753926
-  data.tar.gz: 1405b92f68fc9dc61a82a099be2471027d341e649d2c874158fe6f03e4c9a471
+  metadata.gz: 129f2e9c0c20ba89c1f2f1e854d32f2b746858546b0c6fdb1a88a3fcb14e05cf
+  data.tar.gz: 66b465f2e5989e621487752d0b4dac48a9da0857b5290effaac9af803bd315e2
 SHA512:
-  metadata.gz: 2ded253aa471a55909aeacde1646650abdd29e240b7d5d304ce494c17c223d2869a3872210f7303ca30a307b4a8421c2772487156257a3b10e2d5e9bbda60ece
-  data.tar.gz: fc589bb25c4274d21c708dead07a3e8324eb99975f46d07b9fa721140c8483347fe5f84c082a59c936ff95aa3699d714b341f82f6911041c484a59454d7f8d82
+  metadata.gz: ac6656c0cd168c5c43dd5d1585b639b93f2489b5e017cbdd297c383c19d083c391cd7ac60034dbcd825bc525b42d7a966f884aa9d03eb0185e7bf8b5539a7781
+  data.tar.gz: 4392ca122bb8d865eec08393a6f339343bb2f5fac209b82e845ec6319d04f71a7a309a70d5d6518a70d70fc701c50dbd482646250b4d46f3e0d313e497bf4e3d

data/CHANGELOG.md

@@ -1,3 +1,30 @@
+## [0.3.0] - 2026-08-13
+
+- Queue-time histogram: `/metrics` exposes `kino_request_queue_seconds`, a
+  histogram of how long each request waited for a free worker (the
+  saturation signal), and `server.stats`/`/stats` gain `queue_time`
+  (count and summed seconds). Measured internally with a monotonic clock,
+  so it needs no proxy header and is immune to clock skew.
+- Lifecycle hooks: `after_boot`, `after_worker_boot`,
+  `after_request_complete`, and `on_worker_exit` join `on_error`, so apps
+  can wire their own metrics, readiness, and error tracking. The
+  worker-context hooks must be Ractor-shareable in `:ractor` mode; a raising
+  hook is logged and never kills a worker.
+- Stuck-worker quarantine: past `quarantine_timeout`, a wedged dispatch
+  slot is quarantined and a replacement worker is spawned to restore
+  capacity (capped by `quarantine_max`), surfaced via `/stats`, `/metrics`,
+  and `server.stats`. The wedged worker is never force-killed.
+- Control plane: a read-only monitoring listener (`control_bind`,
+  optional `control_token`) serving live stats as JSON at `/stats`,
+  Prometheus metrics at `/metrics`, and `/ready`/`/live` probes, answered
+  from the native layer on a dedicated thread so it stays responsive
+  while workers are busy, stuck, or draining.
+- Per-worker stats: `/stats`, `/metrics`, and `server.stats` now break the
+  counters down per dispatch slot (served, in-flight, and `busy_ms`, the
+  age of the slot's current request), so a stuck slot is visible
+  individually.
+- Update Rust and Ruby dependencies.
+
 ## [0.2.1] - 2026-07-27
 
 - Update Rust dependencies for Kino.

data/Cargo.lock

@@ -17,9 +17,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "1.1.4"
+version = "1.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
 dependencies = [
  "memchr",
 ]
@@ -51,7 +51,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex 1.3.0",
- "syn",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -74,9 +74,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "cc"
-version = "1.4.0"
+version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9"
+checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e"
 dependencies = [
  "find-msvc-tools",
  "shlex 2.0.1",
@@ -99,9 +99,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
 
 [[package]]
 name = "clang-sys"
-version = "1.8.1"
+version = "1.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
+checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a"
 dependencies = [
  "glob",
  "libc",
@@ -131,9 +131,9 @@ dependencies = [
 
 [[package]]
 name = "find-msvc-tools"
-version = "0.1.9"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
+checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de"
 
 [[package]]
 name = "flume"
@@ -155,24 +155,24 @@ checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
 
 [[package]]
 name = "futures-channel"
-version = "0.3.33"
+version = "0.3.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae"
+checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4"
 dependencies = [
  "futures-core",
 ]
 
 [[package]]
 name = "futures-core"
-version = "0.3.33"
+version = "0.3.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7"
+checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
 
 [[package]]
 name = "futures-sink"
-version = "0.3.33"
+version = "0.3.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307"
+checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d"
 
 [[package]]
 name = "getrandom"
@@ -229,9 +229,9 @@ dependencies = [
 
 [[package]]
 name = "http"
-version = "1.4.2"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
+checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
 dependencies = [
  "bytes",
  "itoa",
@@ -249,9 +249,9 @@ dependencies = [
 
 [[package]]
 name = "http-body-util"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2"
+checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c"
 dependencies = [
  "bytes",
  "futures-core",
@@ -323,9 +323,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "js-sys"
-version = "0.3.103"
+version = "0.3.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102"
+checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a"
 dependencies = [
  "cfg-if",
  "wasm-bindgen",
@@ -333,7 +333,7 @@ dependencies = [
 
 [[package]]
 name = "kino"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
  "ahash",
  "bytes",
@@ -403,9 +403,9 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
 
 [[package]]
 name = "lru"
-version = "0.18.1"
+version = "0.18.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6"
+checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a"
 dependencies = [
  "hashbrown",
 ]
@@ -430,7 +430,7 @@ checksum = "47607461fd8e1513cb4f2076c197d8092d921a1ea75bd08af97398f593751892"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -561,7 +561,7 @@ dependencies = [
  "quote",
  "regex",
  "shell-words",
- "syn",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -593,9 +593,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.16"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -630,9 +630,9 @@ checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
 
 [[package]]
 name = "rustls"
-version = "0.23.42"
+version = "0.23.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
+checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06"
 dependencies = [
  "log",
  "once_cell",
@@ -663,9 +663,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.103.13"
+version = "0.103.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
+checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a"
 dependencies = [
  "ring",
  "rustls-pki-types",
@@ -750,6 +750,17 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "syn"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "tokio"
 version = "1.53.1"
@@ -767,13 +778,13 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "2.7.1"
+version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba"
+checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 3.0.3",
 ]
 
 [[package]]
@@ -821,9 +832,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.126"
+version = "0.2.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4"
+checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -834,9 +845,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.126"
+version = "0.2.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1"
+checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -844,22 +855,22 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.126"
+version = "0.2.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e"
+checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284"
 dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.119",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.126"
+version = "0.2.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24"
+checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf"
 dependencies = [
  "unicode-ident",
 ]
@@ -960,22 +971,22 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
 
 [[package]]
 name = "zerocopy"
-version = "0.8.55"
+version = "0.8.56"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb"
+checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.55"
+version = "0.8.56"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb"
+checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.119",
 ]
 
 [[package]]

data/README.md

@@ -230,6 +230,8 @@ server = Kino::Server.new(app,
   max_body_size: 50 * 1024 * 1024,  # bytes before a 413; nil = let a proxy handle it
   on_error: ->(e, env) { ErrorTracker.capture(e) },  # after the client got its 500
   shutdown_timeout: 30,       # drain deadline
+  control_bind: "127.0.0.1:9293",   # monitoring: /stats /metrics /ready /live; port 0 reads back via server.control_port
+  control_token: ENV["KINO_CONTROL_TOKEN"],  # optional Bearer auth for /stats + /metrics
   tls: { cert: "cert.pem", key: "key.pem" },  # file paths or inline PEM
 )
 server.start
@@ -320,6 +322,44 @@ after the client got its 500—the only place a tracker sees errors
 raised while the response was being written (in `:ractor` mode, build
 the handler with `Ractor.shareable_proc`).
 
+## Lifecycle hooks
+
+Kino fires four lifecycle hooks alongside `on_error`, split by firing context.
+
+**Worker-context hooks** run inside the worker and are available to all workers:
+- `after_worker_boot { |worker_id| }`: runs once before the worker begins serving, with its slot id. In `:ractor` mode it runs inside the worker ractor and must be `Ractor.shareable_proc`.
+- `after_request_complete { |env, status| }`: fires inside the worker after each successful response. This is the hot path—leave it unset for zero cost. In `:ractor` mode it must be `Ractor.shareable_proc`.
+
+**Main-context hooks** run on the main thread, outside workers, and are plain procs:
+- `after_boot { }`: fires once after the worker pool is up. Wire readiness here—sd_notify, a "server ready" metric, and so on.
+- `on_worker_exit { |worker_index, error| }`: fires when a worker exits, with its index and the crash cause (or nil on a clean exit).
+
+`after_worker_boot`'s argument is the worker's slot id, while in `:ractor` mode `on_worker_exit`'s argument identifies the exited ractor (`0`..`workers - 1`)—a different number space—so don't correlate boot and exit by that number in `:ractor` mode.
+
+A raising hook is logged and never kills a worker.
+
+## Stuck-worker quarantine
+
+`quarantine_timeout: seconds` (or `quarantine_timeout 60` in `kino.rb`)
+quarantines a dispatch slot whose request has run longer than the deadline
+and spawns a replacement worker to restore capacity—distinct from
+`request_timeout`, which gives the client a 504 but leaves the slot
+occupied. `quarantine_max` (default: the worker count in `:ractor` mode,
+workers × threads in `:threaded`) caps the total number of replacement
+events over the process lifetime—past it the monitor stops replacing and
+the server runs at reduced capacity.
+
+The wedged worker is never interrupted or force-killed, and its slot stays
+quarantined for good. In `:threaded` mode, if the blocked thread
+eventually returns, it keeps serving requests on that same slot—but the
+slot itself stays flagged quarantined (busy_ms reported as 0) for the rest
+of the process; in `:ractor` mode the wedged ractor (and its supervisor
+thread) leaks until the process exits, since a wedged ractor cannot be
+safely interrupted. Monitor quarantine activity via `server.stats`
+(top-level `quarantined` count and per-slot `worker_status[].quarantined`
+flag), `GET /stats` (same), and `GET /metrics` (`kino_quarantined_workers`
+gauge and `kino_quarantine_replacements_total` counter).
+
 ## Stats
 
 `server.stats` returns a live snapshot: the configuration plus counters
@@ -330,7 +370,7 @@ cost):
 server.stats
 # => {mode: :ractor, lanes: false, workers: 8, threads: 1, batch: 1,
 #     respawns: 0, queued: 0, in_flight: 2, served: 1041, rejected: 0,
-#     timeouts: 0}
+#     timeouts: 0, worker_status: [...]}
 # plus lane_depths: [...] when lane dispatch is on
 ```
 
@@ -341,6 +381,39 @@ From the outside, `kill -USR1 <pid>` prints the same snapshot as one line
 Kino stats: mode=:ractor lanes=false workers=8 threads=1 batch=1 respawns=0 queued=0 in_flight=2 served=1041 rejected=0 timeouts=0
 ```
 
+For pull-based monitoring, `control_bind "127.0.0.1:9293"` (or a
+`unix://` path) serves a read-only **control plane** from the native
+layer on its own thread—it keeps answering even while every Ruby worker
+is busy or stuck, and reports `draining` through a graceful shutdown:
+
+- `GET /stats`—the same snapshot as `server.stats`, as JSON (plus
+  `state` and `version`).
+- `GET /metrics`—Prometheus text format (`kino_requests_served_total`,
+  `kino_queue_depth`, `kino_ready`, …).
+
+Both `/stats` and `/metrics` also break the counters down per dispatch
+slot: `/stats` carries a `worker_status` array (`index`, `served`,
+`in_flight`, `busy_ms`) and `/metrics` emits `kino_worker_*{worker="N"}`
+series, one entry per execution slot (`workers × threads`)—a crashed
+worker's slot is never reused, so it stays in the list with its counters
+frozen where they stopped, meaning the array (and its `worker="N"` metric
+series) grows by one across every respawn. `busy_ms` is how long the
+slot's current request has been running (0 when idle), so a single slot
+climbing while the rest sit at 0 is your stuck worker.
+
+The `/stats` response and `server.stats` carry `queue_time` (count and
+summed seconds), and `/metrics` exposes `kino_request_queue_seconds`—a
+Prometheus histogram of queue-wait time, the worker-saturation signal.
+Counts admitted requests only; a 503 after queue wait goes to `rejected`,
+not `queue_time`.
+
+- `GET /ready`—`200` when serving, `503` while booting or draining:
+  wire it to your load balancer or Kubernetes readiness probe.
+- `GET /live`—`200` whenever the process is alive: the liveness probe.
+
+`control_token "..."` puts `/stats` and `/metrics` behind
+`Authorization: Bearer`; the probes stay open.
+
 ## Logging
 
 With one log line per request, `Kino::Logger` sustained **2.4× the
@@ -442,7 +515,7 @@ For the Rust network stack, thanks to [Sean McArthur](https://github.com/seanmon
 
 ## Assisted by
 
-Claude Code (Mythos, Opus).
+Claude Code (Fable 5, Opus 4.8).
 
 ## Contributing
 

data/ext/kino/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kino"
-version = "0.2.1"
+version = "0.3.0"
 edition = "2021"
 authors = ["Yaroslav Markin <yaroslav@markin.net>"]
 license = "MIT"