kino 0.2.0-aarch64-linux → 0.3.0-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '04184f3cc709476d0cf35ceb23fbe9cbdd46f13d56f55f9e5f6a3fd261c9a8c9'
-  data.tar.gz: 261bf88fb0bd9ca45633c2141d189591e2c183a4002cd300e124b3a16e74ae8c
+  metadata.gz: 18bb12fa02179c4c2e164cf83e605ac893094650fcaee119132b48b2eea0ae2e
+  data.tar.gz: 56c6652d6671f2872e0399f82a789e6b4cc5b0ea018f5503bb91ed575f187985
 SHA512:
-  metadata.gz: 52cb0b30513607501b56c01ed343a119ad0fc1277ad17655fa1453b82bfdaa8a345cb52ce4937058a491e67db485560481d4523ca85255d7ff57546e4d1146f5
-  data.tar.gz: f1fb8a9e78b4c28c2ed3f4e589e10c40775e9605863148cac6edcd1e3f47dc2ab16b7b1e168f481e7e6ff15fc21ad545c680c732a33e5e4a4ccaadf534b0e114
+  metadata.gz: 1c3f92ba0bd339f2415ae94024f4afc5666a356c69b7721bb36deeccc492521d7f75a5ef064f994306a771924a42ccb00ff6e3ce138754b797ef5e2263144622
+  data.tar.gz: ae5f7535f0fc9ef4b9398a1fdd57723aae445d49ef7705fb1cab907d6b8ab01bb25338394509467e7cc755b37105b306ab08f89f755d4496e186e63f32cd6ace

data/CHANGELOG.md

@@ -1,3 +1,35 @@
+## [0.3.0] - 2026-08-13
+
+- Queue-time histogram: `/metrics` exposes `kino_request_queue_seconds`, a
+  histogram of how long each request waited for a free worker (the
+  saturation signal), and `server.stats`/`/stats` gain `queue_time`
+  (count and summed seconds). Measured internally with a monotonic clock,
+  so it needs no proxy header and is immune to clock skew.
+- Lifecycle hooks: `after_boot`, `after_worker_boot`,
+  `after_request_complete`, and `on_worker_exit` join `on_error`, so apps
+  can wire their own metrics, readiness, and error tracking. The
+  worker-context hooks must be Ractor-shareable in `:ractor` mode; a raising
+  hook is logged and never kills a worker.
+- Stuck-worker quarantine: past `quarantine_timeout`, a wedged dispatch
+  slot is quarantined and a replacement worker is spawned to restore
+  capacity (capped by `quarantine_max`), surfaced via `/stats`, `/metrics`,
+  and `server.stats`. The wedged worker is never force-killed.
+- Control plane: a read-only monitoring listener (`control_bind`,
+  optional `control_token`) serving live stats as JSON at `/stats`,
+  Prometheus metrics at `/metrics`, and `/ready`/`/live` probes, answered
+  from the native layer on a dedicated thread so it stays responsive
+  while workers are busy, stuck, or draining.
+- Per-worker stats: `/stats`, `/metrics`, and `server.stats` now break the
+  counters down per dispatch slot (served, in-flight, and `busy_ms`, the
+  age of the slot's current request), so a stuck slot is visible
+  individually.
+- Update Rust and Ruby dependencies.
+
+## [0.2.1] - 2026-07-27
+
+- Update Rust dependencies for Kino.
+- Update: puma 8 in benchmarks.
+
 ## [0.2.0] - 2026-07-13
 
 - Strip debug info from release builds.

data/README.md

@@ -183,7 +183,7 @@ bundle add kino      # or: gem install kino (outside a bundle)
 or put it in the `Gemfile` yourself:
 
 ```ruby
-gem "kino", "~> 0.1"
+gem "kino"
 ```
 
 Then generate a config and serve:
@@ -230,6 +230,8 @@ server = Kino::Server.new(app,
   max_body_size: 50 * 1024 * 1024,  # bytes before a 413; nil = let a proxy handle it
   on_error: ->(e, env) { ErrorTracker.capture(e) },  # after the client got its 500
   shutdown_timeout: 30,       # drain deadline
+  control_bind: "127.0.0.1:9293",   # monitoring: /stats /metrics /ready /live; port 0 reads back via server.control_port
+  control_token: ENV["KINO_CONTROL_TOKEN"],  # optional Bearer auth for /stats + /metrics
   tls: { cert: "cert.pem", key: "key.pem" },  # file paths or inline PEM
 )
 server.start
@@ -320,6 +322,44 @@ after the client got its 500—the only place a tracker sees errors
 raised while the response was being written (in `:ractor` mode, build
 the handler with `Ractor.shareable_proc`).
 
+## Lifecycle hooks
+
+Kino fires four lifecycle hooks alongside `on_error`, split by firing context.
+
+**Worker-context hooks** run inside the worker and are available to all workers:
+- `after_worker_boot { |worker_id| }`: runs once before the worker begins serving, with its slot id. In `:ractor` mode it runs inside the worker ractor and must be `Ractor.shareable_proc`.
+- `after_request_complete { |env, status| }`: fires inside the worker after each successful response. This is the hot path—leave it unset for zero cost. In `:ractor` mode it must be `Ractor.shareable_proc`.
+
+**Main-context hooks** run on the main thread, outside workers, and are plain procs:
+- `after_boot { }`: fires once after the worker pool is up. Wire readiness here—sd_notify, a "server ready" metric, and so on.
+- `on_worker_exit { |worker_index, error| }`: fires when a worker exits, with its index and the crash cause (or nil on a clean exit).
+
+`after_worker_boot`'s argument is the worker's slot id, while in `:ractor` mode `on_worker_exit`'s argument identifies the exited ractor (`0`..`workers - 1`)—a different number space—so don't correlate boot and exit by that number in `:ractor` mode.
+
+A raising hook is logged and never kills a worker.
+
+## Stuck-worker quarantine
+
+`quarantine_timeout: seconds` (or `quarantine_timeout 60` in `kino.rb`)
+quarantines a dispatch slot whose request has run longer than the deadline
+and spawns a replacement worker to restore capacity—distinct from
+`request_timeout`, which gives the client a 504 but leaves the slot
+occupied. `quarantine_max` (default: the worker count in `:ractor` mode,
+workers × threads in `:threaded`) caps the total number of replacement
+events over the process lifetime—past it the monitor stops replacing and
+the server runs at reduced capacity.
+
+The wedged worker is never interrupted or force-killed, and its slot stays
+quarantined for good. In `:threaded` mode, if the blocked thread
+eventually returns, it keeps serving requests on that same slot—but the
+slot itself stays flagged quarantined (busy_ms reported as 0) for the rest
+of the process; in `:ractor` mode the wedged ractor (and its supervisor
+thread) leaks until the process exits, since a wedged ractor cannot be
+safely interrupted. Monitor quarantine activity via `server.stats`
+(top-level `quarantined` count and per-slot `worker_status[].quarantined`
+flag), `GET /stats` (same), and `GET /metrics` (`kino_quarantined_workers`
+gauge and `kino_quarantine_replacements_total` counter).
+
 ## Stats
 
 `server.stats` returns a live snapshot: the configuration plus counters
@@ -330,7 +370,7 @@ cost):
 server.stats
 # => {mode: :ractor, lanes: false, workers: 8, threads: 1, batch: 1,
 #     respawns: 0, queued: 0, in_flight: 2, served: 1041, rejected: 0,
-#     timeouts: 0}
+#     timeouts: 0, worker_status: [...]}
 # plus lane_depths: [...] when lane dispatch is on
 ```
 
@@ -341,6 +381,39 @@ From the outside, `kill -USR1 <pid>` prints the same snapshot as one line
 Kino stats: mode=:ractor lanes=false workers=8 threads=1 batch=1 respawns=0 queued=0 in_flight=2 served=1041 rejected=0 timeouts=0
 ```
 
+For pull-based monitoring, `control_bind "127.0.0.1:9293"` (or a
+`unix://` path) serves a read-only **control plane** from the native
+layer on its own thread—it keeps answering even while every Ruby worker
+is busy or stuck, and reports `draining` through a graceful shutdown:
+
+- `GET /stats`—the same snapshot as `server.stats`, as JSON (plus
+  `state` and `version`).
+- `GET /metrics`—Prometheus text format (`kino_requests_served_total`,
+  `kino_queue_depth`, `kino_ready`, …).
+
+Both `/stats` and `/metrics` also break the counters down per dispatch
+slot: `/stats` carries a `worker_status` array (`index`, `served`,
+`in_flight`, `busy_ms`) and `/metrics` emits `kino_worker_*{worker="N"}`
+series, one entry per execution slot (`workers × threads`)—a crashed
+worker's slot is never reused, so it stays in the list with its counters
+frozen where they stopped, meaning the array (and its `worker="N"` metric
+series) grows by one across every respawn. `busy_ms` is how long the
+slot's current request has been running (0 when idle), so a single slot
+climbing while the rest sit at 0 is your stuck worker.
+
+The `/stats` response and `server.stats` carry `queue_time` (count and
+summed seconds), and `/metrics` exposes `kino_request_queue_seconds`—a
+Prometheus histogram of queue-wait time, the worker-saturation signal.
+Counts admitted requests only; a 503 after queue wait goes to `rejected`,
+not `queue_time`.
+
+- `GET /ready`—`200` when serving, `503` while booting or draining:
+  wire it to your load balancer or Kubernetes readiness probe.
+- `GET /live`—`200` whenever the process is alive: the liveness probe.
+
+`control_token "..."` puts `/stats` and `/metrics` behind
+`Authorization: Bearer`; the probes stay open.
+
 ## Logging
 
 With one log line per request, `Kino::Logger` sustained **2.4× the
@@ -432,9 +505,17 @@ bundle exec rake                       # compile, Rust tests, specs, RBS, lint
 RB_SYS_CARGO_PROFILE=dev bundle exec rake compile   # fast dev rebuilds
 ```
 
+## Acknowledgements
+
+Thanks to [Mat Sadler](https://github.com/matsadler) for [magnus](https://github.com/matsadler/magnus).
+
+For ractors, thanks to [Koichi Sasada](https://github.com/ko1), [John Hawthorn](https://github.com/jhawthorn), [Jean Boussier](https://github.com/byroot), [Luke Gruber](https://github.com/luke-gruber), and other Ruby core contributors.
+
+For the Rust network stack, thanks to [Sean McArthur](https://github.com/seanmonstar) for [hyper](https://github.com/hyperium/hyper), and to [Carl Lerche](https://github.com/carllerche), [Alice Ryhl](https://github.com/Darksonn), and the other [Tokio](https://github.com/tokio-rs/tokio) maintainers for the runtime underneath it. Thanks to [Joshua Barretto](https://github.com/zesterer) for [flume](https://github.com/zesterer/flume)—its channels carry every request between the network side and the workers.
+
 ## Assisted by
 
-Claude Code (Mythos, Opus).
+Claude Code (Fable 5, Opus 4.8).
 
 ## Contributing
 

data/lib/kino/cli.rb

@@ -100,11 +100,14 @@ module Kino
       end.join
     end
 
-    # One-line stats dump (the SIGUSR1 handler's output).
+    # One-line stats dump (the SIGUSR1 handler's output). Excludes
+    # worker_status: it's an array with one entry per execution slot, and
+    # printing it inline would break the one-line contract (see /stats for
+    # per-worker detail).
     # @param stats [Hash{Symbol => Object}] see {Kino::Server#stats}
     # @return [String]
     def stats_line(stats)
-      dim("Kino stats: #{stats.map { |k, v| "#{k}=#{v.inspect}" }.join(" ")}")
+      dim("Kino stats: #{stats.except(:worker_status).map { |k, v| "#{k}=#{v.inspect}" }.join(" ")}")
     end
 
     # The two banner halves around Server#start: credits before, the ready

data/lib/kino/configuration.rb

@@ -23,11 +23,19 @@ module Kino
       lanes: false,
       log_requests: false,
       on_error: nil,
+      after_boot: nil,
+      after_worker_boot: nil,
+      after_request_complete: nil,
+      on_worker_exit: nil,
       shutdown_timeout: 30,
       tokio_threads: nil,
       tls: nil,
       environment: nil,
       pidfile: nil,
+      control_bind: nil,
+      control_token: nil,
+      quarantine_timeout: nil,
+      quarantine_max: nil,
       rackup: nil
     }.freeze
 
@@ -185,6 +193,24 @@ module Kino
       # or a block. Must be Ractor-shareable in :ractor mode.
       def on_error(handler = nil, &block) = @config.set(:on_error, handler || block)
 
+      # Called once on the main thread after the worker pool is up. The
+      # readiness seam (wire sd_notify or a "server ready" metric here).
+      def after_boot(handler = nil, &block) = @config.set(:after_boot, handler || block)
+
+      # Called once inside each worker (a ractor in :ractor mode) before it
+      # serves, with the worker's slot id. Must be Ractor-shareable in
+      # :ractor mode (build it with Ractor.shareable_proc).
+      def after_worker_boot(handler = nil, &block) = @config.set(:after_worker_boot, handler || block)
+
+      # Called inside the worker after each successful response with
+      # (env, status). Hot path: leave unset for zero cost. Must be
+      # Ractor-shareable in :ractor mode.
+      def after_request_complete(handler = nil, &block) = @config.set(:after_request_complete, handler || block)
+
+      # Called on the main thread when a worker exits, with (worker_index,
+      # error_or_nil). error is the crash cause, or nil on a clean exit.
+      def on_worker_exit(handler = nil, &block) = @config.set(:on_worker_exit, handler || block)
+
       # Graceful-shutdown drain deadline in seconds.
       def shutdown_timeout(seconds) = @config.set(:shutdown_timeout, seconds)
 
@@ -200,6 +226,37 @@ module Kino
       # Write the master PID here on start.
       def pidfile(path) = @config.set(:pidfile, path.to_s)
 
+      # Serve the read-only control plane (live stats as JSON at /stats,
+      # Prometheus text at /metrics, /ready and /live probes) on this
+      # address: "host:port" or "unix://path". Off unless set.
+      def control_bind(addr) = @config.set(:control_bind, addr.to_s)
+
+      # When set, /stats and /metrics require "Authorization: Bearer <token>".
+      # The probes stay open; they carry no data.
+      def control_token(token) = @config.set(:control_token, token.to_s)
+
+      # Quarantine a dispatch slot whose current request has run longer
+      # than this many seconds, spawning a replacement to restore capacity.
+      # Off unless set. Set it above your slowest legitimate endpoint (and
+      # typically above request_timeout).
+      def quarantine_timeout(seconds)
+        seconds &&= Float(seconds)
+        if seconds && seconds <= 0
+          raise ArgumentError, "quarantine_timeout must be greater than 0 (got #{seconds})"
+        end
+        @config.set(:quarantine_timeout, seconds)
+      end
+
+      # Cap on the total number of replacement events over the process
+      # lifetime. Past the cap the monitor stops replacing and the server
+      # runs at reduced capacity. Default: the worker count in :ractor
+      # mode, workers x threads in :threaded.
+      def quarantine_max(count)
+        count = Integer(count)
+        raise ArgumentError, "quarantine_max must be >= 1 (got #{count})" if count < 1
+        @config.set(:quarantine_max, count)
+      end
+
       # Rackup file the `kino` CLI loads (positional argument wins).
       def rackup(path) = @config.set(:rackup, path.to_s)
     end

data/lib/kino/hook_fire.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Kino
+  # @private
+  # Fires a lifecycle hook and turns a raise into a logged line instead of
+  # letting it escape. Stateless and touches only its arguments plus
+  # Native.log_error (already called from inside worker ractors today), so
+  # it is safe to call from worker context: no main-ractor state is
+  # captured.
+  module HookFire
+    module_function
+
+    def fire(hook, name, *args)
+      return unless hook
+
+      begin
+        hook.call(*args)
+      rescue => e
+        Native.log_error("#{name} hook raised #{e.class}: #{e.message}")
+      end
+    end
+  end
+end

data/lib/kino/quarantine_monitor.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Kino
+  # @private
+  # Polls per-slot busy_ms and, past the timeout, quarantines a wedged slot
+  # and asks the replacer to spawn a fresh worker. Runs one thread on the
+  # main ractor (uncontended by wedged worker ractors, so it stays
+  # responsive in :ractor mode). Never interrupts the wedged worker.
+  class QuarantineMonitor
+    def initialize(server_id:, timeout_ms:, max:, replacer:, tick: 0.5)
+      @server_id = server_id
+      @timeout_ms = timeout_ms
+      @max = max
+      @replacer = replacer
+      @tick = tick
+      @outstanding = 0
+      @at_cap_logged = false
+      @running = false
+      @thread = nil
+    end
+
+    def start
+      @running = true
+      @thread = Thread.new { run }
+      self
+    end
+
+    def stop
+      @running = false
+      @thread&.join(@tick * 2)
+    end
+
+    private
+
+    def run
+      tick while @running
+    rescue => e
+      Native.log_error("quarantine monitor crashed: #{e.class}: #{e.message}")
+    end
+
+    def tick
+      scan_slots
+    rescue => e
+      # A bad tick must never kill the monitor.
+      Native.log_error("quarantine tick error: #{e.class}: #{e.message}")
+    ensure
+      sleep @tick
+    end
+
+    def scan_slots
+      Native.worker_stats(@server_id).each do |index, _served, _in_flight, busy_ms, quarantined|
+        next if quarantined || busy_ms <= @timeout_ms
+
+        if @outstanding >= @max
+          unless @at_cap_logged
+            Native.log_error("quarantine at cap (#{@max}); serving at reduced capacity")
+            @at_cap_logged = true
+          end
+          next
+        end
+
+        if @replacer.replace(index)
+          Native.record_quarantine_replacement(@server_id)
+          @outstanding += 1
+          @at_cap_logged = false
+        end
+      end
+    end
+  end
+end

data/lib/kino/ractor_supervisor.rb

@@ -7,19 +7,23 @@ module Kino
   # ractor, Exception from app code included) wakes it to 500 the in-flight
   # requests and respawn. Clean exits (queue drained) end supervision.
   class RactorSupervisor
-    attr_reader :respawns
-
-    def initialize(server_id, app, workers:, threads:, batch: 1, on_error: nil)
+    def initialize(server_id, app, workers:, threads:, batch: 1, hooks: nil, on_worker_exit: nil)
       @server_id = server_id
       @app = app
       @workers = workers
       @threads = threads
       @batch = batch
-      @on_error = on_error
-      @respawns = 0
+      @hooks = hooks
+      @on_worker_exit = on_worker_exit
       @draining = false
       @lock = Mutex.new
       @supervisor_threads = []
+      @worker_slots = {}
+      @slot_to_worker = {}
+      @replaced = {}
+      # The first replacement's index; `replace` increments before using it,
+      # so this starts one below the first free index (@workers).
+      @next_worker_index = @workers - 1
     end
 
     def start
@@ -32,20 +36,55 @@ module Kino
     def shutdown(timeout)
       @lock.synchronize { @draining = true }
       deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
-      @supervisor_threads.each do |thread|
+      @lock.synchronize { @supervisor_threads.dup }.each do |thread|
         remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
         thread.join([remaining, 0.01].max)
       end
     end
 
     def done?
-      @supervisor_threads.none?(&:alive?)
+      @lock.synchronize { @supervisor_threads.dup }.none?(&:alive?)
     end
 
     # Block until the workers exit on their own (drain elsewhere): join
     # without flipping the draining flag.
     def join
-      @supervisor_threads.each(&:join)
+      @lock.synchronize { @supervisor_threads.dup }.each(&:join)
+    end
+
+    # Replace the ractor owning slot `worker_id`: spawn a fresh supervised
+    # ractor, then quarantine the old ractor's slots. The old supervisor
+    # thread stays blocked in ractor.value on the wedged ractor (it and the
+    # ractor leak until process exit; a wedged ractor cannot be
+    # force-killed). Returns true if a replacement was spawned.
+    def replace(worker_id)
+      worker_index = @lock.synchronize { @slot_to_worker[worker_id] }
+      return false unless worker_index
+
+      # Idempotent per ractor: a stale monitor snapshot can list two sibling
+      # slots of the same ractor, only the first replaces it.
+      claimed = @lock.synchronize do
+        if @replaced.key?(worker_index)
+          false
+        else
+          @replaced[worker_index] = true
+          true
+        end
+      end
+      return false unless claimed
+
+      new_index = @lock.synchronize { @next_worker_index += 1 }
+      thread =
+        begin
+          supervise(new_index) # spawn FIRST, nothing quarantined yet
+        rescue
+          @lock.synchronize { @replaced.delete(worker_index) } # allow retry next tick
+          raise
+        end
+      slot_ids = @lock.synchronize { @worker_slots[worker_index] } || []
+      slot_ids.each { |id| Native.quarantine_slot(@server_id, id) } # quarantine only after success
+      @lock.synchronize { @supervisor_threads << thread }
+      true
     end
 
     private
@@ -54,20 +93,22 @@ module Kino
       Thread.new do
         crashes = 0
         loop do
-          ractor, worker_ids = spawn_worker
+          ractor, worker_ids = spawn_worker(index)
           begin
             ractor.value # blocks until the ractor terminates
+            HookFire.fire(@on_worker_exit, "on_worker_exit", index, nil) # clean exit: queue drained
             break        # clean exit: queue closed, workers drained
           rescue Ractor::Error => e
             # The ractor died mid-flight. Anything it was serving will never
             # be answered by Ruby: 500 those clients NOW (not when GC gets
             # around to dropping the dead heap), then decide on respawn.
             worker_ids.each { |id| Native.abort_inflight(@server_id, id) }
+            cause = (e.respond_to?(:cause) && e.cause) ? e.cause : e
+            HookFire.fire(@on_worker_exit, "on_worker_exit", index, cause)
             break if draining?
 
             crashes += 1
-            @lock.synchronize { @respawns += 1 }
-            cause = (e.respond_to?(:cause) && e.cause) ? e.cause : e
+            Native.record_respawn(@server_id)
             Native.log_error("worker ractor #{index} crashed (#{cause.class}: #{cause.message}); respawning")
             # Policy (crash recovery): unlimited respawn
             # keeps the server up under rare crashes but turns a
@@ -82,15 +123,19 @@ module Kino
     # Fresh ractor + fresh native slots. Slots are never reused across
     # respawns: stale interrupt kicks and dead weak refs go down with the
     # old slot.
-    def spawn_worker
+    def spawn_worker(worker_index)
       worker_ids = Array.new(@threads) { Native.register_worker(@server_id) }
-      ractor = Ractor.new(@server_id, worker_ids, @app, @batch, @on_error) do |server_id, ids, app, batch, on_error|
+      @lock.synchronize do
+        @worker_slots[worker_index] = worker_ids
+        worker_ids.each { |id| @slot_to_worker[id] = worker_index }
+      end
+      ractor = Ractor.new(@server_id, worker_ids, @app, @batch, @hooks) do |server_id, ids, app, batch, hooks|
         ids.map do |id|
           Thread.new do
             # Crashes surface via Ractor#value in the supervisor; don't also
             # spray the backtrace to stderr from inside the dying ractor.
             Thread.current.report_on_exception = false
-            Kino::Worker.run(server_id, id, app, batch, on_error)
+            Kino::Worker.run(server_id, id, app, batch, hooks)
           end
         end.each(&:join)
       end

data/lib/kino/server.rb

@@ -12,6 +12,10 @@ module Kino
     #   port when configured with port 0)
     attr_reader :port
 
+    # @return [Integer, nil] the control plane's TCP port (nil until #start,
+    #   when the control plane is off, or for a unix-socket bind)
+    attr_reader :control_port
+
     # @return [Symbol] the resolved dispatch mode, :ractor or :threaded
     attr_reader :mode
 
@@ -41,8 +45,17 @@ module Kino
       @bind = settings[:bind]
       @requested_port = settings[:port]
       @workers = Integer(settings[:workers])
-      @on_error = validate_on_error(settings[:on_error])
+      @on_error = validate_hook(settings[:on_error], :on_error)
+      @after_worker_boot = validate_hook(settings[:after_worker_boot], :after_worker_boot)
+      @after_request_complete = validate_hook(settings[:after_request_complete], :after_request_complete)
+      @after_boot = validate_hook(settings[:after_boot], :after_boot)
+      @on_worker_exit = validate_hook(settings[:on_worker_exit], :on_worker_exit)
       @mode = resolve_mode(settings[:mode])
+      @worker_hooks = WorkerHooks.new(
+        on_error: @on_error,
+        after_worker_boot: @after_worker_boot,
+        after_request_complete: @after_request_complete
+      )
       # Default threads per mode: 1 in :ractor (threads inside a ractor
       # share its lock; a measured +17% on fast handlers; raise `workers`
       # for I/O concurrency instead), 3 in :threaded (threads ARE the
@@ -60,8 +73,25 @@ module Kino
       @tokio_threads = settings[:tokio_threads]
       @tls = validate_tls(settings[:tls])
       @pidfile = settings[:pidfile]
+      @control_bind = settings[:control_bind]&.to_s
+      @control_token = settings[:control_token]&.to_s
+      # An empty token (e.g. control_token ENV["KINO_CONTROL_TOKEN"] with the
+      # var unset) must not half-disable auth: treat it as auth off, not as
+      # "require a zero-length Bearer token".
+      @control_token = nil if @control_token && @control_token.empty?
+      @quarantine_timeout_ms = settings[:quarantine_timeout] ? (Float(settings[:quarantine_timeout]) * 1000).round : nil
+      @quarantine_max =
+        if settings[:quarantine_max]
+          Integer(settings[:quarantine_max])
+        elsif @mode == :ractor
+          @workers
+        else
+          @workers * @threads
+        end
       @worker_threads = []
+      @worker_threads_lock = Mutex.new
       @supervisor = nil
+      @quarantine_monitor = nil
       @started = false
     end
 
@@ -78,7 +108,7 @@ module Kino
       write_pidfile if @pidfile
       booted = false
       begin
-        @id, @port = Native.server_start(
+        @id, @port, @control_port = Native.server_start(
           bind: @bind, port: @requested_port,
           queue_depth: @queue_depth, queue_timeout_ms: @queue_timeout_ms,
           request_timeout_ms: @request_timeout_ms,
@@ -86,7 +116,9 @@ module Kino
           max_body_size: @max_body_size,
           tokio_threads: @tokio_threads,
           tls_cert: @tls&.fetch(:cert), tls_key: @tls&.fetch(:key),
-          lanes: @lanes, log_requests: @log_requests
+          lanes: @lanes, log_requests: @log_requests,
+          mode: @mode.to_s, workers: @workers, threads: @threads, batch: @batch,
+          control_bind: @control_bind, control_token: @control_token
         )
         booted = true
       ensure
@@ -97,13 +129,13 @@ module Kino
       @pin_keeper = Native.pin_keeper(@id)
       if @mode == :ractor
         @supervisor = RactorSupervisor.new(@id, @app, workers: @workers, threads: @threads,
-          batch: @batch, on_error: @on_error).start
+          batch: @batch, hooks: @worker_hooks, on_worker_exit: @on_worker_exit).start
       else
-        @worker_threads = (@workers * @threads).times.map do
-          worker_id = Native.register_worker(@id)
-          Thread.new { Worker.run(@id, worker_id, @app, @batch, @on_error) }
-        end
+        @worker_threads = (@workers * @threads).times.map { spawn_worker_thread }
       end
+      start_quarantine_monitor if @quarantine_timeout_ms
+      Native.control_ready(@id)
+      HookFire.fire(@after_boot, "after_boot")
       @started = true
       self
     end
@@ -119,6 +151,7 @@ module Kino
     def shutdown(timeout: nil)
       return unless @started
 
+      @quarantine_monitor&.stop
       deadline = monotonic_now + (timeout || @shutdown_timeout)
       Native.stop_accepting(@id)
 
@@ -144,6 +177,9 @@ module Kino
       end
 
       Native.shutdown_runtime(@id, 1_000)
+      # The control thread reports "draining" for the whole drain and stops
+      # only now, once there is nothing left to report.
+      Native.control_stop(@id)
       # The runtime is gone, so hyper has dropped every pinned buffer;
       # the keeper (and the strings it marked) may now be collected.
       @pin_keeper = nil
@@ -205,22 +241,88 @@ module Kino
     #
     # @return [Hash{Symbol => Object}] mode, lanes, workers, threads,
     #   batch, respawns; plus queued, in_flight, served, rejected,
-    #   timeouts (and lane_depths in lanes mode) once started
+    #   timeouts, worker_status, quarantined, queue_time (and lane_depths in
+    #   lanes mode) once started
     def stats
       base = {
         mode: @mode, lanes: @lanes, workers: @workers, threads: @threads,
-        batch: @batch, respawns: @supervisor ? @supervisor.respawns : 0
+        batch: @batch, respawns: 0
       }
       return base unless @started
 
-      queued, in_flight, served, rejected, timeouts, lane_depths = Native.server_stats(@id)
-      base.merge!(queued:, in_flight:, served:, rejected:, timeouts:)
+      queued, in_flight, served, rejected, timeouts, respawns, lane_depths = Native.server_stats(@id)
+      base.merge!(queued:, in_flight:, served:, rejected:, timeouts:, respawns:)
       base[:lane_depths] = lane_depths if lane_depths
+      rows = Native.worker_stats(@id)
+      base[:worker_status] = rows.map do |index, served, in_flight, busy_ms, quarantined|
+        {index:, served:, in_flight:, busy_ms:, quarantined:}
+      end
+      base[:quarantined] = rows.count { |_index, _served, _in_flight, _busy_ms, quarantined| quarantined }
+      count, sum_seconds = Native.queue_time(@id)
+      base[:queue_time] = {count:, sum_seconds:}
       base
     end
 
     private
 
+    # Register a fresh dispatch slot and run a worker thread on it; returns
+    # the thread. Used at boot and by the quarantine replacer.
+    def spawn_worker_thread
+      worker_id = Native.register_worker(@id)
+      Thread.new do
+        error = nil
+        begin
+          Worker.run(@id, worker_id, @app, @batch, @worker_hooks)
+        rescue Exception => e # rubocop:disable Lint/RescueException -- a hard crash in a threaded worker thread
+          error = e
+          raise
+        ensure
+          HookFire.fire(@on_worker_exit, "on_worker_exit", worker_id, error)
+        end
+      end
+    end
+
+    # Track a replacement thread spawned outside the initial pool assignment
+    # (the quarantine replacer) so shutdown's join/done?/kill sweeps see it.
+    def track_replacement_thread(thread)
+      @worker_threads_lock.synchronize { @worker_threads << thread }
+    end
+
+    # @private
+    # The :threaded-mode quarantine replacer: spawns a replacement worker
+    # thread, quarantines the wedged slot, then tracks the new thread so
+    # shutdown's join/done?/kill sweeps see it. Built from bound Method
+    # objects instead of a server reference, so it drives the server
+    # through those methods without send or instance_variable_get.
+    class ThreadedReplacer
+      def initialize(server_id:, spawner:, tracker:)
+        @server_id = server_id
+        @spawner = spawner
+        @tracker = tracker
+      end
+
+      def replace(worker_id)
+        thread = @spawner.call # spawn FIRST (may raise ThreadError)
+        Native.quarantine_slot(@server_id, worker_id) # quarantine after success
+        @tracker.call(thread)
+        true
+      end
+    end
+    private_constant :ThreadedReplacer
+
+    # A replacer.replace(worker_id) spawns a replacement worker, then
+    # quarantines the wedged slot, mode-appropriately. In :ractor the
+    # supervisor is the replacer; in :threaded a small object over
+    # spawn_worker_thread.
+    def start_quarantine_monitor
+      replacer = @supervisor || ThreadedReplacer.new(server_id: @id, spawner: method(:spawn_worker_thread),
+        tracker: method(:track_replacement_thread))
+      @quarantine_monitor = QuarantineMonitor.new(
+        server_id: @id, timeout_ms: @quarantine_timeout_ms,
+        max: @quarantine_max, replacer: replacer
+      ).start
+    end
+
     def validate_tls(tls)
       return nil if tls.nil?
       unless tls.is_a?(Hash) && tls[:cert] && tls[:key]
@@ -230,10 +332,10 @@ module Kino
       {cert: String(tls[:cert]), key: String(tls[:key])}
     end
 
-    def validate_on_error(handler)
+    def validate_hook(handler, name)
       return nil if handler.nil?
       unless handler.respond_to?(:call)
-        raise ArgumentError, "on_error must respond to #call (got #{handler.class})"
+        raise ArgumentError, "#{name} must respond to #call (got #{handler.class})"
       end
 
       handler
@@ -320,7 +422,8 @@ module Kino
       if @supervisor
         @supervisor.shutdown([deadline - monotonic_now, 0].max)
       else
-        @worker_threads.each do |thread|
+        threads = @worker_threads_lock.synchronize { @worker_threads.dup }
+        threads.each do |thread|
           thread.join([deadline - monotonic_now, 0.01].max)
         end
       end
@@ -330,7 +433,8 @@ module Kino
       if @supervisor
         @supervisor.done?
       else
-        @worker_threads.none?(&:alive?)
+        threads = @worker_threads_lock.synchronize { @worker_threads.dup }
+        threads.none?(&:alive?)
       end
     end
 
@@ -340,7 +444,8 @@ module Kino
         # by abort_all_inflight. The stuck ractor leaks until process exit.
         Native.log_error("shutdown deadline passed with stuck ractor workers") unless @supervisor.done?
       else
-        @worker_threads.each { |thread| thread.kill if thread.alive? }
+        threads = @worker_threads_lock.synchronize { @worker_threads.dup }
+        threads.each { |thread| thread.kill if thread.alive? }
       end
     end
 
@@ -361,9 +466,9 @@ module Kino
             "Ractor.shareable_proc endpoints); try Ractor.make_shareable(app) " \
             "or mode: :threaded"
         end
-        unless @on_error.nil? || Ractor.shareable?(@on_error)
+        if (name = unshareable_worker_hook_name)
           raise Error,
-            "mode: :ractor requires a Ractor-shareable on_error handler " \
+            "mode: :ractor requires a Ractor-shareable #{name} hook " \
             "(build it with Ractor.shareable_proc, or use mode: :threaded)"
         end
         :ractor
@@ -371,8 +476,8 @@ module Kino
         if !Ractor.shareable?(@app)
           warn "Kino: app is not Ractor-shareable; falling back to mode: :threaded"
           :threaded
-        elsif !(@on_error.nil? || Ractor.shareable?(@on_error))
-          warn "Kino: on_error handler is not Ractor-shareable; falling back to mode: :threaded"
+        elsif (name = unshareable_worker_hook_name)
+          warn "Kino: #{name} hook is not Ractor-shareable; falling back to mode: :threaded"
           :threaded
         else
           :ractor
@@ -381,5 +486,22 @@ module Kino
         raise ArgumentError, "mode must be :auto, :ractor, or :threaded (got #{requested.inspect})"
       end
     end
+
+    # The hooks that ride into worker context (a ractor in :ractor mode)
+    # and so must be Ractor-shareable there. after_boot and on_worker_exit
+    # run on the main thread and are exempt.
+    def worker_context_hooks
+      [[:on_error, @on_error], [:after_worker_boot, @after_worker_boot],
+        [:after_request_complete, @after_request_complete]]
+    end
+
+    # The name of the first worker-context hook that is set but not
+    # Ractor-shareable, or nil if all set ones are. Used by both the
+    # :ractor raise and the :auto warn/fallback branches, so "first
+    # offender" is defined once.
+    def unshareable_worker_hook_name
+      bad = worker_context_hooks.find { |_name, hook| !(hook.nil? || Ractor.shareable?(hook)) }
+      bad&.first
+    end
   end
 end

data/lib/kino/templates/kino.rb.tt

@@ -87,6 +87,27 @@
 # must be Ractor-shareable (build it with Ractor.shareable_proc).
 # on_error ->(error, env) { ExceptionService.capture(error) }
 
+# Called once on the main thread after the worker pool is up. Wire
+# readiness here (sd_notify, a "server ready" metric).
+# after_boot { }
+
+# Called once inside each worker before it serves, with the worker's slot
+# id. In :ractor mode it runs inside the worker ractor, so it must be
+# Ractor-shareable (build it with Ractor.shareable_proc).
+# after_worker_boot { |worker_id| }
+
+# Called inside the worker after each successful response, with (env,
+# status). This is the hot path: leave it unset for zero cost. In :ractor
+# mode it must be Ractor-shareable.
+# after_request_complete { |env, status| }
+
+# Called on the main thread when a worker exits, with (worker_index,
+# error). error is the crash cause, or nil on a clean exit. In :ractor
+# mode worker_index identifies the exited ractor (0..workers - 1), a
+# different number space than after_worker_boot's slot id: do not
+# correlate boot and exit by that number in :ractor mode.
+# on_worker_exit { |worker_index, error| }
+
 ## Lifecycle
 
 # On shutdown, give in-flight requests this many seconds to finish.
@@ -102,6 +123,36 @@
 # heavily CPU-bound apps, try 1 to leave more cores for Ruby.
 # tokio_threads 4
 
+## Control plane
+
+# Serve read-only monitoring on a separate address: live stats as JSON
+# at /stats, Prometheus text at /metrics, and /ready and /live probes
+# for an orchestrator. Answered by the native layer on its own thread,
+# so it stays live even when every Ruby worker is busy or stuck.
+# Accepts "host:port" or "unix://path". Off unless set.
+# control_bind "127.0.0.1:9293"
+
+# When set, /stats and /metrics require "Authorization: Bearer <token>".
+# The probes stay open; they carry no data.
+# control_token ENV["KINO_CONTROL_TOKEN"]
+
+## Quarantine
+
+# Quarantine a dispatch slot whose current request has run longer than
+# this many seconds and spawn a replacement to restore capacity. The
+# wedged worker is never interrupted or force-killed; its slot stays
+# quarantined for good (in :ractor mode the ractor itself leaks until
+# process exit, since a wedged ractor cannot be safely interrupted). Off
+# unless set; set it above your slowest legitimate endpoint, and above
+# request_timeout.
+# quarantine_timeout 60
+
+# Cap on the total number of replacement events over the process
+# lifetime. Past the cap the monitor stops replacing and the server runs
+# at reduced capacity. Default: the worker count in :ractor mode, workers
+# x threads in :threaded.
+# quarantine_max 8
+
 ## App
 
 # Rackup file to load (a command-line argument wins).

data/lib/kino/version.rb

@@ -2,5 +2,5 @@
 
 module Kino
   # The gem version (single source of truth; ext/kino/Cargo.toml syncs).
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/kino/worker.rb

@@ -18,13 +18,14 @@ module Kino
 
     module_function
 
-    def run(server_id, worker_id, app, batch_size = 1, on_error = nil)
+    def run(server_id, worker_id, app, batch_size = 1, hooks = nil)
+      fire_after_worker_boot(hooks, worker_id)
       if batch_size <= 1
         env = Native.take_one(server_id, worker_id)
-        env = handle_one(env, server_id, worker_id, app, on_error) while env
+        env = handle_one(env, server_id, worker_id, app, hooks) while env
       else
         batch = Native.take_batch(server_id, worker_id, batch_size)
-        batch = process(batch, server_id, worker_id, app, batch_size, on_error) while batch
+        batch = process(batch, server_id, worker_id, app, batch_size, hooks) while batch
       end
     end
 
@@ -34,8 +35,8 @@ module Kino
     NOT_FUSED = Object.new.freeze
 
     # Handle one request; returns the next env (fused take) or nil.
-    def handle_one(env, server_id, worker_id, app, on_error)
-      result = serve(env, app, on_error) do |request, status, headers, chunks|
+    def handle_one(env, server_id, worker_id, app, hooks)
+      result = serve(env, app, hooks) do |request, status, headers, chunks|
         request.respond_and_take_one(server_id, worker_id, status, headers, chunks)
       end
       result.equal?(NOT_FUSED) ? Native.take_one(server_id, worker_id) : result
@@ -43,10 +44,10 @@ module Kino
 
     # Handle every env in the batch; returns the next batch (the last
     # simple response rides the fused respond_and_take) or nil on shutdown.
-    def process(batch, server_id, worker_id, app, batch_size, on_error)
+    def process(batch, server_id, worker_id, app, batch_size, hooks)
       last = batch.size - 1
       batch.each_with_index do |env, index|
-        result = serve(env, app, on_error) do |request, status, headers, chunks|
+        result = serve(env, app, hooks) do |request, status, headers, chunks|
           if index == last
             request.respond_and_take(server_id, worker_id, batch_size,
               status, headers, chunks)
@@ -66,17 +67,30 @@ module Kino
     # here and return NOT_FUSED. App errors must never kill the worker;
     # hard crashes (Exception) are the supervisor's job; and `abort` does
     # the right thing whether or not the response head already went out.
-    def serve(env, app, on_error)
+    def serve(env, app, hooks)
       request = env[KINO_REQUEST]
       env[RACK_INPUT] ||= Input.new(request)
       status, headers, body = app.call(env)
 
       if body.respond_to?(:to_ary)
-        result = yield(request, status.to_i, headers, join_chunks(body.to_ary))
-        body.close if body.respond_to?(:close)
-        result
+        chunks = join_chunks(body.to_ary)
+        if hooks&.after_request_complete
+          # Hook set: do not fuse. Send the complete response, fire the hook
+          # after it is out, then signal the caller to take the next request
+          # separately (so the hook never waits on the next request).
+          request.send_simple(status.to_i, headers, chunks)
+          body.close if body.respond_to?(:close)
+          fire_after_request_complete(hooks, env, status.to_i)
+          NOT_FUSED
+        else
+          # No hook: fused fast path, unchanged, zero cost.
+          result = yield(request, status.to_i, headers, chunks)
+          body.close if body.respond_to?(:close)
+          result
+        end
       else
         deliver_streaming(request, status.to_i, headers, body, env[RACK_INPUT])
+        fire_after_request_complete(hooks, env, status.to_i)
         NOT_FUSED
       end
     rescue => e
@@ -87,13 +101,7 @@ module Kino
       # because nothing may escape this block and kill the worker.
       Native.log_error(error_log_line(e))
       request.abort
-      if on_error
-        begin
-          on_error.call(e, env)
-        rescue => hook_error
-          Native.log_error("on_error hook raised #{hook_error.class}: #{hook_error.message}")
-        end
-      end
+      HookFire.fire(hooks&.on_error, "on_error", e, env)
       NOT_FUSED
     end
 
@@ -142,7 +150,16 @@ module Kino
       joined
     end
 
+    def fire_after_worker_boot(hooks, worker_id)
+      HookFire.fire(hooks&.after_worker_boot, "after_worker_boot", worker_id)
+    end
+
+    def fire_after_request_complete(hooks, env, status)
+      HookFire.fire(hooks&.after_request_complete, "after_request_complete", env, status)
+    end
+
     private_class_method :handle_one, :process, :serve, :deliver_streaming,
-      :join_chunks, :error_log_line
+      :join_chunks, :error_log_line, :fire_after_worker_boot,
+      :fire_after_request_complete
   end
 end

data/lib/kino/worker_hooks.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Kino
+  # @private
+  # The worker-context lifecycle hooks, bundled so one frozen, shareable
+  # value crosses into each worker (a ractor in :ractor mode) instead of
+  # several bare procs. Any member may be nil. A Data instance is frozen,
+  # so it is Ractor.shareable? exactly when its members are (nil, or a
+  # Ractor.shareable_proc), letting it ride the ractor boundary like the app.
+  WorkerHooks = Data.define(:on_error, :after_worker_boot, :after_request_complete)
+end

data/lib/kino.rb

@@ -43,8 +43,11 @@ require_relative "kino/null_input"
 require_relative "kino/errors_stream"
 require_relative "kino/stream"
 require_relative "kino/configuration"
+require_relative "kino/hook_fire"
+require_relative "kino/worker_hooks"
 require_relative "kino/worker"
 require_relative "kino/ractor_supervisor"
+require_relative "kino/quarantine_monitor"
 require_relative "kino/server"
 
 # Hand the frozen shareable singletons to the native layer: it sets them

data/sig/kino.rbs

@@ -22,6 +22,7 @@ module Kino
 
   class Server
     attr_reader port: Integer?
+    attr_reader control_port: Integer?
     attr_reader mode: Symbol
     attr_reader bind: String
 
@@ -98,11 +99,19 @@ module Kino
       def lanes: (boolish enabled) -> untyped
       def log_requests: (boolish enabled) -> untyped
       def on_error: (?^(Exception, Hash[String, untyped]) -> void handler) ?{ (Exception, Hash[String, untyped]) -> void } -> untyped
+      def after_boot: (?^() -> void handler) ?{ () -> void } -> untyped
+      def after_worker_boot: (?^(Integer) -> void handler) ?{ (Integer) -> void } -> untyped
+      def after_request_complete: (?^(Hash[String, untyped], Integer) -> void handler) ?{ (Hash[String, untyped], Integer) -> void } -> untyped
+      def on_worker_exit: (?^(Integer, Exception?) -> void handler) ?{ (Integer, Exception?) -> void } -> untyped
       def shutdown_timeout: (Numeric seconds) -> untyped
       def tokio_threads: (int count) -> untyped
       def tls: (cert: String, key: String) -> untyped
       def environment: (String | Symbol env) -> untyped
       def pidfile: (String path) -> untyped
+      def control_bind: (String addr) -> String
+      def control_token: (String token) -> String
+      def quarantine_timeout: (Numeric? seconds) -> untyped
+      def quarantine_max: (int count) -> untyped
       def rackup: (String path) -> untyped
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kino
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: aarch64-linux
 authors:
 - Yaroslav Markin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2026-07-13 00:00:00.000000000 Z
+date: 2026-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logger
@@ -147,16 +147,19 @@ files:
 - lib/kino/cli.rb
 - lib/kino/configuration.rb
 - lib/kino/errors_stream.rb
+- lib/kino/hook_fire.rb
 - lib/kino/input.rb
 - lib/kino/kino.so
 - lib/kino/logger.rb
 - lib/kino/null_input.rb
+- lib/kino/quarantine_monitor.rb
 - lib/kino/ractor_supervisor.rb
 - lib/kino/server.rb
 - lib/kino/stream.rb
 - lib/kino/templates/kino.rb.tt
 - lib/kino/version.rb
 - lib/kino/worker.rb
+- lib/kino/worker_hooks.rb
 - sig/kino.rbs
 homepage: https://github.com/yaroslav/kino
 licenses: