kino 0.2.1 → 0.4.0

data/lib/kino/server.rb

@@ -12,6 +12,10 @@ module Kino
     #   port when configured with port 0)
     attr_reader :port
 
+    # @return [Integer, nil] the control plane's TCP port (nil until #start,
+    #   when the control plane is off, or for a unix-socket bind)
+    attr_reader :control_port
+
     # @return [Symbol] the resolved dispatch mode, :ractor or :threaded
     attr_reader :mode
 
@@ -23,6 +27,29 @@ module Kino
       !@tls.nil?
     end
 
+    # @return [Boolean] whether the bind is a unix domain socket
+    #   ("unix:///path/to.sock")
+    def unix?
+      @bind.start_with?("unix://")
+    end
+
+    # Where the server listens, once started: `http://host:port`
+    # (`https` under TLS), or the `unix://` socket path.
+    # @return [String]
+    def url
+      unix? ? @bind : "http#{"s" if tls?}://#{@bind}:#{@port}"
+    end
+
+    # Where the control plane listens, once started, or nil when it is
+    # off: `http://host:port`, or its `unix://` socket path.
+    # @return [String, nil]
+    def control_url
+      return nil unless @control_bind
+      return @control_bind if @control_bind.start_with?("unix://")
+
+      "http://#{@control_bind.rpartition(":").first}:#{@control_port}"
+    end
+
     # Settings precedence: explicit kwargs > config_file DSL > defaults.
     #
     # @param app [#call] a Rack 3 application
@@ -41,8 +68,22 @@ module Kino
       @bind = settings[:bind]
       @requested_port = settings[:port]
       @workers = Integer(settings[:workers])
-      @on_error = validate_on_error(settings[:on_error])
+      @on_error = validate_hook(settings[:on_error], :on_error)
+      @after_worker_boot = validate_hook(settings[:after_worker_boot], :after_worker_boot)
+      @after_request_complete = validate_hook(settings[:after_request_complete], :after_request_complete)
+      @after_boot = validate_hook(settings[:after_boot], :after_boot)
+      @on_worker_exit = validate_hook(settings[:on_worker_exit], :on_worker_exit)
       @mode = resolve_mode(settings[:mode])
+      @worker_hooks = WorkerHooks.new(
+        on_error: @on_error,
+        after_worker_boot: @after_worker_boot,
+        after_request_complete: @after_request_complete,
+        # The access log's GC and allocation figures come from the VM's
+        # process-wide counters, so they are measured only where one
+        # request at a time can own them: the GVL serializes :threaded
+        # mode, and a single ractor has nothing to race.
+        access_timing: !!settings[:log_requests] && (@mode == :threaded || @workers == 1)
+      )
       # Default threads per mode: 1 in :ractor (threads inside a ractor
       # share its lock; a measured +17% on fast handlers; raise `workers`
       # for I/O concurrency instead), 3 in :threaded (threads ARE the
@@ -59,9 +100,29 @@ module Kino
       @shutdown_timeout = settings[:shutdown_timeout]
       @tokio_threads = settings[:tokio_threads]
       @tls = validate_tls(settings[:tls])
+      if @tls && unix?
+        raise ArgumentError, "TLS is not supported on a unix socket bind; terminate TLS at the proxy in front"
+      end
       @pidfile = settings[:pidfile]
+      @control_bind = settings[:control_bind]&.to_s
+      @control_token = settings[:control_token]&.to_s
+      # An empty token (e.g. control_token ENV["KINO_CONTROL_TOKEN"] with the
+      # var unset) must not half-disable auth: treat it as auth off, not as
+      # "require a zero-length Bearer token".
+      @control_token = nil if @control_token && @control_token.empty?
+      @quarantine_timeout_ms = settings[:quarantine_timeout] ? (Float(settings[:quarantine_timeout]) * 1000).round : nil
+      @quarantine_max =
+        if settings[:quarantine_max]
+          Integer(settings[:quarantine_max])
+        elsif @mode == :ractor
+          @workers
+        else
+          @workers * @threads
+        end
       @worker_threads = []
+      @worker_threads_lock = Mutex.new
       @supervisor = nil
+      @quarantine_monitor = nil
       @started = false
     end
 
@@ -78,7 +139,7 @@ module Kino
       write_pidfile if @pidfile
       booted = false
       begin
-        @id, @port = Native.server_start(
+        @id, @port, @control_port = Native.server_start(
           bind: @bind, port: @requested_port,
           queue_depth: @queue_depth, queue_timeout_ms: @queue_timeout_ms,
           request_timeout_ms: @request_timeout_ms,
@@ -86,7 +147,9 @@ module Kino
           max_body_size: @max_body_size,
           tokio_threads: @tokio_threads,
           tls_cert: @tls&.fetch(:cert), tls_key: @tls&.fetch(:key),
-          lanes: @lanes, log_requests: @log_requests
+          lanes: @lanes, log_requests: @log_requests,
+          mode: @mode.to_s, workers: @workers, threads: @threads, batch: @batch,
+          control_bind: @control_bind, control_token: @control_token
         )
         booted = true
       ensure
@@ -97,13 +160,13 @@ module Kino
       @pin_keeper = Native.pin_keeper(@id)
       if @mode == :ractor
         @supervisor = RactorSupervisor.new(@id, @app, workers: @workers, threads: @threads,
-          batch: @batch, on_error: @on_error).start
+          batch: @batch, hooks: @worker_hooks, on_worker_exit: @on_worker_exit).start
       else
-        @worker_threads = (@workers * @threads).times.map do
-          worker_id = Native.register_worker(@id)
-          Thread.new { Worker.run(@id, worker_id, @app, @batch, @on_error) }
-        end
+        @worker_threads = (@workers * @threads).times.map { spawn_worker_thread }
       end
+      start_quarantine_monitor if @quarantine_timeout_ms
+      Native.control_ready(@id)
+      HookFire.fire(@after_boot, "after_boot")
       @started = true
       self
     end
@@ -119,6 +182,7 @@ module Kino
     def shutdown(timeout: nil)
       return unless @started
 
+      @quarantine_monitor&.stop
       deadline = monotonic_now + (timeout || @shutdown_timeout)
       Native.stop_accepting(@id)
 
@@ -144,6 +208,9 @@ module Kino
       end
 
       Native.shutdown_runtime(@id, 1_000)
+      # The control thread reports "draining" for the whole drain and stops
+      # only now, once there is nothing left to report.
+      Native.control_stop(@id)
       # The runtime is gone, so hyper has dropped every pinned buffer;
       # the keeper (and the strings it marked) may now be collected.
       @pin_keeper = nil
@@ -159,22 +226,34 @@ module Kino
       @supervisor ? @supervisor.join : @worker_threads.each(&:join)
     end
 
-    # Production entry point: start, print the banner, trap INT/TERM for
-    # graceful shutdown (second signal force-exits), block until done.
-    # The `kino` CLI funnels into this too (CLI#serve).
+    # Production entry point: build the server and {#run} it. The `kino`
+    # CLI funnels into this too (CLI#serve).
     #
     # @param app [#call] a Rack 3 application
     # @param opts [Hash] see #initialize
     # @return [Kino::Server] the (stopped) server, after shutdown
     def self.run(app, **opts)
-      server = new(app, **opts)
+      new(app, **opts).run
+    end
+
+    # Serve until shut down: start, print the banner, trap INT/TERM for
+    # graceful shutdown (second signal force-exits), block until done.
+    # The Rack handler calls this on a server it built itself.
+    #
+    # @return [self] after shutdown
+    def run
+      # Startup output must land immediately even when stdout is a pipe or
+      # file (process supervisors, `kino > server.log`, `rails server`
+      # under Docker); block buffering would hold the banner back until
+      # exit.
+      $stdout.sync = true
       CLI.opening_credits
-      server.start
-      CLI.action!(server)
+      start
+      CLI.action!(self)
       CLI.fin_at_exit
-      trap_signals(server)
-      server.wait
-      server
+      self.class.trap_signals(self)
+      wait
+      self
     end
 
     # Signal handling shared by Server.run and the kino CLI: INT/TERM drain
@@ -186,14 +265,14 @@ module Kino
       # kill -USR1 <pid> prints a one-line stats snapshot (find the pid in
       # the pidfile when configured).
       trap("USR1") do
-        Thread.new { $stdout.puts Kino::CLI.stats_line(server.stats) }
+        Thread.new { Log.info(CLI.stats_line(server.stats)) }
       end
       signaled = false
       %w[INT TERM].each do |signal|
         trap(signal) do
           Process.exit!(1) if signaled
           signaled = true
-          $stderr.write("Kino: draining (signal again to force exit)\n")
+          Log.warn("draining (signal again to force exit)")
           # Trap context forbids mutexes; do the real work on a thread.
           Thread.new { server.shutdown }
         end
@@ -205,22 +284,90 @@ module Kino
     #
     # @return [Hash{Symbol => Object}] mode, lanes, workers, threads,
     #   batch, respawns; plus queued, in_flight, served, rejected,
-    #   timeouts (and lane_depths in lanes mode) once started
+    #   timeouts, worker_status, quarantined, queue_time (and lane_depths in
+    #   lanes mode) once started
     def stats
       base = {
         mode: @mode, lanes: @lanes, workers: @workers, threads: @threads,
-        batch: @batch, respawns: @supervisor ? @supervisor.respawns : 0
+        batch: @batch, respawns: 0
       }
       return base unless @started
 
-      queued, in_flight, served, rejected, timeouts, lane_depths = Native.server_stats(@id)
-      base.merge!(queued:, in_flight:, served:, rejected:, timeouts:)
+      queued, in_flight, served, rejected, timeouts, respawns, lane_depths = Native.server_stats(@id)
+      base.merge!(queued:, in_flight:, served:, rejected:, timeouts:, respawns:)
       base[:lane_depths] = lane_depths if lane_depths
+      rows = Native.worker_stats(@id)
+      base[:worker_status] = rows.map do |index, served, in_flight, busy_ms, quarantined|
+        {index:, served:, in_flight:, busy_ms:, quarantined:}
+      end
+      base[:quarantined] = rows.count { |_index, _served, _in_flight, _busy_ms, quarantined| quarantined }
+      count, sum_seconds = Native.queue_time(@id)
+      base[:queue_time] = {count:, sum_seconds:}
       base
     end
 
     private
 
+    # Register a fresh dispatch slot and run a worker thread on it; returns
+    # the thread. Used at boot and by the quarantine replacer.
+    def spawn_worker_thread
+      worker_id = Native.register_worker(@id)
+      Thread.new do
+        # Named so log lines from inside say which worker spoke.
+        Thread.current.name = "worker-#{worker_id}"
+        error = nil
+        begin
+          Worker.run(@id, worker_id, @app, @batch, @worker_hooks)
+        rescue Exception => e # rubocop:disable Lint/RescueException -- a hard crash in a threaded worker thread
+          error = e
+          raise
+        ensure
+          HookFire.fire(@on_worker_exit, "on_worker_exit", worker_id, error)
+        end
+      end
+    end
+
+    # Track a replacement thread spawned outside the initial pool assignment
+    # (the quarantine replacer) so shutdown's join/done?/kill sweeps see it.
+    def track_replacement_thread(thread)
+      @worker_threads_lock.synchronize { @worker_threads << thread }
+    end
+
+    # @private
+    # The :threaded-mode quarantine replacer: spawns a replacement worker
+    # thread, quarantines the wedged slot, then tracks the new thread so
+    # shutdown's join/done?/kill sweeps see it. Built from bound Method
+    # objects instead of a server reference, so it drives the server
+    # through those methods without send or instance_variable_get.
+    class ThreadedReplacer
+      def initialize(server_id:, spawner:, tracker:)
+        @server_id = server_id
+        @spawner = spawner
+        @tracker = tracker
+      end
+
+      def replace(worker_id)
+        thread = @spawner.call # spawn FIRST (may raise ThreadError)
+        Native.quarantine_slot(@server_id, worker_id) # quarantine after success
+        @tracker.call(thread)
+        true
+      end
+    end
+    private_constant :ThreadedReplacer
+
+    # A replacer.replace(worker_id) spawns a replacement worker, then
+    # quarantines the wedged slot, mode-appropriately. In :ractor the
+    # supervisor is the replacer; in :threaded a small object over
+    # spawn_worker_thread.
+    def start_quarantine_monitor
+      replacer = @supervisor || ThreadedReplacer.new(server_id: @id, spawner: method(:spawn_worker_thread),
+        tracker: method(:track_replacement_thread))
+      @quarantine_monitor = QuarantineMonitor.new(
+        server_id: @id, timeout_ms: @quarantine_timeout_ms,
+        max: @quarantine_max, replacer: replacer
+      ).start
+    end
+
     def validate_tls(tls)
       return nil if tls.nil?
       unless tls.is_a?(Hash) && tls[:cert] && tls[:key]
@@ -230,10 +377,10 @@ module Kino
       {cert: String(tls[:cert]), key: String(tls[:key])}
     end
 
-    def validate_on_error(handler)
+    def validate_hook(handler, name)
       return nil if handler.nil?
       unless handler.respond_to?(:call)
-        raise ArgumentError, "on_error must respond to #call (got #{handler.class})"
+        raise ArgumentError, "#{name} must respond to #call (got #{handler.class})"
       end
 
       handler
@@ -320,7 +467,8 @@ module Kino
       if @supervisor
         @supervisor.shutdown([deadline - monotonic_now, 0].max)
       else
-        @worker_threads.each do |thread|
+        threads = @worker_threads_lock.synchronize { @worker_threads.dup }
+        threads.each do |thread|
           thread.join([deadline - monotonic_now, 0.01].max)
         end
       end
@@ -330,7 +478,8 @@ module Kino
       if @supervisor
         @supervisor.done?
       else
-        @worker_threads.none?(&:alive?)
+        threads = @worker_threads_lock.synchronize { @worker_threads.dup }
+        threads.none?(&:alive?)
       end
     end
 
@@ -338,9 +487,10 @@ module Kino
       if @supervisor
         # Ractors cannot be force-killed; their clients were already freed
         # by abort_all_inflight. The stuck ractor leaks until process exit.
-        Native.log_error("shutdown deadline passed with stuck ractor workers") unless @supervisor.done?
+        Log.error("shutdown deadline passed with stuck ractor workers") unless @supervisor.done?
       else
-        @worker_threads.each { |thread| thread.kill if thread.alive? }
+        threads = @worker_threads_lock.synchronize { @worker_threads.dup }
+        threads.each { |thread| thread.kill if thread.alive? }
       end
     end
 
@@ -361,18 +511,18 @@ module Kino
             "Ractor.shareable_proc endpoints); try Ractor.make_shareable(app) " \
             "or mode: :threaded"
         end
-        unless @on_error.nil? || Ractor.shareable?(@on_error)
+        if (name = unshareable_worker_hook_name)
           raise Error,
-            "mode: :ractor requires a Ractor-shareable on_error handler " \
+            "mode: :ractor requires a Ractor-shareable #{name} hook " \
             "(build it with Ractor.shareable_proc, or use mode: :threaded)"
         end
         :ractor
       when :auto
         if !Ractor.shareable?(@app)
-          warn "Kino: app is not Ractor-shareable; falling back to mode: :threaded"
+          Log.warn("app is not Ractor-shareable; falling back to mode: :threaded")
           :threaded
-        elsif !(@on_error.nil? || Ractor.shareable?(@on_error))
-          warn "Kino: on_error handler is not Ractor-shareable; falling back to mode: :threaded"
+        elsif (name = unshareable_worker_hook_name)
+          Log.warn("#{name} hook is not Ractor-shareable; falling back to mode: :threaded")
           :threaded
         else
           :ractor
@@ -381,5 +531,22 @@ module Kino
         raise ArgumentError, "mode must be :auto, :ractor, or :threaded (got #{requested.inspect})"
       end
     end
+
+    # The hooks that ride into worker context (a ractor in :ractor mode)
+    # and so must be Ractor-shareable there. after_boot and on_worker_exit
+    # run on the main thread and are exempt.
+    def worker_context_hooks
+      [[:on_error, @on_error], [:after_worker_boot, @after_worker_boot],
+        [:after_request_complete, @after_request_complete]]
+    end
+
+    # The name of the first worker-context hook that is set but not
+    # Ractor-shareable, or nil if all set ones are. Used by both the
+    # :ractor raise and the :auto warn/fallback branches, so "first
+    # offender" is defined once.
+    def unshareable_worker_hook_name
+      bad = worker_context_hooks.find { |_name, hook| !(hook.nil? || Ractor.shareable?(hook)) }
+      bad&.first
+    end
   end
 end

data/lib/kino/templates/kino.rb.tt

@@ -8,7 +8,9 @@
 ## Network
 
 # Address to listen on. Use "0.0.0.0" to accept connections from other
-# machines.
+# machines, or "unix:///run/kino.sock" to listen on a unix domain socket
+# behind a proxy such as nginx (the port below is then unused; a stale
+# socket file is reclaimed, a live one refused).
 # bind "127.0.0.1"
 
 # Port to listen on.
@@ -22,7 +24,8 @@
 
 # How many workers to run. Each worker handles requests independently;
 # in :ractor mode every worker runs Ruby in parallel on its own core.
-# Default: one per CPU core.
+# Default: the CPUs this process may use (Kino.available_parallelism:
+# the affinity mask and, in a container, the cgroup CPU quota).
 # workers 8
 
 # Threads inside each worker. More threads help when your app spends
@@ -74,9 +77,12 @@
 # for quick handlers; behavior under heavy overload differs slightly.
 # lanes false
 
-# Print one line per request to stdout, colored by status on a
-# terminal. This is the server's view: it includes requests your app
-# never saw, such as 503s. Recommended in development.
+# Log every request to stdout: an arrival line before the app runs and a
+# completion line after it, colored by status on a terminal, with a
+# timing breakdown (time in Ruby with its GC pause and allocations, the
+# server's own overhead, and queue wait). This is the server's view: it
+# includes requests your app never saw, such as 503s. Recommended in
+# development; cheap enough for production.
 # log_requests false
 
 # Called when a worker catches an app or delivery error, after the client
@@ -87,6 +93,27 @@
 # must be Ractor-shareable (build it with Ractor.shareable_proc).
 # on_error ->(error, env) { ExceptionService.capture(error) }
 
+# Called once on the main thread after the worker pool is up. Wire
+# readiness here (sd_notify, a "server ready" metric).
+# after_boot { }
+
+# Called once inside each worker before it serves, with the worker's slot
+# id. In :ractor mode it runs inside the worker ractor, so it must be
+# Ractor-shareable (build it with Ractor.shareable_proc).
+# after_worker_boot { |worker_id| }
+
+# Called inside the worker after each successful response, with (env,
+# status). This is the hot path: leave it unset for zero cost. In :ractor
+# mode it must be Ractor-shareable.
+# after_request_complete { |env, status| }
+
+# Called on the main thread when a worker exits, with (worker_index,
+# error). error is the crash cause, or nil on a clean exit. In :ractor
+# mode worker_index identifies the exited ractor (0..workers - 1), a
+# different number space than after_worker_boot's slot id: do not
+# correlate boot and exit by that number in :ractor mode.
+# on_worker_exit { |worker_index, error| }
+
 ## Lifecycle
 
 # On shutdown, give in-flight requests this many seconds to finish.
@@ -102,6 +129,36 @@
 # heavily CPU-bound apps, try 1 to leave more cores for Ruby.
 # tokio_threads 4
 
+## Control plane
+
+# Serve read-only monitoring on a separate address: live stats as JSON
+# at /stats, Prometheus text at /metrics, and /ready and /live probes
+# for an orchestrator. Answered by the native layer on its own thread,
+# so it stays live even when every Ruby worker is busy or stuck.
+# Accepts "host:port" or "unix://path". Off unless set.
+# control_bind "127.0.0.1:9293"
+
+# When set, /stats and /metrics require "Authorization: Bearer <token>".
+# The probes stay open; they carry no data.
+# control_token ENV["KINO_CONTROL_TOKEN"]
+
+## Quarantine
+
+# Quarantine a dispatch slot whose current request has run longer than
+# this many seconds and spawn a replacement to restore capacity. The
+# wedged worker is never interrupted or force-killed; its slot stays
+# quarantined for good (in :ractor mode the ractor itself leaks until
+# process exit, since a wedged ractor cannot be safely interrupted). Off
+# unless set; set it above your slowest legitimate endpoint, and above
+# request_timeout.
+# quarantine_timeout 60
+
+# Cap on the total number of replacement events over the process
+# lifetime. Past the cap the monitor stops replacing and the server runs
+# at reduced capacity. Default: the worker count in :ractor mode, workers
+# x threads in :threaded.
+# quarantine_max 8
+
 ## App
 
 # Rackup file to load (a command-line argument wins).

data/lib/kino/version.rb

@@ -2,5 +2,5 @@
 
 module Kino
   # The gem version (single source of truth; ext/kino/Cargo.toml syncs).
-  VERSION = "0.2.1"
+  VERSION = "0.4.0"
 end

data/lib/kino/worker.rb

@@ -18,13 +18,14 @@ module Kino
 
     module_function
 
-    def run(server_id, worker_id, app, batch_size = 1, on_error = nil)
+    def run(server_id, worker_id, app, batch_size = 1, hooks = nil)
+      fire_after_worker_boot(hooks, worker_id)
       if batch_size <= 1
         env = Native.take_one(server_id, worker_id)
-        env = handle_one(env, server_id, worker_id, app, on_error) while env
+        env = handle_one(env, server_id, worker_id, app, hooks) while env
       else
         batch = Native.take_batch(server_id, worker_id, batch_size)
-        batch = process(batch, server_id, worker_id, app, batch_size, on_error) while batch
+        batch = process(batch, server_id, worker_id, app, batch_size, hooks) while batch
       end
     end
 
@@ -34,8 +35,8 @@ module Kino
     NOT_FUSED = Object.new.freeze
 
     # Handle one request; returns the next env (fused take) or nil.
-    def handle_one(env, server_id, worker_id, app, on_error)
-      result = serve(env, app, on_error) do |request, status, headers, chunks|
+    def handle_one(env, server_id, worker_id, app, hooks)
+      result = serve(env, app, hooks) do |request, status, headers, chunks|
         request.respond_and_take_one(server_id, worker_id, status, headers, chunks)
       end
       result.equal?(NOT_FUSED) ? Native.take_one(server_id, worker_id) : result
@@ -43,10 +44,10 @@ module Kino
 
     # Handle every env in the batch; returns the next batch (the last
     # simple response rides the fused respond_and_take) or nil on shutdown.
-    def process(batch, server_id, worker_id, app, batch_size, on_error)
+    def process(batch, server_id, worker_id, app, batch_size, hooks)
       last = batch.size - 1
       batch.each_with_index do |env, index|
-        result = serve(env, app, on_error) do |request, status, headers, chunks|
+        result = serve(env, app, hooks) do |request, status, headers, chunks|
           if index == last
             request.respond_and_take(server_id, worker_id, batch_size,
               status, headers, chunks)
@@ -66,17 +67,39 @@ module Kino
     # here and return NOT_FUSED. App errors must never kill the worker;
     # hard crashes (Exception) are the supervisor's job; and `abort` does
     # the right thing whether or not the response head already went out.
-    def serve(env, app, on_error)
+    def serve(env, app, hooks)
       request = env[KINO_REQUEST]
       env[RACK_INPUT] ||= Input.new(request)
-      status, headers, body = app.call(env)
+      if hooks&.access_timing
+        # The access log's breakdown: the VM's cumulative GC time and
+        # allocation count, differenced around the app call.
+        gc_before = GC.total_time
+        allocated_before = GC.stat(:total_allocated_objects)
+        status, headers, body = app.call(env)
+        request.timing(GC.total_time - gc_before, GC.stat(:total_allocated_objects) - allocated_before)
+      else
+        status, headers, body = app.call(env)
+      end
 
       if body.respond_to?(:to_ary)
-        result = yield(request, status.to_i, headers, join_chunks(body.to_ary))
-        body.close if body.respond_to?(:close)
-        result
+        chunks = join_chunks(body.to_ary)
+        if hooks&.after_request_complete
+          # Hook set: do not fuse. Send the complete response, fire the hook
+          # after it is out, then signal the caller to take the next request
+          # separately (so the hook never waits on the next request).
+          request.send_simple(status.to_i, headers, chunks)
+          body.close if body.respond_to?(:close)
+          fire_after_request_complete(hooks, env, status.to_i)
+          NOT_FUSED
+        else
+          # No hook: fused fast path, unchanged, zero cost.
+          result = yield(request, status.to_i, headers, chunks)
+          body.close if body.respond_to?(:close)
+          result
+        end
       else
         deliver_streaming(request, status.to_i, headers, body, env[RACK_INPUT])
+        fire_after_request_complete(hooks, env, status.to_i)
         NOT_FUSED
       end
     rescue => e
@@ -85,27 +108,12 @@ module Kino
       # delivery errors (they happen after app.call returned, so no
       # middleware can see them); its own failures are logged, not raised,
       # because nothing may escape this block and kill the worker.
-      Native.log_error(error_log_line(e))
+      Log.exception(e, env)
       request.abort
-      if on_error
-        begin
-          on_error.call(e, env)
-        rescue => hook_error
-          Native.log_error("on_error hook raised #{hook_error.class}: #{hook_error.message}")
-        end
-      end
+      HookFire.fire(hooks&.on_error, "on_error", e, env)
       NOT_FUSED
     end
 
-    # First frames only: the raise site is at the top, and Rails stacks
-    # run hundreds of middleware frames deep. Hooks get the full exception.
-    BACKTRACE_FRAMES = 12
-
-    def error_log_line(error)
-      ["#{error.class}: #{error.message}",
-        *(error.backtrace || []).first(BACKTRACE_FRAMES)].join("\n  ")
-    end
-
     def deliver_streaming(request, status, headers, body, input)
       request.send_headers(status, headers)
       if body.respond_to?(:call) && !body.respond_to?(:each)
@@ -142,7 +150,15 @@ module Kino
       joined
     end
 
+    def fire_after_worker_boot(hooks, worker_id)
+      HookFire.fire(hooks&.after_worker_boot, "after_worker_boot", worker_id)
+    end
+
+    def fire_after_request_complete(hooks, env, status)
+      HookFire.fire(hooks&.after_request_complete, "after_request_complete", env, status)
+    end
+
     private_class_method :handle_one, :process, :serve, :deliver_streaming,
-      :join_chunks, :error_log_line
+      :join_chunks, :fire_after_worker_boot, :fire_after_request_complete
   end
 end

data/lib/kino/worker_hooks.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Kino
+  # @private
+  # The worker-context lifecycle hooks, bundled so one frozen, shareable
+  # value crosses into each worker (a ractor in :ractor mode) instead of
+  # several bare procs. Any member may be nil. A Data instance is frozen,
+  # so it is Ractor.shareable? exactly when its members are (nil, or a
+  # Ractor.shareable_proc), letting it ride the ractor boundary like the app.
+  # `access_timing` rides along: whether the worker measures the GC pause
+  # and allocations around each app call for the access log's breakdown.
+  WorkerHooks = Data.define(:on_error, :after_worker_boot, :after_request_complete, :access_timing)
+end

data/lib/kino.rb

@@ -33,9 +33,20 @@ module Kino
     remaining = Native.sleep_chunk(remaining) while remaining.positive?
     nil
   end
+
+  # How many CPUs this process may actually use: the `workers` default.
+  # Unlike `Etc.nprocessors`, this honours a cgroup CPU quota (a container
+  # limited to 2 CPUs on a 64-core host gets 2, not 64) as well as the
+  # affinity mask; a fractional quota rounds up. Never below 1.
+  #
+  # @return [Integer]
+  def self.available_parallelism
+    Native.available_parallelism
+  end
 end
 
 require_relative "kino/cli"
+require_relative "kino/log"
 require_relative "kino/logger"
 require_relative "kino/check"
 require_relative "kino/input"
@@ -43,8 +54,11 @@ require_relative "kino/null_input"
 require_relative "kino/errors_stream"
 require_relative "kino/stream"
 require_relative "kino/configuration"
+require_relative "kino/hook_fire"
+require_relative "kino/worker_hooks"
 require_relative "kino/worker"
 require_relative "kino/ractor_supervisor"
+require_relative "kino/quarantine_monitor"
 require_relative "kino/server"
 
 # Hand the frozen shareable singletons to the native layer: it sets them