kino 0.2.1 → 0.4.0

data/lib/kino/cli.rb

@@ -100,11 +100,14 @@ module Kino
       end.join
     end
 
-    # One-line stats dump (the SIGUSR1 handler's output).
+    # One-line stats dump (what the SIGUSR1 handler logs). Excludes
+    # worker_status: it's an array with one entry per execution slot, and
+    # printing it inline would break the one-line contract (see /stats for
+    # per-worker detail).
     # @param stats [Hash{Symbol => Object}] see {Kino::Server#stats}
     # @return [String]
     def stats_line(stats)
-      dim("Kino stats: #{stats.map { |k, v| "#{k}=#{v.inspect}" }.join(" ")}")
+      "stats #{stats.except(:worker_status).map { |k, v| "#{k}=#{v.inspect}" }.join(" ")}"
     end
 
     # The two banner halves around Server#start: credits before, the ready
@@ -116,15 +119,27 @@ module Kino
       puts dim("\nKino #{VERSION} presents:")
     end
 
+    # The ready block: what this process is (Ruby build with its JIT and
+    # parser flags, environment, topology, pid) and where it listens.
     # @param server [Kino::Server] a started server
     # @return [void]
     def action!(server)
-      puts dim("- mode:      #{server.mode}")
-      puts dim("- listening: http#{"s" if server.tls?}://#{server.bind}:#{server.port}")
+      stats = server.stats
+      puts dim("- ruby:      #{RUBY_DESCRIPTION}")
+      puts dim("- env:       #{ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"}")
+      puts dim("- mode:      #{server.mode}, #{count(stats[:workers], "worker")} × #{count(stats[:threads], "thread")}")
+      puts dim("- pid:       #{Process.pid}")
+      puts dim("- listening: #{server.url}")
+      puts dim("- control:   #{server.control_url}") if server.control_url
       puts dim("- Ctrl-C to drain and stop")
       puts "\n#{bold("Action!")}\n\n"
     end
 
+    # "1 worker", "8 workers".
+    def count(number, noun)
+      "#{number} #{noun}#{"s" unless number == 1}"
+    end
+
     # Roll credits when the process ends: normal exit or crash (at_exit
     # also runs after an uncaught exception; only a force-exit skips it).
     # @return [void]
@@ -197,7 +212,7 @@ module Kino
     def option_parser(options)
       OptionParser.new do |opts|
         opts.banner = "Usage: kino [options] [rackup file (default: config.ru)]"
-        opts.on("-C", "--config FILE", "Config file (default: kino.rb if present)") { |v| options[:config_file] = v }
+        opts.on("-C", "--config FILE", "Config file (default: kino.rb, then config/kino.rb)") { |v| options[:config_file] = v }
         opts.on("--init [PATH]", "Write a commented sample config (default: kino.rb) and exit") do |v|
           options[:init_path] = v || "kino.rb"
         end
@@ -233,14 +248,13 @@ module Kino
       require "kino"
       require "rack"
 
-      config_file = options[:config_file]
-      config_file ||= ("kino.rb" if File.exist?("kino.rb"))
+      config_file = options[:config_file] || Configuration.default_path
 
       config = Configuration.new
       config.load_file(config_file) if config_file
       config.merge!(options[:overrides])
       # Default port 9292 when neither the file nor a flag chose one.
-      config.set(:port, 9292) unless config.set?(:port)
+      config.set(:port, Configuration::DEFAULT_SERVING_PORT) unless config.set?(:port)
       config
     end
 
@@ -249,6 +263,6 @@ module Kino
     end
 
     private_class_method :print_help, :option_parser, :write_sample,
-      :resolve_config, :serve
+      :resolve_config, :serve, :count
   end
 end

data/lib/kino/configuration.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "etc"
-
 module Kino
   # Server settings with Puma-style precedence:
   #   explicit Server.new kwargs > config file DSL > defaults.
@@ -11,7 +9,7 @@ module Kino
     DEFAULTS = {
       bind: "127.0.0.1",
       port: 0,
-      workers: nil, # resolved to Etc.nprocessors in #to_h
+      workers: nil, # resolved to Kino.available_parallelism in #to_h
       threads: nil, # resolved per mode in Server: 1 in :ractor, 3 in :threaded
       mode: :auto,
       queue_depth: 1024,
@@ -23,11 +21,19 @@ module Kino
       lanes: false,
       log_requests: false,
       on_error: nil,
+      after_boot: nil,
+      after_worker_boot: nil,
+      after_request_complete: nil,
+      on_worker_exit: nil,
       shutdown_timeout: 30,
       tokio_threads: nil,
       tls: nil,
       environment: nil,
       pidfile: nil,
+      control_bind: nil,
+      control_token: nil,
+      quarantine_timeout: nil,
+      quarantine_max: nil,
       rackup: nil
     }.freeze
 
@@ -37,6 +43,21 @@ module Kino
     # Source template for {.sample}.
     SAMPLE_TEMPLATE = File.expand_path("templates/kino.rb.tt", __dir__)
 
+    # Where the `kino` CLI and the Rack handler look for a config file when
+    # none is named: the project root first, then the Rails-style config/.
+    DEFAULT_PATHS = %w[kino.rb config/kino.rb].freeze
+
+    # The port the CLI and the Rack handler serve on when neither a flag
+    # nor the file chose one (Server.new itself defaults to an ephemeral
+    # port, for embedding).
+    DEFAULT_SERVING_PORT = 9292
+
+    # The first of {DEFAULT_PATHS} that exists in the working directory.
+    # @return [String, nil]
+    def self.default_path
+      DEFAULT_PATHS.find { |path| File.exist?(path) }
+    end
+
     # The fully-commented sample config (see `kino --init`).
     # @return [String]
     def self.sample
@@ -105,7 +126,7 @@ module Kino
     # @return [Hash{Symbol => Object}] every setting, defaults filled in
     def to_h
       SETTINGS.to_h { |key| [key, self[key]] }.tap do |h|
-        h[:workers] ||= Etc.nprocessors
+        h[:workers] ||= Kino.available_parallelism
       end
     end
 
@@ -138,7 +159,9 @@ module Kino
         @config = config
       end
 
-      # Address to listen on ("0.0.0.0" accepts non-local connections).
+      # Address to listen on: a host ("0.0.0.0" accepts non-local
+      # connections), or "unix:///path/to.sock" for a unix domain socket
+      # (then `port` is unused).
       def bind(host) = @config.set(:bind, host)
 
       # Port to listen on; 0 picks an ephemeral port.
@@ -185,6 +208,24 @@ module Kino
       # or a block. Must be Ractor-shareable in :ractor mode.
       def on_error(handler = nil, &block) = @config.set(:on_error, handler || block)
 
+      # Called once on the main thread after the worker pool is up. The
+      # readiness seam (wire sd_notify or a "server ready" metric here).
+      def after_boot(handler = nil, &block) = @config.set(:after_boot, handler || block)
+
+      # Called once inside each worker (a ractor in :ractor mode) before it
+      # serves, with the worker's slot id. Must be Ractor-shareable in
+      # :ractor mode (build it with Ractor.shareable_proc).
+      def after_worker_boot(handler = nil, &block) = @config.set(:after_worker_boot, handler || block)
+
+      # Called inside the worker after each successful response with
+      # (env, status). Hot path: leave unset for zero cost. Must be
+      # Ractor-shareable in :ractor mode.
+      def after_request_complete(handler = nil, &block) = @config.set(:after_request_complete, handler || block)
+
+      # Called on the main thread when a worker exits, with (worker_index,
+      # error_or_nil). error is the crash cause, or nil on a clean exit.
+      def on_worker_exit(handler = nil, &block) = @config.set(:on_worker_exit, handler || block)
+
       # Graceful-shutdown drain deadline in seconds.
       def shutdown_timeout(seconds) = @config.set(:shutdown_timeout, seconds)
 
@@ -200,6 +241,37 @@ module Kino
       # Write the master PID here on start.
       def pidfile(path) = @config.set(:pidfile, path.to_s)
 
+      # Serve the read-only control plane (live stats as JSON at /stats,
+      # Prometheus text at /metrics, /ready and /live probes) on this
+      # address: "host:port" or "unix://path". Off unless set.
+      def control_bind(addr) = @config.set(:control_bind, addr.to_s)
+
+      # When set, /stats and /metrics require "Authorization: Bearer <token>".
+      # The probes stay open; they carry no data.
+      def control_token(token) = @config.set(:control_token, token.to_s)
+
+      # Quarantine a dispatch slot whose current request has run longer
+      # than this many seconds, spawning a replacement to restore capacity.
+      # Off unless set. Set it above your slowest legitimate endpoint (and
+      # typically above request_timeout).
+      def quarantine_timeout(seconds)
+        seconds &&= Float(seconds)
+        if seconds && seconds <= 0
+          raise ArgumentError, "quarantine_timeout must be greater than 0 (got #{seconds})"
+        end
+        @config.set(:quarantine_timeout, seconds)
+      end
+
+      # Cap on the total number of replacement events over the process
+      # lifetime. Past the cap the monitor stops replacing and the server
+      # runs at reduced capacity. Default: the worker count in :ractor
+      # mode, workers x threads in :threaded.
+      def quarantine_max(count)
+        count = Integer(count)
+        raise ArgumentError, "quarantine_max must be >= 1 (got #{count})" if count < 1
+        @config.set(:quarantine_max, count)
+      end
+
       # Rackup file the `kino` CLI loads (positional argument wins).
       def rackup(path) = @config.set(:rackup, path.to_s)
     end

data/lib/kino/errors_stream.rb

@@ -2,17 +2,18 @@
 
 module Kino
   # @private
-  # rack.errors: stateless writer into the native logger. Frozen singleton,
+  # rack.errors: stateless writer into the server log (one line per
+  # call, labelled like every other line Kino writes). Frozen singleton,
   # which also makes it Ractor-shareable; one instance serves all workers.
   class ErrorsStream
     def puts(message)
-      Native.log_error(message.to_s)
+      Log.error(message.to_s.chomp)
       nil
     end
 
     def write(message)
       message = message.to_s
-      Native.log_error(message)
+      Log.error(message.chomp)
       message.bytesize
     end
 

data/lib/kino/hook_fire.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Kino
+  # @private
+  # Fires a lifecycle hook and turns a raise into a logged line instead of
+  # letting it escape. Stateless and touches only its arguments plus
+  # Kino::Log (safe inside worker ractors), so it is safe to call from
+  # worker context: no main-ractor state is captured.
+  module HookFire
+    module_function
+
+    def fire(hook, name, *args)
+      return unless hook
+
+      begin
+        hook.call(*args)
+      rescue => e
+        Log.error("#{name} hook raised #{e.class}: #{e.message}")
+      end
+    end
+  end
+end

data/lib/kino/log.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Kino
+  # Server log lines: the lifecycle notices, crash and respawn reports,
+  # hook failures, the failed-request report, and whatever apps write to
+  # `rack.errors`, all in one shape:
+  #
+  #   kino[4213] worker-3: after_worker_boot hook raised RuntimeError: boom
+  #
+  # The label is syslog's `ident[pid]` tag plus the source that spoke: the
+  # worker ractor and/or thread by name, `main` for neither. On color
+  # terminals the label is dim, yellow, or red by level; the message stays
+  # plain. Notes go to stdout, warnings and errors to stderr.
+  #
+  # Hooks may log through here too (`Kino::Log.info "cache warm"`). Every
+  # method is safe inside a worker ractor: the line is handed to the
+  # native layer, which owns the streams, so no ractor touches `$stdout`
+  # or `$stderr` itself.
+  module Log
+    # Frames shown in a failed-request report before the rest are folded.
+    FRAMES = 12
+
+    # The working directory at boot, stripped from backtrace frames so the
+    # app's own code reads `app/controllers/x.rb:9` rather than an
+    # absolute path (frozen: worker ractors read it).
+    WORKING_DIR = File.join(Dir.pwd, "").freeze
+
+    module_function
+
+    # @param message [#to_s]
+    # @return [void]
+    def info(message)
+      Native.log_line("info", source, message.to_s)
+    end
+
+    # @param message [#to_s]
+    # @return [void]
+    def warn(message)
+      Native.log_line("warn", source, message.to_s)
+    end
+
+    # @param message [#to_s]
+    # @return [void]
+    def error(message)
+      Native.log_line("error", source, message.to_s)
+    end
+
+    # The failed-request report: the request line, the error, and where it
+    # raised in the app, then the backtrace with the app's own frames
+    # first (relative to the working directory) and the rest folded.
+    #
+    #   500 GET /boom · RuntimeError: kaboom (app.rb:12:in 'explode')
+    #       app.rb:12:in 'explode'
+    #       /gems/rack-3.2.7/lib/rack/builder.rb:...
+    #       … 38 more
+    #
+    # @param error [Exception]
+    # @param env [Hash] the Rack env of the failed request
+    # @param status [Integer] the status the client got
+    # @return [void]
+    def exception(error, env, status: 500)
+      frames, depth = trace(error)
+      site = frames.first ? " (#{frames.first})" : ""
+      lines = ["#{status} #{env["REQUEST_METHOD"]} #{env["PATH_INFO"]} · #{error.class}: #{error.message}#{site}"]
+      frames.each { |frame| lines << "    #{frame}" }
+      lines << "    … #{depth - frames.size} more" if depth > frames.size
+      error(lines.join("\n"))
+    end
+
+    # The `kino[<pid>] <source>:` tag a line from here carries.
+    # @return [String]
+    def label
+      "kino[#{Process.pid}] #{source}:"
+    end
+
+    # Who is speaking: the ractor's name, the thread's name, both joined
+    # with a slash, or `main` when neither is named. Kino names its
+    # worker ractors and threads `worker-N`.
+    # @return [String]
+    def source
+      parts = [Ractor.current.name, Thread.current.name].compact
+      parts.empty? ? "main" : parts.join("/")
+    end
+
+    # The backtrace as [frames, depth]: each frame relativized to the
+    # working directory, the app's own frames floated to the front (the
+    # raise site in your code reads first; gem and stdlib frames keep
+    # their order below), capped at FRAMES; depth is the real length.
+    def trace(error)
+      raw = error.backtrace || []
+      app, rest = raw.map { |frame| frame.delete_prefix(WORKING_DIR) }.partition { |frame| app_frame?(frame) }
+      [(app + rest).first(FRAMES), raw.size]
+    end
+
+    # A project-relative path (the working-directory prefix came off, so
+    # it does not start with `/`) that is not a synthetic frame (`(eval)`,
+    # `<internal:...>`). Gem and stdlib frames stay absolute.
+    def app_frame?(frame)
+      !frame.start_with?("/", "<", "(")
+    end
+
+    private_class_method :trace, :app_frame?
+  end
+end

data/lib/kino/quarantine_monitor.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Kino
+  # @private
+  # Polls per-slot busy_ms and, past the timeout, quarantines a wedged slot
+  # and asks the replacer to spawn a fresh worker. Runs one thread on the
+  # main ractor (uncontended by wedged worker ractors, so it stays
+  # responsive in :ractor mode). Never interrupts the wedged worker.
+  class QuarantineMonitor
+    def initialize(server_id:, timeout_ms:, max:, replacer:, tick: 0.5)
+      @server_id = server_id
+      @timeout_ms = timeout_ms
+      @max = max
+      @replacer = replacer
+      @tick = tick
+      @outstanding = 0
+      @at_cap_logged = false
+      @running = false
+      @thread = nil
+    end
+
+    def start
+      @running = true
+      @thread = Thread.new do
+        Thread.current.name = "quarantine"
+        run
+      end
+      self
+    end
+
+    def stop
+      @running = false
+      @thread&.join(@tick * 2)
+    end
+
+    private
+
+    def run
+      tick while @running
+    rescue => e
+      Log.error("quarantine monitor crashed: #{e.class}: #{e.message}")
+    end
+
+    def tick
+      scan_slots
+    rescue => e
+      # A bad tick must never kill the monitor.
+      Log.error("quarantine tick error: #{e.class}: #{e.message}")
+    ensure
+      sleep @tick
+    end
+
+    def scan_slots
+      Native.worker_stats(@server_id).each do |index, _served, _in_flight, busy_ms, quarantined|
+        next if quarantined || busy_ms <= @timeout_ms
+
+        if @outstanding >= @max
+          unless @at_cap_logged
+            Log.warn("quarantine at cap (#{@max}); serving at reduced capacity")
+            @at_cap_logged = true
+          end
+          next
+        end
+
+        if @replacer.replace(index)
+          Native.record_quarantine_replacement(@server_id)
+          @outstanding += 1
+          @at_cap_logged = false
+        end
+      end
+    end
+  end
+end

data/lib/kino/ractor_supervisor.rb

@@ -7,19 +7,23 @@ module Kino
   # ractor, Exception from app code included) wakes it to 500 the in-flight
   # requests and respawn. Clean exits (queue drained) end supervision.
   class RactorSupervisor
-    attr_reader :respawns
-
-    def initialize(server_id, app, workers:, threads:, batch: 1, on_error: nil)
+    def initialize(server_id, app, workers:, threads:, batch: 1, hooks: nil, on_worker_exit: nil)
       @server_id = server_id
       @app = app
       @workers = workers
       @threads = threads
       @batch = batch
-      @on_error = on_error
-      @respawns = 0
+      @hooks = hooks
+      @on_worker_exit = on_worker_exit
       @draining = false
       @lock = Mutex.new
       @supervisor_threads = []
+      @worker_slots = {}
+      @slot_to_worker = {}
+      @replaced = {}
+      # The first replacement's index; `replace` increments before using it,
+      # so this starts one below the first free index (@workers).
+      @next_worker_index = @workers - 1
     end
 
     def start
@@ -32,43 +36,81 @@ module Kino
     def shutdown(timeout)
       @lock.synchronize { @draining = true }
       deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
-      @supervisor_threads.each do |thread|
+      @lock.synchronize { @supervisor_threads.dup }.each do |thread|
         remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
         thread.join([remaining, 0.01].max)
       end
     end
 
     def done?
-      @supervisor_threads.none?(&:alive?)
+      @lock.synchronize { @supervisor_threads.dup }.none?(&:alive?)
     end
 
     # Block until the workers exit on their own (drain elsewhere): join
     # without flipping the draining flag.
     def join
-      @supervisor_threads.each(&:join)
+      @lock.synchronize { @supervisor_threads.dup }.each(&:join)
+    end
+
+    # Replace the ractor owning slot `worker_id`: spawn a fresh supervised
+    # ractor, then quarantine the old ractor's slots. The old supervisor
+    # thread stays blocked in ractor.value on the wedged ractor (it and the
+    # ractor leak until process exit; a wedged ractor cannot be
+    # force-killed). Returns true if a replacement was spawned.
+    def replace(worker_id)
+      worker_index = @lock.synchronize { @slot_to_worker[worker_id] }
+      return false unless worker_index
+
+      # Idempotent per ractor: a stale monitor snapshot can list two sibling
+      # slots of the same ractor, only the first replaces it.
+      claimed = @lock.synchronize do
+        if @replaced.key?(worker_index)
+          false
+        else
+          @replaced[worker_index] = true
+          true
+        end
+      end
+      return false unless claimed
+
+      new_index = @lock.synchronize { @next_worker_index += 1 }
+      thread =
+        begin
+          supervise(new_index) # spawn FIRST, nothing quarantined yet
+        rescue
+          @lock.synchronize { @replaced.delete(worker_index) } # allow retry next tick
+          raise
+        end
+      slot_ids = @lock.synchronize { @worker_slots[worker_index] } || []
+      slot_ids.each { |id| Native.quarantine_slot(@server_id, id) } # quarantine only after success
+      @lock.synchronize { @supervisor_threads << thread }
+      true
     end
 
     private
 
     def supervise(index)
       Thread.new do
+        Thread.current.name = "supervisor-#{index}"
         crashes = 0
         loop do
-          ractor, worker_ids = spawn_worker
+          ractor, worker_ids = spawn_worker(index)
           begin
             ractor.value # blocks until the ractor terminates
+            HookFire.fire(@on_worker_exit, "on_worker_exit", index, nil) # clean exit: queue drained
             break        # clean exit: queue closed, workers drained
           rescue Ractor::Error => e
             # The ractor died mid-flight. Anything it was serving will never
             # be answered by Ruby: 500 those clients NOW (not when GC gets
             # around to dropping the dead heap), then decide on respawn.
             worker_ids.each { |id| Native.abort_inflight(@server_id, id) }
+            cause = (e.respond_to?(:cause) && e.cause) ? e.cause : e
+            HookFire.fire(@on_worker_exit, "on_worker_exit", index, cause)
             break if draining?
 
             crashes += 1
-            @lock.synchronize { @respawns += 1 }
-            cause = (e.respond_to?(:cause) && e.cause) ? e.cause : e
-            Native.log_error("worker ractor #{index} crashed (#{cause.class}: #{cause.message}); respawning")
+            Native.record_respawn(@server_id)
+            Log.error("worker-#{index} crashed (#{cause.class}: #{cause.message}); respawning")
             # Policy (crash recovery): unlimited respawn
             # keeps the server up under rare crashes but turns a
             # crash-on-every-request bug into a busy loop. A circuit breaker
@@ -82,15 +124,23 @@ module Kino
     # Fresh ractor + fresh native slots. Slots are never reused across
     # respawns: stale interrupt kicks and dead weak refs go down with the
     # old slot.
-    def spawn_worker
+    def spawn_worker(worker_index)
       worker_ids = Array.new(@threads) { Native.register_worker(@server_id) }
-      ractor = Ractor.new(@server_id, worker_ids, @app, @batch, @on_error) do |server_id, ids, app, batch, on_error|
-        ids.map do |id|
+      @lock.synchronize do
+        @worker_slots[worker_index] = worker_ids
+        worker_ids.each { |id| @slot_to_worker[id] = worker_index }
+      end
+      # Named so log lines from inside say which worker spoke: the ractor
+      # alone for a single thread, `worker-N/thread-M` for more.
+      ractor = Ractor.new(@server_id, worker_ids, @app, @batch, @hooks,
+        name: "worker-#{worker_index}") do |server_id, ids, app, batch, hooks|
+        ids.each_with_index.map do |id, position|
           Thread.new do
             # Crashes surface via Ractor#value in the supervisor; don't also
             # spray the backtrace to stderr from inside the dying ractor.
             Thread.current.report_on_exception = false
-            Kino::Worker.run(server_id, id, app, batch, on_error)
+            Thread.current.name = "thread-#{position + 1}" if ids.size > 1
+            Kino::Worker.run(server_id, id, app, batch, hooks)
           end
         end.each(&:join)
       end