kino 0.2.1-aarch64-linux → 0.4.0-aarch64-linux

data/lib/kino/log.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Kino
+  # Server log lines: the lifecycle notices, crash and respawn reports,
+  # hook failures, the failed-request report, and whatever apps write to
+  # `rack.errors`, all in one shape:
+  #
+  #   kino[4213] worker-3: after_worker_boot hook raised RuntimeError: boom
+  #
+  # The label is syslog's `ident[pid]` tag plus the source that spoke: the
+  # worker ractor and/or thread by name, `main` for neither. On color
+  # terminals the label is dim, yellow, or red by level; the message stays
+  # plain. Notes go to stdout, warnings and errors to stderr.
+  #
+  # Hooks may log through here too (`Kino::Log.info "cache warm"`). Every
+  # method is safe inside a worker ractor: the line is handed to the
+  # native layer, which owns the streams, so no ractor touches `$stdout`
+  # or `$stderr` itself.
+  module Log
+    # Frames shown in a failed-request report before the rest are folded.
+    FRAMES = 12
+
+    # The working directory at boot, stripped from backtrace frames so the
+    # app's own code reads `app/controllers/x.rb:9` rather than an
+    # absolute path (frozen: worker ractors read it).
+    WORKING_DIR = File.join(Dir.pwd, "").freeze
+
+    module_function
+
+    # @param message [#to_s]
+    # @return [void]
+    def info(message)
+      Native.log_line("info", source, message.to_s)
+    end
+
+    # @param message [#to_s]
+    # @return [void]
+    def warn(message)
+      Native.log_line("warn", source, message.to_s)
+    end
+
+    # @param message [#to_s]
+    # @return [void]
+    def error(message)
+      Native.log_line("error", source, message.to_s)
+    end
+
+    # The failed-request report: the request line, the error, and where it
+    # raised in the app, then the backtrace with the app's own frames
+    # first (relative to the working directory) and the rest folded.
+    #
+    #   500 GET /boom · RuntimeError: kaboom (app.rb:12:in 'explode')
+    #       app.rb:12:in 'explode'
+    #       /gems/rack-3.2.7/lib/rack/builder.rb:...
+    #       … 38 more
+    #
+    # @param error [Exception]
+    # @param env [Hash] the Rack env of the failed request
+    # @param status [Integer] the status the client got
+    # @return [void]
+    def exception(error, env, status: 500)
+      frames, depth = trace(error)
+      site = frames.first ? " (#{frames.first})" : ""
+      lines = ["#{status} #{env["REQUEST_METHOD"]} #{env["PATH_INFO"]} · #{error.class}: #{error.message}#{site}"]
+      frames.each { |frame| lines << "    #{frame}" }
+      lines << "    … #{depth - frames.size} more" if depth > frames.size
+      error(lines.join("\n"))
+    end
+
+    # The `kino[<pid>] <source>:` tag a line from here carries.
+    # @return [String]
+    def label
+      "kino[#{Process.pid}] #{source}:"
+    end
+
+    # Who is speaking: the ractor's name, the thread's name, both joined
+    # with a slash, or `main` when neither is named. Kino names its
+    # worker ractors and threads `worker-N`.
+    # @return [String]
+    def source
+      parts = [Ractor.current.name, Thread.current.name].compact
+      parts.empty? ? "main" : parts.join("/")
+    end
+
+    # The backtrace as [frames, depth]: each frame relativized to the
+    # working directory, the app's own frames floated to the front (the
+    # raise site in your code reads first; gem and stdlib frames keep
+    # their order below), capped at FRAMES; depth is the real length.
+    def trace(error)
+      raw = error.backtrace || []
+      app, rest = raw.map { |frame| frame.delete_prefix(WORKING_DIR) }.partition { |frame| app_frame?(frame) }
+      [(app + rest).first(FRAMES), raw.size]
+    end
+
+    # A project-relative path (the working-directory prefix came off, so
+    # it does not start with `/`) that is not a synthetic frame (`(eval)`,
+    # `<internal:...>`). Gem and stdlib frames stay absolute.
+    def app_frame?(frame)
+      !frame.start_with?("/", "<", "(")
+    end
+
+    private_class_method :trace, :app_frame?
+  end
+end

data/lib/kino/quarantine_monitor.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Kino
+  # @private
+  # Polls per-slot busy_ms and, past the timeout, quarantines a wedged slot
+  # and asks the replacer to spawn a fresh worker. Runs one thread on the
+  # main ractor (uncontended by wedged worker ractors, so it stays
+  # responsive in :ractor mode). Never interrupts the wedged worker.
+  class QuarantineMonitor
+    def initialize(server_id:, timeout_ms:, max:, replacer:, tick: 0.5)
+      @server_id = server_id
+      @timeout_ms = timeout_ms
+      @max = max
+      @replacer = replacer
+      @tick = tick
+      @outstanding = 0
+      @at_cap_logged = false
+      @running = false
+      @thread = nil
+    end
+
+    def start
+      @running = true
+      @thread = Thread.new do
+        Thread.current.name = "quarantine"
+        run
+      end
+      self
+    end
+
+    def stop
+      @running = false
+      @thread&.join(@tick * 2)
+    end
+
+    private
+
+    def run
+      tick while @running
+    rescue => e
+      Log.error("quarantine monitor crashed: #{e.class}: #{e.message}")
+    end
+
+    def tick
+      scan_slots
+    rescue => e
+      # A bad tick must never kill the monitor.
+      Log.error("quarantine tick error: #{e.class}: #{e.message}")
+    ensure
+      sleep @tick
+    end
+
+    def scan_slots
+      Native.worker_stats(@server_id).each do |index, _served, _in_flight, busy_ms, quarantined|
+        next if quarantined || busy_ms <= @timeout_ms
+
+        if @outstanding >= @max
+          unless @at_cap_logged
+            Log.warn("quarantine at cap (#{@max}); serving at reduced capacity")
+            @at_cap_logged = true
+          end
+          next
+        end
+
+        if @replacer.replace(index)
+          Native.record_quarantine_replacement(@server_id)
+          @outstanding += 1
+          @at_cap_logged = false
+        end
+      end
+    end
+  end
+end

data/lib/kino/ractor_supervisor.rb

@@ -7,19 +7,23 @@ module Kino
   # ractor, Exception from app code included) wakes it to 500 the in-flight
   # requests and respawn. Clean exits (queue drained) end supervision.
   class RactorSupervisor
-    attr_reader :respawns
-
-    def initialize(server_id, app, workers:, threads:, batch: 1, on_error: nil)
+    def initialize(server_id, app, workers:, threads:, batch: 1, hooks: nil, on_worker_exit: nil)
       @server_id = server_id
       @app = app
       @workers = workers
       @threads = threads
       @batch = batch
-      @on_error = on_error
-      @respawns = 0
+      @hooks = hooks
+      @on_worker_exit = on_worker_exit
       @draining = false
       @lock = Mutex.new
       @supervisor_threads = []
+      @worker_slots = {}
+      @slot_to_worker = {}
+      @replaced = {}
+      # The first replacement's index; `replace` increments before using it,
+      # so this starts one below the first free index (@workers).
+      @next_worker_index = @workers - 1
     end
 
     def start
@@ -32,43 +36,81 @@ module Kino
     def shutdown(timeout)
       @lock.synchronize { @draining = true }
       deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
-      @supervisor_threads.each do |thread|
+      @lock.synchronize { @supervisor_threads.dup }.each do |thread|
         remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
         thread.join([remaining, 0.01].max)
       end
     end
 
     def done?
-      @supervisor_threads.none?(&:alive?)
+      @lock.synchronize { @supervisor_threads.dup }.none?(&:alive?)
     end
 
     # Block until the workers exit on their own (drain elsewhere): join
     # without flipping the draining flag.
     def join
-      @supervisor_threads.each(&:join)
+      @lock.synchronize { @supervisor_threads.dup }.each(&:join)
+    end
+
+    # Replace the ractor owning slot `worker_id`: spawn a fresh supervised
+    # ractor, then quarantine the old ractor's slots. The old supervisor
+    # thread stays blocked in ractor.value on the wedged ractor (it and the
+    # ractor leak until process exit; a wedged ractor cannot be
+    # force-killed). Returns true if a replacement was spawned.
+    def replace(worker_id)
+      worker_index = @lock.synchronize { @slot_to_worker[worker_id] }
+      return false unless worker_index
+
+      # Idempotent per ractor: a stale monitor snapshot can list two sibling
+      # slots of the same ractor, only the first replaces it.
+      claimed = @lock.synchronize do
+        if @replaced.key?(worker_index)
+          false
+        else
+          @replaced[worker_index] = true
+          true
+        end
+      end
+      return false unless claimed
+
+      new_index = @lock.synchronize { @next_worker_index += 1 }
+      thread =
+        begin
+          supervise(new_index) # spawn FIRST, nothing quarantined yet
+        rescue
+          @lock.synchronize { @replaced.delete(worker_index) } # allow retry next tick
+          raise
+        end
+      slot_ids = @lock.synchronize { @worker_slots[worker_index] } || []
+      slot_ids.each { |id| Native.quarantine_slot(@server_id, id) } # quarantine only after success
+      @lock.synchronize { @supervisor_threads << thread }
+      true
     end
 
     private
 
     def supervise(index)
       Thread.new do
+        Thread.current.name = "supervisor-#{index}"
         crashes = 0
         loop do
-          ractor, worker_ids = spawn_worker
+          ractor, worker_ids = spawn_worker(index)
           begin
             ractor.value # blocks until the ractor terminates
+            HookFire.fire(@on_worker_exit, "on_worker_exit", index, nil) # clean exit: queue drained
             break        # clean exit: queue closed, workers drained
           rescue Ractor::Error => e
             # The ractor died mid-flight. Anything it was serving will never
             # be answered by Ruby: 500 those clients NOW (not when GC gets
             # around to dropping the dead heap), then decide on respawn.
             worker_ids.each { |id| Native.abort_inflight(@server_id, id) }
+            cause = (e.respond_to?(:cause) && e.cause) ? e.cause : e
+            HookFire.fire(@on_worker_exit, "on_worker_exit", index, cause)
             break if draining?
 
             crashes += 1
-            @lock.synchronize { @respawns += 1 }
-            cause = (e.respond_to?(:cause) && e.cause) ? e.cause : e
-            Native.log_error("worker ractor #{index} crashed (#{cause.class}: #{cause.message}); respawning")
+            Native.record_respawn(@server_id)
+            Log.error("worker-#{index} crashed (#{cause.class}: #{cause.message}); respawning")
             # Policy (crash recovery): unlimited respawn
             # keeps the server up under rare crashes but turns a
             # crash-on-every-request bug into a busy loop. A circuit breaker
@@ -82,15 +124,23 @@ module Kino
     # Fresh ractor + fresh native slots. Slots are never reused across
     # respawns: stale interrupt kicks and dead weak refs go down with the
     # old slot.
-    def spawn_worker
+    def spawn_worker(worker_index)
       worker_ids = Array.new(@threads) { Native.register_worker(@server_id) }
-      ractor = Ractor.new(@server_id, worker_ids, @app, @batch, @on_error) do |server_id, ids, app, batch, on_error|
-        ids.map do |id|
+      @lock.synchronize do
+        @worker_slots[worker_index] = worker_ids
+        worker_ids.each { |id| @slot_to_worker[id] = worker_index }
+      end
+      # Named so log lines from inside say which worker spoke: the ractor
+      # alone for a single thread, `worker-N/thread-M` for more.
+      ractor = Ractor.new(@server_id, worker_ids, @app, @batch, @hooks,
+        name: "worker-#{worker_index}") do |server_id, ids, app, batch, hooks|
+        ids.each_with_index.map do |id, position|
           Thread.new do
             # Crashes surface via Ractor#value in the supervisor; don't also
             # spray the backtrace to stderr from inside the dying ractor.
             Thread.current.report_on_exception = false
-            Kino::Worker.run(server_id, id, app, batch, on_error)
+            Thread.current.name = "thread-#{position + 1}" if ids.size > 1
+            Kino::Worker.run(server_id, id, app, batch, hooks)
           end
         end.each(&:join)
       end