kino 0.6.0 → 0.7.0

data/lib/kino/server.rb

@@ -68,6 +68,16 @@ module Kino
       @bind = settings[:bind]
       @requested_port = settings[:port]
       @workers = Integer(settings[:workers])
+      # The pool ceiling; equal to the floor for a fixed pool.
+      @max_workers = settings[:max_workers].nil? ? @workers : Integer(settings[:max_workers])
+      if @max_workers < @workers
+        raise ArgumentError, "max_workers (#{@max_workers}) must be at least workers (#{@workers})"
+      end
+      @scale_down_after = settings[:scale_down_after].nil? ? 30.0 : Float(settings[:scale_down_after])
+      raise ArgumentError, "scale_down_after must be positive" unless @scale_down_after.positive?
+      if !settings[:scale_down_after].nil? && !elastic?
+        Log.warn("scale_down_after has no effect unless max_workers is above workers")
+      end
       @on_error = validate_hook(settings[:on_error], :on_error)
       @after_worker_boot = validate_hook(settings[:after_worker_boot], :after_worker_boot)
       @after_request_complete = validate_hook(settings[:after_request_complete], :after_request_complete)
@@ -81,8 +91,9 @@ module Kino
         # The access log's GC and allocation figures come from the VM's
         # process-wide counters, so they are measured only where one
         # request at a time can own them: the GVL serializes :threaded
-        # mode, and a single ractor has nothing to race.
-        access_timing: !!settings[:log_requests] && (@mode == :threaded || @workers == 1)
+        # mode, and a single ractor (a pool that can never grow past one)
+        # has nothing to race.
+        access_timing: !!settings[:log_requests] && (@mode == :threaded || @max_workers == 1)
       )
       # Default threads per mode: 1 in :ractor (threads inside a ractor
       # share its lock; a measured +17% on fast handlers; raise `workers`
@@ -126,10 +137,10 @@ module Kino
         else
           @workers * @threads
         end
-      @worker_threads = []
-      @worker_threads_lock = Mutex.new
       @supervisor = nil
+      @threaded_pool = nil
       @quarantine_monitor = nil
+      @pool_scaler = nil
       @started = false
     end
 
@@ -158,7 +169,8 @@ module Kino
           tls_cert: @tls&.fetch(:cert), tls_key: @tls&.fetch(:key),
           http2: @http2,
           lanes: @lanes, log_requests: @log_requests,
-          mode: @mode.to_s, workers: @workers, threads: @threads, batch: @batch,
+          mode: @mode.to_s, workers: @workers, max_workers: @max_workers,
+          threads: @threads, batch: @batch,
           control_bind: @control_bind, control_token: @control_token
         )
         booted = true
@@ -169,12 +181,15 @@ module Kino
       # lifetime so in-flight buffers survive even a worker ractor crash.
       @pin_keeper = Native.pin_keeper(@id)
       if @mode == :ractor
+        warn_scheduler_cap
         @supervisor = RactorSupervisor.new(@id, @app, workers: @workers, threads: @threads,
           batch: @batch, hooks: @worker_hooks, on_worker_exit: @on_worker_exit).start
       else
-        @worker_threads = (@workers * @threads).times.map { spawn_worker_thread }
+        @threaded_pool = ThreadedPool.new(@id, @app, threads: @threads, batch: @batch,
+          hooks: @worker_hooks, on_worker_exit: @on_worker_exit).start(@workers)
       end
       start_quarantine_monitor if @quarantine_timeout_ms
+      start_pool_scaler if elastic?
       Native.control_ready(@id)
       HookFire.fire(@after_boot, "after_boot")
       @started = true
@@ -192,6 +207,7 @@ module Kino
     def shutdown(timeout: nil)
       return unless @started
 
+      @pool_scaler&.stop
       @quarantine_monitor&.stop
       deadline = monotonic_now + (timeout || @shutdown_timeout)
       Native.stop_accepting(@id)
@@ -224,7 +240,6 @@ module Kino
       # The runtime is gone, so hyper has dropped every pinned buffer;
       # the keeper (and the strings it marked) may now be collected.
       @pin_keeper = nil
-      @worker_threads.clear
       @started = false
       remove_pidfile if @pidfile
       nil
@@ -233,7 +248,7 @@ module Kino
     # Block until every worker has exited (i.e. until shutdown).
     # @return [void]
     def wait
-      @supervisor ? @supervisor.join : @worker_threads.each(&:join)
+      pool.join
     end
 
     # Production entry point: build the server and {#run} it. The `kino`
@@ -298,19 +313,22 @@ module Kino
     #   lanes mode) once started
     def stats
       base = {
-        mode: @mode, lanes: @lanes, workers: @workers, threads: @threads,
-        batch: @batch, respawns: 0
+        mode: @mode, lanes: @lanes, workers: @workers, max_workers: @max_workers,
+        threads: @threads, batch: @batch, respawns: 0,
+        active_workers: @workers, scale_ups: 0, scale_downs: 0
       }
       return base unless @started
 
       queued, in_flight, served, rejected, timeouts, respawns, lane_depths = Native.server_stats(@id)
       base.merge!(queued:, in_flight:, served:, rejected:, timeouts:, respawns:)
       base[:lane_depths] = lane_depths if lane_depths
+      active_workers, _max_workers, scale_ups, scale_downs = Native.pool_stats(@id)
+      base.merge!(active_workers:, scale_ups:, scale_downs:)
       rows = Native.worker_stats(@id)
-      base[:worker_status] = rows.map do |index, served, in_flight, busy_ms, quarantined|
-        {index:, served:, in_flight:, busy_ms:, quarantined:}
+      base[:worker_status] = rows.map do |index, served, in_flight, busy_ms, quarantined, retired|
+        {index:, served:, in_flight:, busy_ms:, quarantined:, retired:}
       end
-      base[:quarantined] = rows.count { |_index, _served, _in_flight, _busy_ms, quarantined| quarantined }
+      base[:quarantined] = rows.count { |row| row[4] }
       count, sum_seconds = Native.queue_time(@id)
       base[:queue_time] = {count:, sum_seconds:}
       base
@@ -318,63 +336,38 @@ module Kino
 
     private
 
-    # Register a fresh dispatch slot and run a worker thread on it; returns
-    # the thread. Used at boot and by the quarantine replacer.
-    def spawn_worker_thread
-      worker_id = Native.register_worker(@id)
-      Thread.new do
-        # Named so log lines from inside say which worker spoke.
-        Thread.current.name = "worker-#{worker_id}"
-        error = nil
-        begin
-          Worker.run(@id, worker_id, @app, @batch, @worker_hooks)
-        rescue Exception => e # rubocop:disable Lint/RescueException -- a hard crash in a threaded worker thread
-          error = e
-          raise
-        ensure
-          HookFire.fire(@on_worker_exit, "on_worker_exit", worker_id, error)
-        end
-      end
+    # The worker pool this mode runs: the ractor supervisor, or the
+    # threaded pool. Both spawn, retire, replace and join workers behind
+    # the same methods.
+    def pool
+      @supervisor || @threaded_pool
     end
 
-    # Track a replacement thread spawned outside the initial pool assignment
-    # (the quarantine replacer) so shutdown's join/done?/kill sweeps see it.
-    def track_replacement_thread(thread)
-      @worker_threads_lock.synchronize { @worker_threads << thread }
+    # Both pools are quarantine replacers: replace(worker_id) spawns a
+    # replacement worker, then quarantines the wedged slot.
+    def start_quarantine_monitor
+      @quarantine_monitor = QuarantineMonitor.new(
+        server_id: @id, timeout_ms: @quarantine_timeout_ms,
+        max: @quarantine_max, replacer: pool
+      ).start
     end
 
-    # @private
-    # The :threaded-mode quarantine replacer: spawns a replacement worker
-    # thread, quarantines the wedged slot, then tracks the new thread so
-    # shutdown's join/done?/kill sweeps see it. Built from bound Method
-    # objects instead of a server reference, so it drives the server
-    # through those methods without send or instance_variable_get.
-    class ThreadedReplacer
-      def initialize(server_id:, spawner:, tracker:)
-        @server_id = server_id
-        @spawner = spawner
-        @tracker = tracker
-      end
+    # Ruby's M:N scheduler runs non-main ractors' Ruby code on at most
+    # RUBY_MAX_CPU native threads (default 8). Workers past that cap share
+    # timeslices instead of adding parallelism, and a fresh ractor can wait
+    # seconds for its first one while the others are CPU-bound.
+    def warn_scheduler_cap
+      cap = Integer(ENV.fetch("RUBY_MAX_CPU", "8"), exception: false) || 8
+      return if @max_workers <= cap
 
-      def replace(worker_id)
-        thread = @spawner.call # spawn FIRST (may raise ThreadError)
-        Native.quarantine_slot(@server_id, worker_id) # quarantine after success
-        @tracker.call(thread)
-        true
-      end
+      Log.warn("#{@max_workers} ractor workers exceed RUBY_MAX_CPU=#{cap}: only #{cap} can run " \
+        "Ruby code at once; set RUBY_MAX_CPU=#{@max_workers} for CPU-bound apps")
     end
-    private_constant :ThreadedReplacer
 
-    # A replacer.replace(worker_id) spawns a replacement worker, then
-    # quarantines the wedged slot, mode-appropriately. In :ractor the
-    # supervisor is the replacer; in :threaded a small object over
-    # spawn_worker_thread.
-    def start_quarantine_monitor
-      replacer = @supervisor || ThreadedReplacer.new(server_id: @id, spawner: method(:spawn_worker_thread),
-        tracker: method(:track_replacement_thread))
-      @quarantine_monitor = QuarantineMonitor.new(
-        server_id: @id, timeout_ms: @quarantine_timeout_ms,
-        max: @quarantine_max, replacer: replacer
+    def start_pool_scaler
+      @pool_scaler = PoolScaler.new(
+        server_id: @id, pool: pool, floor: @workers, ceiling: @max_workers,
+        scale_down_after: @scale_down_after
       ).start
     end
 
@@ -400,6 +393,11 @@ module Kino
       Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end
 
+    # A pool that can grow: the ceiling is above the floor.
+    def elastic?
+      @max_workers > @workers
+    end
+
     # Default connection cap: most of the process open-file limit. A
     # connection flood's failure mode is descriptor exhaustion, and in
     # :ractor/:threaded mode the app's own sockets and files share this
@@ -474,34 +472,15 @@ module Kino
     end
 
     def join_workers(deadline)
-      if @supervisor
-        @supervisor.shutdown([deadline - monotonic_now, 0].max)
-      else
-        threads = @worker_threads_lock.synchronize { @worker_threads.dup }
-        threads.each do |thread|
-          thread.join([deadline - monotonic_now, 0.01].max)
-        end
-      end
+      pool.shutdown([deadline - monotonic_now, 0].max)
     end
 
     def workers_done?
-      if @supervisor
-        @supervisor.done?
-      else
-        threads = @worker_threads_lock.synchronize { @worker_threads.dup }
-        threads.none?(&:alive?)
-      end
+      pool.done?
     end
 
     def kill_stragglers
-      if @supervisor
-        # Ractors cannot be force-killed; their clients were already freed
-        # by abort_all_inflight. The stuck ractor leaks until process exit.
-        Log.error("shutdown deadline passed with stuck ractor workers") unless @supervisor.done?
-      else
-        threads = @worker_threads_lock.synchronize { @worker_threads.dup }
-        threads.each { |thread| thread.kill if thread.alive? }
-      end
+      pool.kill_stragglers
     end
 
     # Policy (mode resolution): when is an app safe for ractor

data/lib/kino/slot_bank.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Kino
+  # @private
+  # Dispatch slots for a worker pool: fresh ones from the native registry,
+  # or ones that retired workers handed back, reset for their next
+  # occupant. The native side never removes a slot, so recycling is what
+  # keeps the slot table from growing as an elastic pool breathes. Only
+  # cleanly exited workers return slots; a crashed worker's slots are
+  # abandoned (stale interrupt kicks and dead weak refs go down with
+  # them).
+  class SlotBank
+    def initialize(server_id)
+      @server_id = server_id
+      @free = []
+      @lock = Mutex.new
+    end
+
+    def claim
+      id = @lock.synchronize { @free.pop }
+      return Native.register_worker(@server_id) unless id
+
+      Native.reset_slot(@server_id, id)
+      id
+    end
+
+    def release(ids)
+      @lock.synchronize { @free.concat(ids) }
+    end
+  end
+end

data/lib/kino/templates/kino.rb.tt

@@ -40,6 +40,21 @@
 # `workers` instead.
 # threads 1
 
+## Elastic pool (experimental)
+#
+# Let the pool grow past `workers` under load and shrink back when
+# idle: one worker is added every 100 ms while requests wait in the
+# queue, and a worker above `workers` retires after `scale_down_after`
+# seconds idle. Helps apps that wait on databases or other services;
+# pure CPU work gains nothing past the core count. Unset (the default)
+# keeps the pool fixed. In :ractor mode, Ruby runs at most RUBY_MAX_CPU
+# (default 8) ractors' Ruby code at once; set that variable to match
+# `max_workers` on bigger boxes.
+# max_workers 32
+
+# Seconds an extra worker must sit idle before it is retired.
+# scale_down_after 30
+
 ## Dispatch mode
 #
 # :auto     - picks :ractor when your app supports it, else :threaded.

data/lib/kino/threaded_pool.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+module Kino
+  # @private
+  # The :threaded-mode worker pool: `workers` groups of `threads` plain
+  # Threads, each thread on its own dispatch slot. A group is one ractor's
+  # worth of capacity, so `workers` and `max_workers` mean the same thing
+  # in both modes. Behind PoolScaler here (`grow`, `retire`, `groups`,
+  # `active_count`), the quarantine monitor's replacer (`replace`), and
+  # the join/kill sweeps Server#shutdown runs.
+  class ThreadedPool
+    def initialize(server_id, app, threads:, batch: 1, hooks: nil, on_worker_exit: nil)
+      @server_id = server_id
+      @app = app
+      @threads = threads
+      @batch = batch
+      @hooks = hooks
+      @on_worker_exit = on_worker_exit
+      @lock = Mutex.new
+      # index => {slots:, threads:}; the groups asked to leave; quarantine
+      # replacements (one thread each, standing in for a wedged slot:
+      # neither counted nor retired, so the wedged group keeps counting as
+      # the capacity it still is). Retired groups hand their slots back to
+      # the bank.
+      @groups = {}
+      @slot_to_group = {}
+      @retiring = {}
+      @bank = SlotBank.new(server_id)
+      @replacements = {}
+      @wedged = {}
+      @next_index = -1
+    end
+
+    def start(workers)
+      workers.times { spawn_group }
+      report_active
+      self
+    end
+
+    # Groups that are staying: quarantine replacements aside, and minus
+    # those already told to leave. A retiring group leaves the count at
+    # once, not when its last thread exits, so the scaler never sees a
+    # stale surplus and retires past its floor.
+    def active_count
+      @lock.synchronize do
+        @groups.count { |index, _| !@replacements.key?(index) && !@retiring.key?(index) }
+      end
+    end
+
+    # Group index => slot ids for every group the scaler may retire: not
+    # already leaving, not wedged, not a quarantine replacement.
+    def groups
+      reap
+      @lock.synchronize do
+        @groups
+          .reject { |index, _| @retiring.key?(index) || @wedged.key?(index) || @replacements.key?(index) }
+          .transform_values { |group| group[:slots].dup }
+      end
+    end
+
+    # Add one group; returns its index.
+    def grow
+      reap
+      index = spawn_group
+      report_active
+      index
+    end
+
+    # Send a group home: its slots stop receiving work now, each thread
+    # finishes what it holds and leaves at its next idle tick, and the
+    # slots come back to the free list once every thread has exited.
+    def retire(index)
+      slots = @lock.synchronize do
+        next nil unless @groups.key?(index) && !@retiring.key?(index)
+
+        @retiring[index] = true
+        @groups[index][:slots]
+      end
+      return false unless slots
+
+      slots.each { |id| Native.retire_slot(@server_id, id) }
+      true
+    end
+
+    # The quarantine replacer: spawn a replacement thread on a fresh slot
+    # FIRST (may raise ThreadError), then quarantine the wedged slot.
+    def replace(worker_id)
+      spawn_group(slots: 1, replacement: true)
+      Native.quarantine_slot(@server_id, worker_id)
+      @lock.synchronize do
+        wedged = @slot_to_group[worker_id]
+        @wedged[wedged] = true if wedged
+      end
+      true
+    end
+
+    # Join every thread up to the (numeric) deadline.
+    def shutdown(timeout)
+      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+      all_threads.each do |thread|
+        remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        thread.join([remaining, 0.01].max)
+      end
+    end
+
+    def done?
+      all_threads.none?(&:alive?)
+    end
+
+    # Block until every thread exits on its own (drain elsewhere).
+    def join
+      all_threads.each(&:join)
+    end
+
+    def kill_stragglers
+      all_threads.each { |thread| thread.kill if thread.alive? }
+    end
+
+    private
+
+    def all_threads
+      @lock.synchronize { @groups.values.flat_map { |group| group[:threads] } }
+    end
+
+    # The group is in the table before its first thread starts, so a
+    # ThreadError partway through leaves nothing untracked for shutdown.
+    def spawn_group(slots: @threads, replacement: false)
+      group = {slots: [], threads: []}
+      index = @lock.synchronize do
+        @next_index += 1
+        @groups[@next_index] = group
+        @replacements[@next_index] = true if replacement
+        @next_index
+      end
+      slots.times do
+        id = @bank.claim
+        @lock.synchronize do
+          group[:slots] << id
+          @slot_to_group[id] = index
+        end
+        thread = spawn_thread(id)
+        @lock.synchronize { group[:threads] << thread }
+      end
+      index
+    end
+
+    def spawn_thread(worker_id)
+      Thread.new do
+        # Named so log lines from inside say which worker spoke.
+        Thread.current.name = "worker-#{worker_id}"
+        error = nil
+        begin
+          Worker.run(@server_id, worker_id, @app, @batch, @hooks)
+        rescue Exception => e # rubocop:disable Lint/RescueException -- a hard crash in a threaded worker thread
+          error = e
+          raise
+        ensure
+          HookFire.fire(@on_worker_exit, "on_worker_exit", worker_id, error)
+        end
+      end
+    end
+
+    # Retiring groups whose threads have all exited give their slots back
+    # and leave the table. Runs before every pool decision, so the count
+    # the control plane sees never lags by more than a scaler tick.
+    def reap
+      freed = @lock.synchronize do
+        done = @retiring.keys.select { |index| @groups[index][:threads].none?(&:alive?) }
+        done.flat_map do |index|
+          group = @groups.delete(index)
+          @retiring.delete(index)
+          group[:slots].each { |id| @slot_to_group.delete(id) }
+          group[:slots]
+        end
+      end
+      return if freed.empty?
+
+      @bank.release(freed)
+      report_active
+    end
+
+    def report_active
+      Native.set_active_workers(@server_id, active_count)
+    end
+  end
+end

data/lib/kino/version.rb

@@ -2,5 +2,5 @@
 
 module Kino
   # The gem version (single source of truth; ext/kino/Cargo.toml syncs).
-  VERSION = "0.6.0"
+  VERSION = "0.7.0"
 end

data/lib/kino.rb

@@ -57,8 +57,12 @@ require_relative "kino/configuration"
 require_relative "kino/hook_fire"
 require_relative "kino/worker_hooks"
 require_relative "kino/worker"
+require_relative "kino/slot_bank"
 require_relative "kino/ractor_supervisor"
+require_relative "kino/threaded_pool"
+require_relative "kino/monitor"
 require_relative "kino/quarantine_monitor"
+require_relative "kino/pool_scaler"
 require_relative "kino/server"
 
 # Hand the frozen shareable singletons to the native layer: it sets them

data/lib/rackup/handler/kino.rb

@@ -9,13 +9,13 @@ module Rackup
     module Kino
       # Host option name => Kino setting plus the coercion it needs: rackup
       # hands `-O NAME=VALUE` values (and its own -p) over as strings.
-      OPTION_MAP = {
+      OPTION_MAP = Ractor.make_shareable({
         Host: [:bind, ->(value) { value.to_s }],
         Port: [:port, ->(value) { Integer(value) }],
         Workers: [:workers, ->(value) { Integer(value) }],
         Threads: [:threads, ->(value) { Integer(value) }],
         Mode: [:mode, ->(value) { value.to_sym }]
-      }.freeze
+      })
       private_constant :OPTION_MAP
 
       # Boot a server for `app` and block until it shuts down, the way the
@@ -27,7 +27,7 @@ module Rackup
       #   want a handle on it
       # @return [::Kino::Server] the stopped server, after shutdown
       def self.run(app, **options)
-        require "kino"
+        require "kino" # audition:disable runtime-require
         server = ::Kino::Server.new(app, **server_options(options))
         yield server if block_given?
         server.run
@@ -58,7 +58,7 @@ module Rackup
       # @param options [Hash{Symbol => Object}]
       # @return [Hash{Symbol => Object}]
       def self.server_options(options)
-        require "kino"
+        require "kino" # audition:disable runtime-require
         options = options.dup
         host_defaults = {}
         if (typed = options.delete(:user_supplied_options))

data/sig/kino.rbs

@@ -110,6 +110,8 @@ module Kino
       def bind: (String host) -> untyped
       def port: (int port) -> untyped
       def workers: (int count) -> untyped
+      def max_workers: (int count) -> untyped
+      def scale_down_after: (Numeric seconds) -> untyped
       def threads: (int count) -> untyped
       def mode: (Symbol | String mode) -> untyped
       def queue_depth: (int depth) -> untyped

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kino
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Yaroslav Markin
@@ -219,12 +219,16 @@ files:
 - lib/kino/input.rb
 - lib/kino/log.rb
 - lib/kino/logger.rb
+- lib/kino/monitor.rb
 - lib/kino/null_input.rb
+- lib/kino/pool_scaler.rb
 - lib/kino/quarantine_monitor.rb
 - lib/kino/ractor_supervisor.rb
 - lib/kino/server.rb
+- lib/kino/slot_bank.rb
 - lib/kino/stream.rb
 - lib/kino/templates/kino.rb.tt
+- lib/kino/threaded_pool.rb
 - lib/kino/version.rb
 - lib/kino/worker.rb
 - lib/kino/worker_hooks.rb