raptor 0.22.3 → 0.24.0

data/lib/raptor/cluster.rb

@@ -9,16 +9,19 @@ require "concurrent/utility/processor_counter"
 require "rack/builder"
 require "ractor-pool"
 
-require_relative "log"
 require_relative "binder"
 require_relative "control_server"
-require_relative "reactor"
-require_relative "server"
-require_relative "reuseport_bpf"
 require_relative "http1"
 require_relative "http2"
+require_relative "log"
+require_relative "plugin"
+require_relative "raptor_native"
+require_relative "reactor"
+require_relative "reuseport_bpf"
+require_relative "server"
 require_relative "stats"
 require_relative "systemd"
+require_relative "version"
 
 module Raptor
   # Forks and supervises worker processes. Handles graceful shutdown on
@@ -100,7 +103,7 @@ module Raptor
     # @rbs @access_log_io: IO?
     # @rbs @launch_command: String?
     # @rbs @launch_argv: Array[String]?
-    # @rbs @on_error: ^(Hash[String, untyped]?, Exception) -> void | nil
+    # @rbs @on_error: (^(Hash[String, untyped]?, Exception) -> void)?
     # @rbs @binder: Binder
     # @rbs @server_port: Integer
     # @rbs @app: untyped
@@ -115,7 +118,7 @@ module Raptor
     # @rbs @hot_restart_requested: bool
     # @rbs @refork_thresholds: Array[Integer]
     # @rbs @refork_requested: bool
-    # @rbs @refork_threshold_idx: Integer
+    # @rbs @refork_threshold_index: Integer
     # @rbs @seed_pid: Integer?
     # @rbs @seed_ready: bool
     # @rbs @seed_vacated_index: Integer?
@@ -124,6 +127,12 @@ module Raptor
     # @rbs @resp_r: IO?
     # @rbs @resp_w: IO?
     # @rbs @bpf_active: bool
+    # @rbs @plugins: Array[Plugin]
+
+    # Returns the cluster configuration available to plugins.
+    #
+    # @return [Hash{Symbol => untyped}]
+    attr_reader :options #: Hash[Symbol, untyped]
 
     # Creates a new Cluster with the given options.
     #
@@ -153,6 +162,7 @@ module Raptor
     # @option options [Array<Proc>] :before_worker_boot procs called with the worker index before it begins serving
     # @option options [Array<Proc>] :before_worker_shutdown procs called with the worker index before its graceful shutdown
     # @option options [Array<Proc>] :before_refork procs called in a worker before it transitions to a seed
+    # @option options [Array<String, Symbol>] :plugins plugins to load and start in order
     # @option options [String, nil] :stats_file path to write per-worker stats JSON, or nil to disable
     # @option options [String, nil] :control_url `unix://` URL serving cluster stats, or nil to disable
     # @option options [String, nil] :pid_file path to write the master PID to, or nil to disable
@@ -166,6 +176,8 @@ module Raptor
     #
     # @rbs (Hash[Symbol, untyped] options) -> void
     def initialize(options)
+      @options = options = options.dup
+      @plugins = []
       @drain_accept_queue = options[:drain_accept_queue]
       @worker_count = options[:workers]
       @http1_ractor_count = options[:http1][:ractors] || self.class.default_http1_ractor_count(@worker_count)
@@ -183,10 +195,10 @@ module Raptor
       @worker_timeout = options[:worker_timeout]
       @worker_drain_timeout = options[:worker_drain_timeout]
       @worker_shutdown_timeout = options[:worker_shutdown_timeout]
-      @before_fork = Array(options[:before_fork])
-      @before_worker_boot = Array(options[:before_worker_boot])
-      @before_worker_shutdown = Array(options[:before_worker_shutdown])
-      @before_refork = Array(options[:before_refork])
+      @before_fork = options[:before_fork] = Array(options[:before_fork]).dup
+      @before_worker_boot = options[:before_worker_boot] = Array(options[:before_worker_boot]).dup
+      @before_worker_shutdown = options[:before_worker_shutdown] = Array(options[:before_worker_shutdown]).dup
+      @before_refork = options[:before_refork] = Array(options[:before_refork]).dup
       @stats_file = options[:stats_file]
       @control_server = ControlServer.new(options[:control_url]) { stats_hash } if options[:control_url]
       @pid_file = options[:pid_file]
@@ -200,7 +212,7 @@ module Raptor
 
       Dir.chdir(options[:chdir]) if options[:chdir]
 
-      inherited_fds = if raw = ENV.delete(INHERITED_FDS_ENV)
+      inherited_fds = if (raw = ENV.delete(INHERITED_FDS_ENV))
         JSON.parse(raw)
       elsif (systemd_fds = Systemd.listen_fds).any?
         Systemd.clear_listen_env
@@ -225,7 +237,7 @@ module Raptor
 
       @refork_thresholds = normalize_refork_thresholds(options[:refork_after])
       @refork_requested = false
-      @refork_threshold_idx = 0
+      @refork_threshold_index = 0
       @seed_pid = nil
       @seed_ready = false
       @seed_vacated_index = nil
@@ -261,42 +273,63 @@ module Raptor
 
       File.open(@pid_file, File::CREAT | File::EXCL | File::WRONLY) { |file| file.write(Process.pid.to_s) } if @pid_file
 
-      @bpf_active = ReuseportBPF.setup(@worker_count)
-
-      @control_server&.bind
-      @worker_count.times { |index| spawn_worker(index) }
-      @control_server&.start
-
-      stats_file_thread = if @stats_file
-        Thread.new do
-          Thread.current.name = "Stats File Writer"
+      @bpf_active = ReuseportBPF.setup(@worker_count, @binder.listeners.grep(TCPServer))
 
-          write_stats_file_loop
+      begin
+        Array(@options[:plugins]).each do |name|
+          plugin = Plugin.load(name)
+          @plugins << plugin
+          plugin.start(self)
         end
-      end
 
-      Systemd.notify("READY=1\nMAINPID=#{Process.pid}")
+        @control_server&.bind
+        @worker_count.times { |index| spawn_worker(index) }
+        @control_server&.start
 
-      until @shutdown
-        break if reap_workers == :no_children
+        stats_file_thread = if @stats_file
+          Thread.new do
+            Thread.current.name = "Stats File Writer"
 
-        perform_hot_restart if @hot_restart_requested
-        poll_seed_ready if @seed_pid && !@seed_ready
-        perform_phased_restart if @phased_restart_requested && !@phased_restarting
-        check_refork_trigger if @refork_thresholds.any? && !@phased_restarting
-        timeout_hung_workers
+            write_stats_file_loop
+          end
+        end
 
-        sleep 0.1
-      end
+        ready = false
+
+        until @shutdown
+          break if reap_workers == :no_children
+
+          perform_hot_restart if @hot_restart_requested
+          poll_seed_ready if @seed_pid && !@seed_ready
+          perform_phased_restart if @phased_restart_requested && !@phased_restarting
+          check_refork_trigger if @refork_thresholds.any? && !@phased_restarting
+          timeout_hung_workers
+
+          unless ready || @shutdown
+            stats = @stats.all
+            if @workers.size == @worker_count && @workers.all? { |index, pid|
+              stat = stats[index]
+              stat[:pid] == pid && stat[:phase] == @phase && stat[:booted] && !@timed_out.include?(pid)
+            }
+              Systemd.notify("READY=1\nMAINPID=#{Process.pid}")
+              ready = true
+            end
+          end
 
-      Systemd.notify("STOPPING=1")
-      stop_workers
-      stats_file_thread&.join
-      @control_server&.shutdown
-      File.delete(@stats_file) rescue nil if @stats_file
-      File.delete(@pid_file) rescue nil if @pid_file
-      @stats.unmap
-      @binder.close
+          sleep 0.1
+        end
+      ensure
+        @shutdown = true
+        Systemd.notify("STOPPING=1")
+        stop_workers
+        stats_file_thread&.join
+        stop_plugins
+        @control_server&.shutdown
+        File.delete(@stats_file) rescue nil if @stats_file
+        File.delete(@pid_file) rescue nil if @pid_file
+        @stats.unmap
+        @binder.close
+      end
     end
 
     # Returns stats for all worker processes.
@@ -310,7 +343,7 @@ module Raptor
       @stats.all
     end
 
-    # Returns cluster statistics for the control server.
+    # Returns cluster statistics for plugins and the control server.
     # For an adaptive pool, `max_threads` is the number of threads currently
     # running so utilization reflects the capacity available at that moment.
     #
@@ -356,9 +389,23 @@ module Raptor
 
     private
 
-    # @rbs (Float timestamp) -> String
-    def timestamp(timestamp)
-      Time.at(timestamp).utc.iso8601(6)
+    # Stops plugins in reverse startup order.
+    #
+    # @rbs () -> void
+    def stop_plugins
+      until @plugins.empty?
+        plugin = @plugins.pop
+        begin
+          plugin.shutdown
+        rescue StandardError => error
+          Log.rescued_error(error)
+        end
+      end
+    end
+
+    # @rbs (Numeric time) -> String
+    def timestamp(time)
+      Time.at(time).utc.iso8601(6)
     end
 
     # Returns the inherited-FDs hash for a systemd socket-activation handoff,
@@ -449,7 +496,7 @@ module Raptor
     def pid_alive?(pid)
       Process.kill(0, pid)
       true
-    rescue Errno::ESRCH, Errno::ECHILD
+    rescue Errno::ESRCH
       false
     rescue Errno::EPERM
       true
@@ -513,14 +560,14 @@ module Raptor
       return unless candidate
 
       candidate_index, candidate_requests = candidate
-      threshold = @refork_thresholds[@refork_threshold_idx]
+      threshold = @refork_thresholds[@refork_threshold_index]
 
       if @refork_requested
         @refork_requested = false
       elsif !threshold || candidate_requests < threshold
         return
       else
-        @refork_threshold_idx += 1
+        @refork_threshold_index += 1
       end
 
       promote_worker_to_seed(candidate_index)
@@ -561,7 +608,6 @@ module Raptor
 
       retire_current_seed
       Log.info "Promoting worker #{index} to seed for phased refork"
-      ReuseportBPF.mark_unavailable(index) if @bpf_active
       Process.kill("URG", pid) rescue nil
       @workers.delete(index)
       @seed_pid = pid
@@ -766,6 +812,7 @@ module Raptor
       Systemd.notify("RELOADING=1\nMONOTONIC_USEC=#{monotonic_usec}")
       @shutdown = true
       stop_workers
+      stop_plugins
       @binder.clear_close_on_exec
       ENV[INHERITED_FDS_ENV] = JSON.generate(@binder.inheritable_fds)
       @control_server&.shutdown
@@ -800,7 +847,7 @@ module Raptor
       trap("USR2", "IGNORE")
       trap("URG") { promote_to_seed = true } if @fork_r
 
-      Raptor::CPU.pin(index) if @cpu_affinity && Raptor::CPU.count >= @worker_count
+      CPU.pin(index) if @cpu_affinity && CPU.count >= @worker_count
 
       started_at = Process.clock_gettime(Process::CLOCK_REALTIME)
       request_count = 0
@@ -813,11 +860,13 @@ module Raptor
         backlog: 0,
         busy_threads: 0,
         thread_capacity: @thread_count,
-        started_at:,
+        started_at: started_at,
         last_checkin: started_at,
         booted: false
       )
 
+      @before_worker_boot.each { |hook| hook.parameters.empty? ? hook.call : hook.call(index) }
+
       reactor = nil
 
       counting_app = ->(env) {
@@ -872,6 +921,7 @@ module Raptor
       end
 
       reactor = Reactor.new(
+        http1,
         http1_ractor_pool,
         http2_ractor_pool,
         thread_pool,
@@ -881,30 +931,25 @@ module Raptor
       )
       reactor_thread = reactor.run
 
-      worker_listeners = if @bpf_active
-        bpf_listeners = ReuseportBPF.create_worker_listeners(@binder.bind_uris, index, @binder.socket_backlog)
+      bpf_listeners = if @bpf_active
         ReuseportBPF.enable_load_reporting(index)
-        non_tcp_listeners = @binder.listeners.reject { |listener| listener.is_a?(TCPServer) }
-        bpf_listeners + non_tcp_listeners
+        ReuseportBPF.create_worker_listeners(index, @binder.socket_backlog)
       else
-        @binder.listeners
+        []
       end
 
       server = Server.new(
-        @binder,
         reactor,
         thread_pool,
         http1,
         http2,
         connection_options: @connection_options,
         drain_accept_queue: @drain_accept_queue,
-        listeners: worker_listeners,
+        listeners: bpf_listeners + @binder.listeners,
         worker_index: (index if @bpf_active)
       )
       server_thread = server.run
 
-      @before_worker_boot.each { |hook| hook.parameters.empty? ? hook.call : hook.call(index) }
-
       Log.info "Worker #{index} booted"
 
       stats_thread = Thread.new do
@@ -919,7 +964,7 @@ module Raptor
             backlog: reactor.backlog,
             busy_threads: thread_pool.active_count,
             thread_capacity: thread_pool.size,
-            started_at:,
+            started_at: started_at,
             last_checkin: Process.clock_gettime(Process::CLOCK_REALTIME),
             booted: true
           )
@@ -944,6 +989,8 @@ module Raptor
       end
 
       server_thread.join
+      # A seed never accepts again, and the workers it forks bind their own.
+      bpf_listeners.each(&:close) if promote_to_seed
       http1.shutdown
       http2.shutdown(reactor)
       reactor.drain_detached_bodies(@worker_drain_timeout)
@@ -972,11 +1019,6 @@ module Raptor
       trap("TERM") { seed_shutdown = true }
       trap("URG", "IGNORE")
 
-      child_pids = []
-      trap("CHLD") do
-        child_pids.reject! { Process.wait(_1, Process::WNOHANG) rescue true }
-      end
-
       @resp_w.write([0].pack("L"))
 
       until seed_shutdown
@@ -986,9 +1028,15 @@ module Raptor
         break unless bytes.is_a?(String) && bytes.bytesize == 8
 
         slot_index, child_phase = bytes.unpack("LL")
-        pid = fork { run_worker(slot_index, child_phase) }
-        child_pids << pid
-        @resp_w.write([pid].pack("L")) rescue nil
+        # Forks through a short-lived intermediate process so the worker
+        # reparents to the subreaper master, which supervises and reaps it
+        # like a worker it forked itself.
+        intermediate_pid = fork do
+          pid = fork { run_worker(slot_index, child_phase) }
+          @resp_w.write([pid].pack("L")) rescue nil
+          exit!
+        end
+        Process.wait(intermediate_pid)
       end
     rescue Errno::EPIPE, IOError
     end
@@ -1002,16 +1050,16 @@ module Raptor
     #
     # @rbs (AtomicThreadPool thread_pool) -> void
     def drain_thread_pool(thread_pool)
-      drain = Thread.new do
+      drain_thread = Thread.new do
         Thread.current.name = "Thread Pool Drain"
 
         thread_pool.shutdown
       end
-      return if drain.join(@worker_drain_timeout)
+      return if drain_thread.join(@worker_drain_timeout)
 
       Log.warn "Force-killing in-flight app threads after #{@worker_drain_timeout}s drain timeout"
       thread_pool.instance_variable_get(:@threads).each(&:kill)
-      drain.join
+      drain_thread.join
     end
 
     # Returns a human-readable description of how a process exited.
@@ -1036,8 +1084,6 @@ module Raptor
     #
     # @rbs () -> void
     def shutdown
-      return if @shutdown
-
       @shutdown = true
     end
 

data/lib/raptor/control_server.rb

@@ -19,9 +19,10 @@ module Raptor
     # @rbs @client: Atom
     # @rbs @thread: Thread?
 
-    # Creates a control server for `url` without binding it.
+    # Creates a new ControlServer for `url` without binding it.
     #
     # @param url [String] `unix://` URL to listen on
+    # @yield for each stats request
     # @yieldreturn [Hash] cluster statistics
     # @return [void]
     # @raise [ArgumentError] if the URL is not a Unix socket
@@ -70,6 +71,8 @@ module Raptor
     #
     # @rbs () -> void
     def shutdown
+      return unless @server
+
       client = nil
       @client.swap do |current|
         client = current
@@ -131,7 +134,7 @@ module Raptor
     # @rbs (UNIXSocket client) -> void
     def handle(client)
       request_line = client.gets
-      while line = client.gets
+      while (line = client.gets)
         break if line == "\r\n"
       end
 

data/lib/raptor/detached_body.rb

@@ -66,6 +66,7 @@ module Raptor
     #
     # @param max_buffer_size [Integer] maximum body bytes waiting to be written
     # @return [void]
+    # @raise [ArgumentError] if `max_buffer_size` is not positive
     #
     # @rbs (?max_buffer_size: Integer) -> void
     def initialize(max_buffer_size: DEFAULT_MAX_BUFFER_SIZE)
@@ -84,6 +85,7 @@ module Raptor
     # Registers a callback that runs on the request thread once the server
     # accepts the response stream.
     #
+    # @yield once the response stream opens
     # @yieldparam stream [DetachedBody] the opened response stream
     # @return [DetachedBody]
     #
@@ -95,6 +97,7 @@ module Raptor
 
     # Registers a callback for when the response stream closes.
     #
+    # @yield once the response stream closes
     # @yieldparam reason [Symbol] why the stream closed
     # @return [DetachedBody]
     #
@@ -108,6 +111,7 @@ module Raptor
     #
     # @param chunk [String] response body bytes
     # @return [Symbol] `:accepted`, `:full`, or `:closed`
+    # @raise [TypeError] if the chunk is not a String
     #
     # @rbs (String chunk) -> Symbol
     def try_write(chunk)
@@ -150,7 +154,7 @@ module Raptor
       result
     end
 
-    # Finishes the body after all accepted bytes have been written.
+    # Marks the body complete; the stream finishes once buffered bytes are written.
     #
     # @param trailers [Hash] trailing response headers
     # @return [void]
@@ -158,7 +162,7 @@ module Raptor
     # @rbs (?trailers: Hash[String, String | Array[String]]) -> void
     def close(trailers: {})
       trailers = trailers.to_h do |name, value|
-        value = value.is_a?(Array) ? value.map { _1.dup.freeze }.freeze : value.dup.freeze
+        value = value.is_a?(Array) ? value.map { |item| item.dup.freeze }.freeze : value.dup.freeze
         [name.dup.freeze, value]
       end.freeze
       wake = false

data/lib/raptor/http.rb

@@ -16,11 +16,10 @@ module Raptor
 
     CONTENT_LENGTH = "CONTENT_LENGTH"
     CONTENT_TYPE = "CONTENT_TYPE"
-    HTTP_VERSION = "HTTP_VERSION"
     REMOTE_ADDR = "REMOTE_ADDR"
     REQUEST_URI = "REQUEST_URI"
     SERVER_SOFTWARE = "SERVER_SOFTWARE"
-    SERVER_SOFTWARE_VALUE = "Raptor/#{Raptor::VERSION}".freeze
+    SERVER_SOFTWARE_VALUE = "Raptor/#{VERSION}".freeze
 
     class WriteError < StandardError
       # @rbs () -> String
@@ -56,6 +55,7 @@ module Raptor
           bytes += socket.write_nonblock(bytes.zero? ? string : string.byteslice(bytes..-1))
         rescue IO::WaitWritable
           raise WriteError unless socket.wait_writable(timeout)
+
           retry
         rescue IOError
           raise WriteError
@@ -79,9 +79,10 @@ module Raptor
       return if total.zero?
 
       begin
-        written = Raptor::VectorIO.writev_nonblock(socket, strings)
+        written = VectorIO.writev_nonblock(socket, strings)
       rescue IO::WaitWritable
         raise WriteError unless socket.wait_writable(timeout)
+
         retry
       rescue IOError
         raise WriteError
@@ -113,7 +114,7 @@ module Raptor
     # @rbs (Hash[String, untyped] env_template, Integer? max_body_size) -> ^(Hash[Symbol, untyped]) -> Hash[Symbol, untyped]
     def self.parser_worker(env_template, max_body_size)
       proc do |data|
-        Raptor::Http1.parse(data, env_template, max_body_size)
+        Http1.parse(data, env_template, max_body_size)
       end
     end
 
@@ -121,12 +122,12 @@ module Raptor
     # registration order, rescuing any that raise.
     #
     # @param env [Hash, nil] the Rack environment
-    # @param status [Integer, nil] the response status code
+    # @param status [Object, nil] the status returned by the app, if any
     # @param headers [Hash, nil] the response headers
     # @param error [Exception, nil] any error raised during processing, or nil on success
     # @return [void]
     #
-    # @rbs (Hash[String, untyped]? env, Integer? status, Hash[String, String | Array[String]]? headers, Exception? error) -> void
+    # @rbs (Hash[String, untyped]? env, untyped status, Hash[String, String | Array[String]]? headers, Exception? error) -> void
     def self.call_response_finished(env, status, headers, error)
       return unless env && env[Rack::RACK_RESPONSE_FINISHED].is_a?(Array)