raptor 0.3.0 → 0.5.0

data/lib/raptor/cluster.rb

@@ -7,6 +7,7 @@ require "atomic-ruby/atomic_thread_pool"
 require "rack/builder"
 require "ractor-pool"
 
+require_relative "log"
 require_relative "binder"
 require_relative "server"
 require_relative "reactor"
@@ -18,14 +19,15 @@ module Raptor
   # Multi-process web server cluster with advanced concurrency architecture.
   #
   # Cluster manages multiple worker processes, each running a complete server
-  # stack including a reactor thread, server thread, ractor pool for HTTP
-  # parsing, and thread pool for application processing. It handles process
-  # forking, signal management, graceful shutdown, and automatic worker
-  # restart when a worker process unexpectedly exits.
+  # stack including a ractor pool for HTTP parsing, a thread pool for
+  # application processing, plus dedicated reactor and server threads. It
+  # handles process forking, signal management, graceful shutdown, and
+  # automatic worker restart when a worker process unexpectedly exits.
   #
   # The architecture provides horizontal scaling through processes while
-  # maintaining efficient I/O and CPU utilization within each process through
-  # the combination of NIO reactors, ractor-based parsing, and thread pools.
+  # maintaining efficient I/O and CPU utilization within each process
+  # through the combination of ractor-based parsing and thread pools on
+  # top of NIO reactors.
   #
   # Flow per worker process:
   # 1. Server continuously accepts connections but skips acceptance when backlog is high
@@ -36,7 +38,7 @@ module Raptor
   #
   # @example Basic usage
   #   options = {
-  #     threads: 8, ractors: 2, workers: 4,
+  #     workers: 4, ractors: 2, threads: 8,
   #     binds: ["tcp://0.0.0.0:3000"],
   #     rackup: "config.ru",
   #     client: { first_data_timeout: 30, chunk_data_timeout: 10 }
@@ -54,50 +56,61 @@ module Raptor
       new(options).run
     end
 
-    # @rbs @thread_count: Integer
-    # @rbs @ractor_count: Integer
     # @rbs @worker_count: Integer
+    # @rbs @ractor_count: Integer
+    # @rbs @thread_count: Integer
     # @rbs @client_options: Hash[Symbol, Integer]
-    # @rbs @on_error: ^(Hash[String, untyped]?, Exception) -> void | nil
+    # @rbs @worker_timeout: Integer
+    # @rbs @worker_boot_timeout: Integer
+    # @rbs @worker_shutdown_timeout: Integer
     # @rbs @stats_file: String?
-    # @rbs @pidfile: String?
+    # @rbs @pid_file: String?
+    # @rbs @on_error: ^(Hash[String, untyped]?, Exception) -> void | nil
     # @rbs @binder: Binder
     # @rbs @server_port: Integer
     # @rbs @app: untyped
     # @rbs @shutdown: bool
     # @rbs @workers: Hash[Integer, Integer]
+    # @rbs @timed_out: Set[Integer]
     # @rbs @stats: Stats
+    # @rbs @phase: Integer
     # @rbs @phased_restart_requested: bool
     # @rbs @phased_restarting: bool
 
     # Creates a new Cluster with the specified configuration.
     #
-    # Initializes the cluster with thread, ractor, and worker counts,
+    # Initializes the cluster with worker, ractor, and thread counts,
     # sets up network binding, loads the Rack application, and prepares
     # for multi-process operation.
     #
     # @param options [Hash] cluster configuration options
-    # @option options [Integer] :threads number of threads per worker process
-    # @option options [Integer] :ractors number of ractors per worker process
-    # @option options [Integer] :workers number of worker processes
     # @option options [Array<String>] :binds array of bind URIs
+    # @option options [Integer] :workers number of worker processes
+    # @option options [Integer] :ractors number of ractors per worker process
+    # @option options [Integer] :threads number of threads per worker process
     # @option options [#call] :app pre-built Rack application
     # @option options [String] :rackup path to Rack configuration file
     # @option options [Hash] :client client configuration
-    # @option options [#call] :on_error callback invoked with (env, exception) when the Rack app raises
+    # @option options [Integer] :worker_timeout seconds to wait for a booted worker to check in before killing it
+    # @option options [Integer] :worker_boot_timeout seconds to wait for a worker to finish booting before killing it
+    # @option options [Integer] :worker_shutdown_timeout seconds to wait for graceful worker exit before force-killing
     # @option options [String, nil] :stats_file path to write per-worker stats JSON, or nil to disable
-    # @option options [String, nil] :pidfile path to write the master PID to, or nil to disable
+    # @option options [String, nil] :pid_file path to write the master PID to, or nil to disable
+    # @option options [#call] :on_error callback invoked with (env, exception) when the Rack app raises
     # @return [void]
     #
     # @rbs (Hash[Symbol, untyped] options) -> void
     def initialize(options)
-      @thread_count = options[:threads]
-      @ractor_count = options[:ractors]
       @worker_count = options[:workers]
+      @ractor_count = options[:ractors]
+      @thread_count = options[:threads]
       @client_options = options[:client]
-      @on_error = options[:on_error]
+      @worker_timeout = options[:worker_timeout]
+      @worker_boot_timeout = options[:worker_boot_timeout]
+      @worker_shutdown_timeout = options[:worker_shutdown_timeout]
       @stats_file = options[:stats_file]
-      @pidfile = options[:pidfile]
+      @pid_file = options[:pid_file]
+      @on_error = options[:on_error]
 
       @binder = Binder.new(options[:binds])
       @server_port = @binder.server_port
@@ -106,7 +119,9 @@ module Raptor
 
       @shutdown = false
       @workers = {}
+      @timed_out = Set.new
       @stats = Stats.new(@worker_count)
+      @phase = 0
       @phased_restart_requested = false
       @phased_restarting = false
     end
@@ -114,15 +129,15 @@ module Raptor
     # Starts the multi-process cluster and manages worker processes.
     #
     # Forks the configured number of worker processes and monitors them,
-    # automatically restarting any that exit unexpectedly. Handles graceful
-    # shutdown via INT or TERM signals, stats logging via USR1, and phased
-    # restart via USR2.
+    # restarting any that exit unexpectedly or stop checking in. Handles
+    # graceful shutdown via INT or TERM signals, stats logging via USR1,
+    # and phased restart via USR2.
     #
     # Each worker process includes:
     # - 1 server thread (continuously accepts connections with backpressure control)
     # - 1 reactor thread (I/O multiplexing, timeout handling, backlog monitoring)
-    # - N ractor workers (parallel HTTP parsing)
-    # - 1 ractor collector thread (coordinates parsing results)
+    # - N pipeline ractors (parallel HTTP parsing)
+    # - 1 pipeline collector thread (coordinates parsing results)
     # - M worker threads (Rack application processing and response writing)
     # - 1 stats thread (writes per-worker metrics to shared memory every second)
     #
@@ -135,7 +150,7 @@ module Raptor
       trap("USR1") { log_stats }
       trap("USR2") { @phased_restart_requested = true }
 
-      File.open(@pidfile, File::CREAT | File::EXCL | File::WRONLY) { |file| file.write(Process.pid.to_s) } if @pidfile
+      File.open(@pid_file, File::CREAT | File::EXCL | File::WRONLY) { |file| file.write(Process.pid.to_s) } if @pid_file
 
       @worker_count.times { |index| spawn_worker(index) }
 
@@ -151,15 +166,15 @@ module Raptor
         break if reap_workers == :no_children
 
         perform_phased_restart if @phased_restart_requested && !@phased_restarting
+        timeout_hung_workers
 
         sleep 0.1
       end
 
-      @workers.values.each { |pid| Process.kill("TERM", pid) rescue nil }
-      @workers.values.each { |pid| Process.wait(pid) rescue nil }
+      stop_workers
       stats_file_thread&.join
       File.delete(@stats_file) rescue nil if @stats_file
-      File.delete(@pidfile) rescue nil if @pidfile
+      File.delete(@pid_file) rescue nil if @pid_file
       @stats.unmap
     end
 
@@ -176,13 +191,14 @@ module Raptor
     private
 
     # Forks a new worker process and registers it at the given index.
+    # The worker inherits the cluster's current phase.
     #
     # @param index [Integer] slot index for this worker in the stats region
     # @return [void]
     #
     # @rbs (Integer index) -> void
     def spawn_worker(index)
-      pid = fork { run_worker(index) }
+      pid = fork { run_worker(index, @phase) }
       @workers[index] = pid
     end
 
@@ -199,9 +215,10 @@ module Raptor
 
         index = @workers.key(pid)
         @workers.delete(index)
+        @timed_out.delete(pid)
 
         unless @shutdown
-          warn "[#{Process.pid}] Restarting worker #{index} (#{pid}), #{exit_description(status)}"
+          Log.warn "Restarting worker #{index} (#{pid}), #{exit_description(status)}"
           spawn_worker(index)
         end
       end
@@ -209,6 +226,57 @@ module Raptor
       :no_children
     end
 
+    # Stops every worker, escalating from TERM to KILL if any fail to
+    # exit within `worker_shutdown_timeout`.
+    #
+    # @return [void]
+    #
+    # @rbs () -> void
+    def stop_workers
+      @workers.values.each { |pid| Process.kill("TERM", pid) rescue nil }
+
+      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @worker_shutdown_timeout
+      until @workers.empty? || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
+        reap_workers
+        sleep 0.05
+      end
+      return if @workers.empty?
+
+      Log.warn "Force-killing #{@workers.size} worker(s) after #{@worker_shutdown_timeout}s"
+      @workers.values.each { |pid| Process.kill("KILL", pid) rescue nil }
+      @workers.values.each { |pid| Process.wait(pid) rescue nil }
+    end
+
+    # Kills workers that have stopped checking in. A booted worker that
+    # fails to update its stats slot within `worker_timeout` seconds is
+    # assumed to be hung (deadlocked app, runaway loop, blocked syscall);
+    # a worker still in startup is held to `worker_boot_timeout`. Killed
+    # workers are then restarted by `reap_workers`.
+    #
+    # @return [void]
+    #
+    # @rbs () -> void
+    def timeout_hung_workers
+      now = Process.clock_gettime(Process::CLOCK_REALTIME)
+      stats = @stats.all
+
+      @workers.each do |index, pid|
+        next if @timed_out.include?(pid)
+
+        stat = stats[index]
+        next unless stat[:pid] == pid
+
+        timeout = stat[:booted] ? @worker_timeout : @worker_boot_timeout
+        elapsed = now - stat[:last_checkin]
+        next if elapsed <= timeout
+
+        action = stat[:booted] ? "check in" : "boot"
+        Log.warn "Killing worker #{index} (#{pid}), failed to #{action} within #{timeout}s"
+        Process.kill("KILL", pid) rescue nil
+        @timed_out << pid
+      end
+    end
+
     # Replaces each worker process one at a time, waiting for the new
     # worker to boot before moving on to the next. Triggered by SIGUSR2.
     #
@@ -218,7 +286,8 @@ module Raptor
     def perform_phased_restart
       @phased_restart_requested = false
       @phased_restarting = true
-      puts "[#{Process.pid}] Phased restart starting"
+      @phase += 1
+      Log.info "Phased restart starting"
 
       begin
         @workers.keys.sort.each do |index|
@@ -240,7 +309,7 @@ module Raptor
           end
         end
 
-        puts "[#{Process.pid}] Phased restart complete"
+        Log.info "Phased restart complete"
       ensure
         @phased_restarting = false
       end
@@ -253,10 +322,11 @@ module Raptor
     # critical component fails.
     #
     # @param index [Integer] slot index for this worker in the stats region
+    # @param phase [Integer] the cluster phase this worker was forked at
     # @return [void]
     #
-    # @rbs (Integer index) -> void
-    def run_worker(index)
+    # @rbs (Integer index, Integer phase) -> void
+    def run_worker(index, phase)
       shutdown_requested = false
       trap("INT") { shutdown_requested = true }
       trap("TERM") { shutdown_requested = true }
@@ -267,8 +337,11 @@ module Raptor
       @stats.write(
         index,
         pid: Process.pid,
+        phase: phase,
         requests: 0,
         backlog: 0,
+        busy_threads: 0,
+        thread_capacity: @thread_count,
         started_at:,
         last_checkin: started_at,
         booted: false
@@ -288,20 +361,24 @@ module Raptor
         size: @ractor_count,
         worker: request.http_parser_worker
       ) do |parsed_result|
-        if parsed_result[:protocol] == :http2
-          http2.handle_parsed_request(parsed_result, reactor, thread_pool)
-        else
-          request.handle_parsed_request(parsed_result, reactor, thread_pool)
+        begin
+          if parsed_result[:protocol] == :http2
+            http2.handle_parsed_request(parsed_result, reactor, thread_pool)
+          else
+            request.handle_parsed_request(parsed_result, reactor, thread_pool)
+          end
+        rescue => error
+          Log.rescued_error(error)
         end
       end
 
-      reactor = Reactor.new(thread_pool, ractor_pool, client_options: @client_options)
+      reactor = Reactor.new(ractor_pool, thread_pool, client_options: @client_options)
       reactor_thread = reactor.run
 
-      server = Server.new(@binder, reactor, thread_pool, request)
+      server = Server.new(@binder, reactor, thread_pool, request, client_options: @client_options)
       server_thread = server.run
 
-      puts "[#{Process.pid}] Worker #{index} booted"
+      Log.info "Worker #{index} booted"
 
       stats_thread = Thread.new do
         Thread.current.name = "Raptor Stats"
@@ -310,8 +387,11 @@ module Raptor
           @stats.write(
             index,
             pid: Process.pid,
+            phase: phase,
             requests: request_count,
             backlog: reactor.backlog,
+            busy_threads: thread_pool.active_count,
+            thread_capacity: @thread_count,
             started_at:,
             last_checkin: Process.clock_gettime(Process::CLOCK_REALTIME),
             booted: true
@@ -333,6 +413,7 @@ module Raptor
       reactor.shutdown
       reactor_thread.join
       ractor_pool.shutdown
+      request.shutdown
       thread_pool.shutdown
       stats_thread.join
     end
@@ -364,28 +445,25 @@ module Raptor
       @shutdown = true
     end
 
-    # Logs cluster initialization details including architecture and bind addresses.
-    #
-    # Outputs a hierarchical view of the cluster configuration showing
-    # the master process, worker processes, and per-process thread/ractor
-    # allocation along with listening addresses.
+    # Prints the cluster's startup banner showing process structure
+    # and bind addresses.
     #
     # @return [void]
     #
     # @rbs () -> void
     def log_initialization
-      puts "Raptor Cluster initializing:"
-      puts "├─ Version: #{VERSION}"
-      puts "├─ Ruby Version: #{RUBY_DESCRIPTION}"
-      puts "├─ Master PID: #{Process.pid}"
-      puts "│  └─ #{@worker_count} worker process#{"es" if @worker_count > 1}"
-      puts "│     ├─ 1 server thread"
-      puts "│     ├─ 1 reactor thread"
-      puts "│     ├─ #{@ractor_count} pipeline ractor#{"s" if @ractor_count > 1}"
-      puts "│     ├─ 1 pipeline collector thread"
-      puts "│     ├─ #{@thread_count} worker thread#{"s" if @thread_count > 1}"
-      puts "│     └─ 1 stats thread"
-      puts "└─ Listening on #{@binder.addresses.join(", ")}"
+      Log.info "Cluster initializing:"
+      Log.info "├─ Version: #{VERSION}"
+      Log.info "├─ Ruby Version: #{RUBY_DESCRIPTION}"
+      Log.info "├─ Master PID: #{Process.pid}"
+      Log.info "│  └─ #{@worker_count} worker process#{"es" if @worker_count > 1}"
+      Log.info "│     ├─ 1 server thread"
+      Log.info "│     ├─ 1 reactor thread"
+      Log.info "│     ├─ #{@ractor_count} pipeline ractor#{"s" if @ractor_count > 1}"
+      Log.info "│     ├─ 1 pipeline collector thread"
+      Log.info "│     ├─ #{@thread_count} worker thread#{"s" if @thread_count > 1}"
+      Log.info "│     └─ 1 stats thread"
+      Log.info "└─ Listening on #{@binder.addresses.join(", ")}"
     end
 
     # Logs current stats for all workers to stdout.
@@ -396,11 +474,11 @@ module Raptor
     #
     # @rbs () -> void
     def log_stats
-      @stats.all.each_with_index do |stat, index|
+      @stats.all.each do |stat|
         status = stat[:booted] ? "booted" : "starting"
-        puts "Worker #{index}: pid=#{stat[:pid]}, requests=#{stat[:requests]}, " \
-             "backlog=#{stat[:backlog]}, #{status}, " \
-             "last_checkin=#{Time.at(stat[:last_checkin]).strftime("%H:%M:%S")}"
+        Log.info "Worker #{stat[:index]} (phase #{stat[:phase]}): pid=#{stat[:pid]}, requests=#{stat[:requests]}, " \
+                 "busy=#{stat[:busy_threads]}/#{stat[:thread_capacity]}, backlog=#{stat[:backlog]}, " \
+                 "#{status}, last_checkin=#{Time.at(stat[:last_checkin]).strftime("%H:%M:%S")}"
       end
     end