puma 3.11.1 → 6.6.0

data/lib/puma/cluster.rb

@@ -1,36 +1,51 @@
-require 'puma/runner'
-require 'puma/util'
-require 'puma/plugin'
+# frozen_string_literal: true
 
-require 'time'
+require_relative 'runner'
+require_relative 'util'
+require_relative 'plugin'
+require_relative 'cluster/worker_handle'
+require_relative 'cluster/worker'
 
 module Puma
+  # This class is instantiated by the `Puma::Launcher` and used
+  # to boot and serve a Ruby application when puma "workers" are needed
+  # i.e. when using multi-processes. For example `$ puma -w 5`
+  #
+  # An instance of this class will spawn the number of processes passed in
+  # via the `spawn_workers` method call. Each worker will have it's own
+  # instance of a `Puma::Server`.
   class Cluster < Runner
-    WORKER_CHECK_INTERVAL = 5
-
-    def initialize(cli, events)
-      super cli, events
+    def initialize(launcher)
+      super(launcher)
 
       @phase = 0
       @workers = []
-      @next_check = nil
+      @next_check = Time.now
 
-      @phased_state = :idle
       @phased_restart = false
     end
 
+    # Returns the list of cluster worker handles.
+    # @return [Array<Puma::Cluster::WorkerHandle>]
+    attr_reader :workers
+
     def stop_workers
       log "- Gracefully shutting down workers..."
       @workers.each { |x| x.term }
 
       begin
-        @workers.each { |w| Process.waitpid(w.pid) }
+        loop do
+          wait_workers
+          break if @workers.reject {|w| w.pid.nil?}.empty?
+          sleep 0.2
+        end
       rescue Interrupt
         log "! Cancelled waiting for workers"
       end
     end
 
     def start_phased_restart
+      @events.fire_on_restart!
       @phase += 1
       log "- Starting phased worker restart, phase: #{@phase}"
 
@@ -47,155 +62,117 @@ module Puma
       @workers.each { |x| x.hup }
     end
 
-    class Worker
-      def initialize(idx, pid, phase, options)
-        @index = idx
-        @pid = pid
-        @phase = phase
-        @stage = :started
-        @signal = "TERM"
-        @options = options
-        @first_term_sent = nil
-        @last_checkin = Time.now
-        @last_status = '{}'
-        @dead = false
-      end
-
-      attr_reader :index, :pid, :phase, :signal, :last_checkin, :last_status
-
-      def booted?
-        @stage == :booted
-      end
-
-      def boot!
-        @last_checkin = Time.now
-        @stage = :booted
-      end
-
-      def dead?
-        @dead
-      end
-
-      def dead!
-        @dead = true
-      end
-
-      def ping!(status)
-        @last_checkin = Time.now
-        @last_status = status
-      end
-
-      def ping_timeout?(which)
-        Time.now - @last_checkin > which
-      end
-
-      def term
-        begin
-          if @first_term_sent && (Time.now - @first_term_sent) > @options[:worker_shutdown_timeout]
-            @signal = "KILL"
-          else
-            @first_term_sent ||= Time.now
-          end
-
-          Process.kill @signal, @pid
-        rescue Errno::ESRCH
-        end
-      end
-
-      def kill
-        Process.kill "KILL", @pid
-      rescue Errno::ESRCH
-      end
-
-      def hup
-        Process.kill "HUP", @pid
-      rescue Errno::ESRCH
-      end
-    end
-
     def spawn_workers
       diff = @options[:workers] - @workers.size
       return if diff < 1
 
       master = Process.pid
+      if @options[:fork_worker]
+        @fork_writer << "-1\n"
+      end
 
       diff.times do
         idx = next_worker_index
-        @launcher.config.run_hooks :before_worker_fork, idx
 
-        pid = fork { worker(idx, master) }
-        if !pid
-          log "! Complete inability to spawn new workers detected"
-          log "! Seppuku is the only choice."
-          exit! 1
+        if @options[:fork_worker] && idx != 0
+          @fork_writer << "#{idx}\n"
+          pid = nil
+        else
+          pid = spawn_worker(idx, master)
         end
 
         debug "Spawned worker: #{pid}"
-        @workers << Worker.new(idx, pid, @phase, @options)
+        @workers << WorkerHandle.new(idx, pid, @phase, @options)
+      end
 
-        @launcher.config.run_hooks :after_worker_fork, idx
+      if @options[:fork_worker] && all_workers_in_phase?
+        @fork_writer << "0\n"
+
+        if worker_at(0).phase > 0
+          @fork_writer << "-2\n"
+        end
       end
+    end
+
+    # @version 5.0.0
+    def spawn_worker(idx, master)
+      @config.run_hooks(:before_worker_fork, idx, @log_writer)
 
-      if diff > 0
-        @phased_state = :idle
+      pid = fork { worker(idx, master) }
+      if !pid
+        log "! Complete inability to spawn new workers detected"
+        log "! Seppuku is the only choice."
+        exit! 1
       end
+
+      @config.run_hooks(:after_worker_fork, idx, @log_writer)
+      pid
     end
 
     def cull_workers
       diff = @workers.size - @options[:workers]
       return if diff < 1
+      debug "Culling #{diff} workers"
 
-      debug "Culling #{diff.inspect} workers"
-
-      workers_to_cull = @workers[-diff,diff]
-      debug "Workers to cull: #{workers_to_cull.inspect}"
+      workers = workers_to_cull(diff)
+      debug "Workers to cull: #{workers.inspect}"
 
-      workers_to_cull.each do |worker|
-        log "- Worker #{worker.index} (pid: #{worker.pid}) terminating"
+      workers.each do |worker|
+        log "- Worker #{worker.index} (PID: #{worker.pid}) terminating"
         worker.term
       end
     end
 
-    def next_worker_index
-      all_positions =  0...@options[:workers]
-      occupied_positions = @workers.map { |w| w.index }
-      available_positions = all_positions.to_a - occupied_positions
-      available_positions.first
-    end
+    def workers_to_cull(diff)
+      workers = @workers.sort_by(&:started_at)
 
-    def all_workers_booted?
-      @workers.count { |w| !w.booted? } == 0
+      # In fork_worker mode, worker 0 acts as our master process.
+      # We should avoid culling it to preserve copy-on-write memory gains.
+      workers.reject! { |w| w.index == 0 } if @options[:fork_worker]
+
+      workers[cull_start_index(diff), diff]
     end
 
-    def check_workers(force=false)
-      return if !force && @next_check && @next_check >= Time.now
+    def cull_start_index(diff)
+      case @options[:worker_culling_strategy]
+      when :oldest
+        0
+      else # :youngest
+        -diff
+      end
+    end
 
-      @next_check = Time.now + WORKER_CHECK_INTERVAL
+    # @!attribute [r] next_worker_index
+    def next_worker_index
+      occupied_positions = @workers.map(&:index)
+      idx = 0
+      idx += 1 until !occupied_positions.include?(idx)
+      idx
+    end
 
-      any = false
+    def worker_at(idx)
+      @workers.find { |w| w.index == idx }
+    end
 
-      @workers.each do |w|
-        next if !w.booted? && !w.ping_timeout?(@options[:worker_boot_timeout])
-        if w.ping_timeout?(@options[:worker_timeout])
-          log "! Terminating timed out worker: #{w.pid}"
-          w.kill
-          any = true
-        end
-      end
+    def all_workers_booted?
+      @workers.count { |w| !w.booted? } == 0
+    end
 
-      # If we killed any timed out workers, try to catch them
-      # during this loop by giving the kernel time to kill them.
-      sleep 1 if any
+    def all_workers_in_phase?
+      @workers.all? { |w| w.phase == @phase }
+    end
 
-      while @workers.any?
-        pid = Process.waitpid(-1, Process::WNOHANG)
-        break unless pid
+    def all_workers_idle_timed_out?
+      (@workers.map(&:pid) - idle_timed_out_worker_pids).empty?
+    end
 
-        @workers.delete_if { |w| w.pid == pid }
-      end
+    def check_workers
+      return if @next_check >= Time.now
 
-      @workers.delete_if(&:dead?)
+      @next_check = Time.now + @options[:worker_check_interval]
 
+      timeout_workers
+      wait_workers
       cull_workers
       spawn_workers
 
@@ -207,97 +184,40 @@ module Puma
         w = @workers.find { |x| x.phase != @phase }
 
         if w
-          if @phased_state == :idle
-            @phased_state = :waiting
-            log "- Stopping #{w.pid} for phased upgrade..."
+          log "- Stopping #{w.pid} for phased upgrade..."
+          unless w.term?
+            w.term
+            log "- #{w.signal} sent to #{w.pid}..."
           end
-
-          w.term
-          log "- #{w.signal} sent to #{w.pid}..."
         end
       end
-    end
 
-    def wakeup!
-      return unless @wakeup
+      t = @workers.reject(&:term?)
+      t.map!(&:ping_timeout)
 
-      begin
-        @wakeup.write "!" unless @wakeup.closed?
-      rescue SystemCallError, IOError
-        Thread.current.purge_interrupt_queue if Thread.current.respond_to? :purge_interrupt_queue
-      end
+      @next_check = [t.min, @next_check].compact.min
     end
 
     def worker(index, master)
-      title  = "puma: cluster worker #{index}: #{master}"
-      title += " [#{@options[:tag]}]" if @options[:tag] && !@options[:tag].empty?
-      $0 = title
-
-      Signal.trap "SIGINT", "IGNORE"
-
       @workers = []
+
       @master_read.close
       @suicide_pipe.close
+      @fork_writer.close
 
-      Thread.new do
-        IO.select [@check_pipe]
-        log "! Detected parent died, dying"
-        exit! 1
+      pipes = { check_pipe: @check_pipe, worker_write: @worker_write }
+      if @options[:fork_worker]
+        pipes[:fork_pipe] = @fork_pipe
+        pipes[:wakeup] = @wakeup
       end
 
-      # If we're not running under a Bundler context, then
-      # report the info about the context we will be using
-      if !ENV['BUNDLE_GEMFILE']
-        if File.exist?("Gemfile")
-          log "+ Gemfile in context: #{File.expand_path("Gemfile")}"
-        elsif File.exist?("gems.rb")
-          log "+ Gemfile in context: #{File.expand_path("gems.rb")}"
-        end
-      end
-
-      # Invoke any worker boot hooks so they can get
-      # things in shape before booting the app.
-      @launcher.config.run_hooks :before_worker_boot, index
-
-      server = start_server
-
-      Signal.trap "SIGTERM" do
-        server.stop
-      end
-
-      begin
-        @worker_write << "b#{Process.pid}\n"
-      rescue SystemCallError, IOError
-        Thread.current.purge_interrupt_queue if Thread.current.respond_to? :purge_interrupt_queue
-        STDERR.puts "Master seems to have exited, exiting."
-        return
-      end
-
-      Thread.new(@worker_write) do |io|
-        base_payload = "p#{Process.pid}"
-
-        while true
-          sleep WORKER_CHECK_INTERVAL
-          begin
-            b = server.backlog || 0
-            r = server.running || 0
-            payload = %Q!#{base_payload}{ "backlog":#{b}, "running":#{r} }\n!
-            io << payload
-          rescue IOError
-            Thread.current.purge_interrupt_queue if Thread.current.respond_to? :purge_interrupt_queue
-            break
-          end
-        end
-      end
-
-      server.run.join
-
-      # Invoke any worker shutdown hooks so they can prevent the worker
-      # exiting until any background operations are completed
-      @launcher.config.run_hooks :before_worker_shutdown, index
-    ensure
-      @worker_write << "t#{Process.pid}\n" rescue nil
-      @worker_write.close
+      server = start_server if preload?
+      new_worker = Worker.new index: index,
+                              master: master,
+                              launcher: @launcher,
+                              pipes: pipes,
+                              server: server
+      new_worker.run
     end
 
     def restart
@@ -305,8 +225,8 @@ module Puma
       stop
     end
 
-    def phased_restart
-      return false if @options[:preload_app]
+    def phased_restart(refork = false)
+      return false if @options[:preload_app] && !refork
 
       @phased_restart = true
       wakeup!
@@ -322,7 +242,7 @@ module Puma
     def stop_blocked
       @status = :stop if @status == :run
       wakeup!
-      @control.stop(true) if @control
+      @control&.stop true
       Process.waitall
     end
 
@@ -337,20 +257,63 @@ module Puma
       Dir.chdir dir
     end
 
+    # Inside of a child process, this will return all zeroes, as @workers is only populated in
+    # the master process.
+    # @!attribute [r] stats
     def stats
       old_worker_count = @workers.count { |w| w.phase != @phase }
-      booted_worker_count = @workers.count { |w| w.booted? }
-      worker_status = '[' + @workers.map { |w| %Q!{ "pid": #{w.pid}, "index": #{w.index}, "phase": #{w.phase}, "booted": #{w.booted?}, "last_checkin": "#{w.last_checkin.utc.iso8601}", "last_status": #{w.last_status} }!}.join(",") + ']'
-      %Q!{ "workers": #{@workers.size}, "phase": #{@phase}, "booted_workers": #{booted_worker_count}, "old_workers": #{old_worker_count}, "worker_status": #{worker_status} }!
+      worker_status = @workers.map do |w|
+        {
+          started_at: utc_iso8601(w.started_at),
+          pid: w.pid,
+          index: w.index,
+          phase: w.phase,
+          booted: w.booted?,
+          last_checkin: utc_iso8601(w.last_checkin),
+          last_status: w.last_status,
+        }
+      end
+
+      {
+        started_at: utc_iso8601(@started_at),
+        workers: @workers.size,
+        phase: @phase,
+        booted_workers: worker_status.count { |w| w[:booted] },
+        old_workers: old_worker_count,
+        worker_status: worker_status,
+      }.merge(super)
     end
 
     def preload?
       @options[:preload_app]
     end
 
+    # @version 5.0.0
+    def fork_worker!
+      if (worker = worker_at 0)
+        worker.phase += 1
+      end
+      phased_restart(true)
+    end
+
     # We do this in a separate method to keep the lambda scope
     # of the signals handlers as small as possible.
     def setup_signals
+      if @options[:fork_worker]
+        Signal.trap "SIGURG" do
+          fork_worker!
+        end
+
+        # Auto-fork after the specified number of requests.
+        if (fork_requests = @options[:fork_worker].to_i) > 0
+          @events.register(:ping!) do |w|
+            fork_worker! if w.index == 0 &&
+              w.phase == 0 &&
+              w.last_status[:requests_count] >= fork_requests
+          end
+        end
+      end
+
       Signal.trap "SIGCHLD" do
         wakeup!
       end
@@ -375,10 +338,13 @@ module Puma
           log "Early termination of worker"
           exit! 0
         else
+          @launcher.close_binder_listeners
+
           stop_workers
           stop
-
-          raise SignalException, "SIGTERM"
+          @events.fire_on_stopped!
+          raise(SignalException, "SIGTERM") if @options[:raise_exception_on_sigterm]
+          exit 0 # Clean exit, workers were stopped
         end
       end
     end
@@ -388,15 +354,25 @@ module Puma
 
       output_header "cluster"
 
-      log "* Process workers: #{@options[:workers]}"
-
-      before = Thread.list
+      # This is aligned with the output from Runner, see Runner#output_header
+      log "*      Workers: #{@options[:workers]}"
 
       if preload?
+        # Threads explicitly marked as fork safe will be ignored. Used in Rails,
+        # but may be used by anyone. Note that we need to explicit
+        # Process::Waiter check here because there's a bug in Ruby 2.6 and below
+        # where calling thread_variable_get on a Process::Waiter will segfault.
+        # We can drop that clause once those versions of Ruby are no longer
+        # supported.
+        fork_safe = ->(t) { !t.is_a?(Process::Waiter) && t.thread_variable_get(:fork_safe) }
+
+        before = Thread.list.reject(&fork_safe)
+
+        log "*     Restarts: (\u2714) hot (\u2716) phased"
         log "* Preloading application"
         load_and_bind
 
-        after = Thread.list
+        after = Thread.list.reject(&fork_safe)
 
         if after.size > before.size
           threads = (after - before)
@@ -410,14 +386,14 @@ module Puma
           end
         end
       else
-        log "* Phased restart available"
+        log "*     Restarts: (\u2714) hot (\u2714) phased"
 
-        unless @launcher.config.app_configured?
+        unless @config.app_configured?
           error "No application configured, nothing to run"
           exit 1
         end
 
-        @launcher.binder.parse @options[:binds], self
+        @launcher.binder.parse @options[:binds]
       end
 
       read, @wakeup = Puma::Util.pipe
@@ -431,12 +407,13 @@ module Puma
       #
       @check_pipe, @suicide_pipe = Puma::Util.pipe
 
-      if daemon?
-        log "* Daemonizing..."
-        Process.daemon(true)
-      else
-        log "Use Ctrl-C to stop"
-      end
+      # Separate pipe used by worker 0 to receive commands to
+      # fork new worker processes.
+      @fork_pipe, @fork_writer = Puma::Util.pipe
+
+      log "Use Ctrl-C to stop"
+
+      single_worker_warning
 
       redirect_io
 
@@ -448,7 +425,9 @@ module Puma
 
       @master_read, @worker_write = read, @wakeup
 
-      @launcher.config.run_hooks :before_fork, nil
+      @options[:worker_write] = @worker_write
+
+      @config.run_hooks(:before_fork, nil, @log_writer)
 
       spawn_workers
 
@@ -456,49 +435,89 @@ module Puma
         stop
       end
 
-      @launcher.events.fire_on_booted!
-
       begin
-        force_check = false
+        booted = false
+        in_phased_restart = false
+        workers_not_booted = @options[:workers]
 
         while @status == :run
           begin
+            if @options[:idle_timeout] && all_workers_idle_timed_out?
+              log "- All workers reached idle timeout"
+              break
+            end
+
             if @phased_restart
               start_phased_restart
               @phased_restart = false
+              in_phased_restart = true
+              workers_not_booted = @options[:workers]
             end
 
-            check_workers force_check
-
-            force_check = false
-
-            res = IO.select([read], nil, nil, WORKER_CHECK_INTERVAL)
+            check_workers
 
-            if res
+            if read.wait_readable([0, @next_check - Time.now].max)
               req = read.read_nonblock(1)
+              next unless req
 
-              next if !req || req == "!"
+              if req == PIPE_WAKEUP
+                @next_check = Time.now
+                next
+              end
 
               result = read.gets
               pid = result.to_i
 
+              if req == PIPE_BOOT || req == PIPE_FORK
+                pid, idx = result.split(':').map(&:to_i)
+                w = worker_at idx
+                w.pid = pid if w.pid.nil?
+              end
+
               if w = @workers.find { |x| x.pid == pid }
                 case req
-                when "b"
+                when PIPE_BOOT
                   w.boot!
-                  log "- Worker #{w.index} (pid: #{pid}) booted, phase: #{w.phase}"
-                  force_check = true
-                when "t"
-                  w.dead!
-                  force_check = true
-                when "p"
-                  w.ping!(result.sub(/^\d+/,'').chomp)
+                  log "- Worker #{w.index} (PID: #{pid}) booted in #{w.uptime.round(2)}s, phase: #{w.phase}"
+                  @next_check = Time.now
+                  workers_not_booted -= 1
+                when PIPE_EXTERNAL_TERM
+                  # external term, see worker method, Signal.trap "SIGTERM"
+                  w.term!
+                when PIPE_TERM
+                  w.term unless w.term?
+                when PIPE_PING
+                  status = result.sub(/^\d+/,'').chomp
+                  w.ping!(status)
+                  @events.fire(:ping!, w)
+
+                  if in_phased_restart && @options[:fork_worker] && workers_not_booted.positive? && w0 = worker_at(0)
+                    w0.ping!(status)
+                    @events.fire(:ping!, w0)
+                  end
+
+                  if !booted && @workers.none? {|worker| worker.last_status.empty?}
+                    @events.fire_on_booted!
+                    debug_loaded_extensions("Loaded Extensions - master:") if @log_writer.debug?
+                    booted = true
+                  end
+                when PIPE_IDLE
+                  if idle_workers[pid]
+                    idle_workers.delete pid
+                  else
+                    idle_workers[pid] = true
+                  end
                 end
               else
                 log "! Out-of-sync worker list, no #{pid} worker"
               end
             end
 
+            if in_phased_restart && workers_not_booted.zero?
+              @events.fire_on_booted!
+              debug_loaded_extensions("Loaded Extensions - master:") if @log_writer.debug?
+              in_phased_restart = false
+            end
           rescue Interrupt
             @status = :stop
           end
@@ -512,5 +531,90 @@ module Puma
         @wakeup.close
       end
     end
+
+    private
+
+    def single_worker_warning
+      return if @options[:workers] != 1 || @options[:silence_single_worker_warning]
+
+      log "! WARNING: Detected running cluster mode with 1 worker."
+      log "! Running Puma in cluster mode with a single worker is often a misconfiguration."
+      log "! Consider running Puma in single-mode (workers = 0) in order to reduce memory overhead."
+      log "! Set the `silence_single_worker_warning` option to silence this warning message."
+    end
+
+    # loops thru @workers, removing workers that exited, and calling
+    # `#term` if needed
+    def wait_workers
+      # Reap all children, known workers or otherwise.
+      # If puma has PID 1, as it's common in containerized environments,
+      # then it's responsible for reaping orphaned processes, so we must reap
+      # all our dead children, regardless of whether they are workers we spawned
+      # or some reattached processes.
+      reaped_children = {}
+      loop do
+        begin
+          pid, status = Process.wait2(-1, Process::WNOHANG)
+          break unless pid
+          reaped_children[pid] = status
+        rescue Errno::ECHILD
+          break
+        end
+      end
+
+      @workers.reject! do |w|
+        next false if w.pid.nil?
+        begin
+          # We may need to check the PID individually because:
+          # 1. From Ruby versions 2.6 to 3.2, `Process.detach` can prevent or delay
+          #    `Process.wait2(-1)` from detecting a terminated process: https://bugs.ruby-lang.org/issues/19837.
+          # 2. When `fork_worker` is enabled, some worker may not be direct children,
+          #    but grand children.  Because of this they won't be reaped by `Process.wait2(-1)`.
+          if reaped_children.delete(w.pid) || Process.wait(w.pid, Process::WNOHANG)
+            true
+          else
+            w.term if w.term?
+            nil
+          end
+        rescue Errno::ECHILD
+          begin
+            Process.kill(0, w.pid)
+            # child still alive but has another parent (e.g., using fork_worker)
+            w.term if w.term?
+            false
+          rescue Errno::ESRCH, Errno::EPERM
+            true # child is already terminated
+          end
+        end
+      end
+
+      # Log unknown children
+      reaped_children.each do |pid, status|
+        log "! reaped unknown child process pid=#{pid} status=#{status}"
+      end
+    end
+
+    # @version 5.0.0
+    def timeout_workers
+      @workers.each do |w|
+        if !w.term? && w.ping_timeout <= Time.now
+          details = if w.booted?
+                      "(Worker #{w.index} failed to check in within #{@options[:worker_timeout]} seconds)"
+                    else
+                      "(Worker #{w.index} failed to boot within #{@options[:worker_boot_timeout]} seconds)"
+                    end
+          log "! Terminating timed out worker #{details}: #{w.pid}"
+          w.kill
+        end
+      end
+    end
+
+    def idle_timed_out_worker_pids
+      idle_workers.keys
+    end
+
+    def idle_workers
+      @idle_workers ||= {}
+    end
   end
 end