puma 6.4.3 → 8.0.2

data/lib/puma/client_env.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+module Puma
+
+  #———————————————————————— DO NOT USE — this class is for internal use only ———
+
+
+  # This module is included in `Client`.  It contains code to process the `env`
+  # before it is passed to the app.
+  #
+  module ClientEnv # :nodoc:
+
+    include Puma::Const
+
+    # Given a Hash +env+ for the request read from +client+, add
+    # and fixup keys to comply with Rack's env guidelines.
+    # @param env [Hash] see Puma::Client#env, from request
+    # @param client [Puma::Client] only needed for Client#peerip
+    #
+    def normalize_env
+      if host = @env[HTTP_HOST]
+        # host can be a hostname, ipv4 or bracketed ipv6. Followed by an optional port.
+        if colon = host.rindex("]:") # IPV6 with port
+          @env[SERVER_NAME] = host[0, colon+1]
+          @env[SERVER_PORT] = host[colon+2, host.bytesize]
+        elsif !host.start_with?("[") && colon = host.index(":") # not hostname or IPV4 with port
+          @env[SERVER_NAME] = host[0, colon]
+          @env[SERVER_PORT] = host[colon+1, host.bytesize]
+        else
+          @env[SERVER_NAME] = host
+          @env[SERVER_PORT] = default_server_port
+        end
+      else
+        @env[SERVER_NAME] = LOCALHOST
+        @env[SERVER_PORT] = default_server_port
+      end
+
+      unless @env[REQUEST_PATH]
+        # it might be a dumbass full host request header
+        uri = begin
+          URI.parse(@env[REQUEST_URI])
+        rescue URI::InvalidURIError
+          raise Puma::HttpParserError
+        end
+        @env[REQUEST_PATH] = uri.path
+
+        # A nil env value will cause a LintError (and fatal errors elsewhere),
+        # so only set the env value if there actually is a value.
+        @env[QUERY_STRING] = uri.query if uri.query
+      end
+
+      @env[PATH_INFO] = @env[REQUEST_PATH].to_s # #to_s in case it's nil
+
+      # From https://www.ietf.org/rfc/rfc3875 :
+      # "Script authors should be aware that the REMOTE_ADDR and
+      # REMOTE_HOST meta-variables (see sections 4.1.8 and 4.1.9)
+      # may not identify the ultimate source of the request.
+      # They identify the client for the immediate request to the
+      # server; that client may be a proxy, gateway, or other
+      # intermediary acting on behalf of the actual source client."
+      #
+
+      unless @env.key?(REMOTE_ADDR)
+        begin
+          addr = peerip
+        rescue Errno::ENOTCONN
+          # Client disconnects can result in an inability to get the
+          # peeraddr from the socket; default to unspec.
+          if peer_family == Socket::AF_INET6
+            addr = UNSPECIFIED_IPV6
+          else
+            addr = UNSPECIFIED_IPV4
+          end
+        end
+
+        # Set unix socket addrs to localhost
+        if addr.empty?
+          addr = peer_family == Socket::AF_INET6 ? LOCALHOST_IPV6 : LOCALHOST_IPV4
+        end
+
+        @env[REMOTE_ADDR] = addr
+      end
+
+      # The legacy HTTP_VERSION header can be sent as a client header.
+      # Rack v4 may remove using HTTP_VERSION.  If so, remove this line.
+      @env[HTTP_VERSION] = @env[SERVER_PROTOCOL] if @env_set_http_version
+
+      @env[PUMA_SOCKET] = @io
+
+      if @env[HTTPS_KEY] && @io.peercert
+        @env[PUMA_PEERCERT] = @io.peercert
+      end
+
+      @env[HIJACK_P] = true
+      @env[HIJACK] = method(:full_hijack).to_proc
+
+      @env[RACK_INPUT] = @body || EmptyBody
+      @env[RACK_URL_SCHEME] ||= default_server_port == PORT_443 ? HTTPS : HTTP
+    end
+
+    # Fixup any headers with `,` in the name to have `_` now. We emit
+    # headers with `,` in them during the parse phase to avoid ambiguity
+    # with the `-` to `_` conversion for critical headers. But here for
+    # compatibility, we'll convert them back. This code is written to
+    # avoid allocation in the common case (ie there are no headers
+    # with `,` in their names), that's why it has the extra conditionals.
+    #
+    # @note If a normalized version of a `,` header already exists, we ignore
+    #       the `,` version. This prevents clobbering headers managed by proxies
+    #       but not by clients (Like X-Forwarded-For).
+    #
+    # @param env [Hash] see Puma::Client#env, from request, modifies in place
+    # @version 5.0.3
+    #
+    def req_env_post_parse
+      to_delete = nil
+      to_add = nil
+
+      @env.each do |k,v|
+        if k.start_with?("HTTP_") && k.include?(",") && !UNMASKABLE_HEADERS.key?(k)
+          if to_delete
+            to_delete << k
+          else
+            to_delete = [k]
+          end
+
+          new_k = k.tr(",", "_")
+          if @env.key?(new_k)
+            next
+          end
+
+          unless to_add
+            to_add = {}
+          end
+
+          to_add[new_k] = v
+        end
+      end
+
+      if to_delete # rubocop:disable Style/SafeNavigation
+        to_delete.each { |k| env.delete(k) }
+      end
+
+      if to_add
+        @env.merge! to_add
+      end
+
+      # A rack extension. If the app writes #call'ables to this
+      # array, we will invoke them when the request is done.
+      #
+      env[RACK_AFTER_REPLY] ||= []
+      env[RACK_RESPONSE_FINISHED] ||= []
+    end
+
+    HTTP_ON_VALUES = { "on" => true, HTTPS => true }
+    private_constant :HTTP_ON_VALUES
+
+    # @return [Puma::Const::PORT_443,Puma::Const::PORT_80]
+    #
+    def default_server_port
+      if HTTP_ON_VALUES[@env[HTTPS_KEY]] ||
+          @env[HTTP_X_FORWARDED_PROTO]&.start_with?(HTTPS) ||
+          @env[HTTP_X_FORWARDED_SCHEME] == HTTPS ||
+          @env[HTTP_X_FORWARDED_SSL] == "on"
+        PORT_443
+      else
+        PORT_80
+      end
+    end
+  end
+end

data/lib/puma/cluster/worker.rb

@@ -14,7 +14,7 @@ module Puma
     class Worker < Puma::Runner # :nodoc:
       attr_reader :index, :master
 
-      def initialize(index:, master:, launcher:, pipes:, server: nil)
+      def initialize(index:, master:, launcher:, pipes:, app: nil)
         super(launcher)
 
         @index = index
@@ -23,7 +23,8 @@ module Puma
         @worker_write = pipes[:worker_write]
         @fork_pipe = pipes[:fork_pipe]
         @wakeup = pipes[:wakeup]
-        @server = server
+        @app = app
+        @server = nil
         @hook_data = {}
       end
 
@@ -57,7 +58,7 @@ module Puma
         @config.run_hooks(:before_worker_boot, index, @log_writer, @hook_data)
 
         begin
-        server = @server ||= start_server
+          @server = start_server
         rescue Exception => e
           log "! Unable to start worker"
           log e
@@ -85,36 +86,37 @@ module Puma
               if idx == -1 # stop server
                 if restart_server.length > 0
                   restart_server.clear
-                  server.begin_restart(true)
+                  @server.begin_restart(true)
                   @config.run_hooks(:before_refork, nil, @log_writer, @hook_data)
                 end
+              elsif idx == -2 # refork cycle is done
+                @config.run_hooks(:after_refork, nil, @log_writer, @hook_data)
               elsif idx == 0 # restart server
                 restart_server << true << false
               else # fork worker
                 worker_pids << pid = spawn_worker(idx)
-                @worker_write << "f#{pid}:#{idx}\n" rescue nil
+                @worker_write << "#{PIPE_FORK}#{pid}:#{idx}\n" rescue nil
               end
             end
           end
         end
 
         Signal.trap "SIGTERM" do
-          @worker_write << "e#{Process.pid}\n" rescue nil
+          @worker_write << "#{PIPE_EXTERNAL_TERM}#{Process.pid}\n" rescue nil
           restart_server.clear
-          server.stop
+          @server.stop
           restart_server << false
         end
 
         begin
-          @worker_write << "b#{Process.pid}:#{index}\n"
+          @worker_write << "#{PIPE_BOOT}#{Process.pid}:#{index}\n"
         rescue SystemCallError, IOError
-          Puma::Util.purge_interrupt_queue
           STDERR.puts "Master seems to have exited, exiting."
           return
         end
 
         while restart_server.pop
-          server_thread = server.run
+          server_thread = @server.run
 
           if @log_writer.debug? && index == 0
             debug_loaded_extensions "Loaded Extensions - worker 0:"
@@ -122,19 +124,20 @@ module Puma
 
           stat_thread ||= Thread.new(@worker_write) do |io|
             Puma.set_thread_name "stat pld"
-            base_payload = "p#{Process.pid}"
+            base_payload = "#{PIPE_PING}#{Process.pid}"
 
             while true
               begin
-                b = server.backlog || 0
-                r = server.running || 0
-                t = server.pool_capacity || 0
-                m = server.max_threads || 0
-                rc = server.requests_count || 0
-                payload = %Q!#{base_payload}{ "backlog":#{b}, "running":#{r}, "pool_capacity":#{t}, "max_threads": #{m}, "requests_count": #{rc} }\n!
-                io << payload
+                payload = base_payload.dup
+
+                hsh = @server.stats
+                hsh.each do |k, v|
+                  payload << %Q! "#{k}":#{v || 0},!
+                end
+                # sub call properly adds 'closing' string
+                io << payload.sub(/,\z/, " }\n")
+                @server.reset_max
               rescue IOError
-                Puma::Util.purge_interrupt_queue
                 break
               end
               sleep @options[:worker_check_interval]
@@ -147,7 +150,7 @@ module Puma
         # exiting until any background operations are completed
         @config.run_hooks(:before_worker_shutdown, index, @log_writer, @hook_data)
       ensure
-        @worker_write << "t#{Process.pid}\n" rescue nil
+        @worker_write << "#{PIPE_TERM}#{Process.pid}\n" rescue nil
         @worker_write.close
       end
 
@@ -162,7 +165,7 @@ module Puma
                                   launcher: @launcher,
                                   pipes: { check_pipe: @check_pipe,
                                            worker_write: @worker_write },
-                                  server: @server
+                                  app: @app
           new_worker.run
         end
 

data/lib/puma/cluster/worker_handle.rb

@@ -4,13 +4,15 @@ module Puma
   class Cluster < Runner
     #—————————————————————— DO NOT USE — this class is for internal use only ———
 
-
     # This class represents a worker process from the perspective of the puma
     # master process. It contains information about the process and its health
     # and it exposes methods to control the process via IPC. It does not
     # include the actual logic executed by the worker process itself. For that,
     # see Puma::Cluster::Worker.
     class WorkerHandle # :nodoc:
+      # array of stat 'max' keys
+      WORKER_MAX_KEYS = [:backlog_max, :reactor_max]
+
       def initialize(idx, pid, phase, options)
         @index = idx
         @pid = pid
@@ -23,12 +25,13 @@ module Puma
         @last_checkin = Time.now
         @last_status = {}
         @term = false
+        @worker_max = Array.new WORKER_MAX_KEYS.length, 0
       end
 
-      attr_reader :index, :pid, :phase, :signal, :last_checkin, :last_status, :started_at
+      attr_reader :index, :pid, :phase, :signal, :last_checkin, :last_status, :started_at, :process_status
 
       # @version 5.0.0
-      attr_writer :pid, :phase
+      attr_writer :pid, :phase, :process_status
 
       def booted?
         @stage == :booted
@@ -52,12 +55,39 @@ module Puma
       end
 
       def ping!(status)
-        @last_checkin = Time.now
-        captures = status.match(/{ "backlog":(?<backlog>\d*), "running":(?<running>\d*), "pool_capacity":(?<pool_capacity>\d*), "max_threads": (?<max_threads>\d*), "requests_count": (?<requests_count>\d*) }/)
-        @last_status = captures.names.inject({}) do |hash, key|
-          hash[key.to_sym] = captures[key].to_i
-          hash
+        hsh = {}
+        k, v = nil, nil
+        status.tr('}{"', '').strip.split(", ") do |kv|
+          cntr = 0
+          kv.split(':') do |t|
+            if cntr == 0
+              k = t
+              cntr = 1
+            else
+              v = t
+            end
+          end
+          hsh[k.to_sym] = v.to_i
+        end
+
+        # check stat max values, we can't signal workers to reset the max values,
+        # so we do so here
+        WORKER_MAX_KEYS.each_with_index do |key, idx|
+          next unless hsh[key]
+
+          if hsh[key] < @worker_max[idx]
+            hsh[key] = @worker_max[idx]
+          else
+            @worker_max[idx] = hsh[key]
+          end
         end
+        @last_checkin = Time.now
+        @last_status = hsh
+      end
+
+      # Resets max values to zero.  Called whenever `Cluster#stats` is called
+      def reset_max
+        WORKER_MAX_KEYS.length.times { |idx| @worker_max[idx] = 0 }
       end
 
       # @see Puma::Cluster#check_workers

data/lib/puma/cluster.rb

@@ -22,7 +22,8 @@ module Puma
       @workers = []
       @next_check = Time.now
 
-      @phased_restart = false
+      @worker_max = [] # keeps track of 'max' stat values
+      @pending_phased_restart = false
     end
 
     # Returns the list of cluster worker handles.
@@ -44,10 +45,14 @@ module Puma
       end
     end
 
-    def start_phased_restart
-      @events.fire_on_restart!
+    def start_phased_restart(refork = false)
+      @events.fire_before_restart!
       @phase += 1
-      log "- Starting phased worker restart, phase: #{@phase}"
+      if refork
+        log "- Starting worker refork, phase: #{@phase}"
+      else
+        log "- Starting phased worker restart, phase: #{@phase}"
+      end
 
       # Be sure to change the directory again before loading
       # the app. This way we can pick up new code.
@@ -82,11 +87,15 @@ module Puma
         end
 
         debug "Spawned worker: #{pid}"
-        @workers << WorkerHandle.new(idx, pid, @phase, @options)
+        @workers.insert(idx, WorkerHandle.new(idx, pid, @phase, @options))
       end
 
       if @options[:fork_worker] && all_workers_in_phase?
         @fork_writer << "0\n"
+
+        if worker_at(0).phase > 0
+          @fork_writer << "-2\n"
+        end
       end
     end
 
@@ -162,7 +171,7 @@ module Puma
       (@workers.map(&:pid) - idle_timed_out_worker_pids).empty?
     end
 
-    def check_workers
+    def check_workers(refork = false)
       return if @next_check >= Time.now
 
       @next_check = Time.now + @options[:worker_check_interval]
@@ -177,10 +186,15 @@ module Puma
         # we need to phase any workers out (which will restart
         # in the right phase).
         #
-        w = @workers.find { |x| x.phase != @phase }
+        w = @workers.find { |x| x.phase < @phase }
 
         if w
-          log "- Stopping #{w.pid} for phased upgrade..."
+          if refork
+            log "- Stopping #{w.pid} for refork..."
+          else
+            log "- Stopping #{w.pid} for phased upgrade..."
+          end
+
           unless w.term?
             w.term
             log "- #{w.signal} sent to #{w.pid}..."
@@ -207,12 +221,11 @@ module Puma
         pipes[:wakeup] = @wakeup
       end
 
-      server = start_server if preload?
       new_worker = Worker.new index: index,
                               master: master,
                               launcher: @launcher,
                               pipes: pipes,
-                              server: server
+                              app: (app if preload?)
       new_worker.run
     end
 
@@ -224,7 +237,7 @@ module Puma
     def phased_restart(refork = false)
       return false if @options[:preload_app] && !refork
 
-      @phased_restart = true
+      @pending_phased_restart = refork ? :refork : true
       wakeup!
 
       true
@@ -254,11 +267,14 @@ module Puma
     end
 
     # Inside of a child process, this will return all zeroes, as @workers is only populated in
-    # the master process.
+    # the master process.  Calling this also resets stat 'max' values to zero.
     # @!attribute [r] stats
+    # @return [Hash]
+
     def stats
       old_worker_count = @workers.count { |w| w.phase != @phase }
       worker_status = @workers.map do |w|
+        w.reset_max
         {
           started_at: utc_iso8601(w.started_at),
           pid: w.pid,
@@ -269,7 +285,6 @@ module Puma
           last_status: w.last_status,
         }
       end
-
       {
         started_at: utc_iso8601(@started_at),
         workers: @workers.size,
@@ -338,7 +353,7 @@ module Puma
 
           stop_workers
           stop
-          @events.fire_on_stopped!
+          @events.fire_after_stopped!
           raise(SignalException, "SIGTERM") if @options[:raise_exception_on_sigterm]
           exit 0 # Clean exit, workers were stopped
         end
@@ -348,8 +363,6 @@ module Puma
     def run
       @status = :run
 
-      @idle_workers = {}
-
       output_header "cluster"
 
       # This is aligned with the output from Runner, see Runner#output_header
@@ -357,16 +370,12 @@ module Puma
 
       if preload?
         # Threads explicitly marked as fork safe will be ignored. Used in Rails,
-        # but may be used by anyone. Note that we need to explicit
-        # Process::Waiter check here because there's a bug in Ruby 2.6 and below
-        # where calling thread_variable_get on a Process::Waiter will segfault.
-        # We can drop that clause once those versions of Ruby are no longer
-        # supported.
-        fork_safe = ->(t) { !t.is_a?(Process::Waiter) && t.thread_variable_get(:fork_safe) }
+        # but may be used by anyone.
+        fork_safe = ->(t) { t.thread_variable_get(:fork_safe) }
 
         before = Thread.list.reject(&fork_safe)
 
-        log "*     Restarts: (\u2714) hot (\u2716) phased"
+        log "*     Restarts: (\u2714) hot (\u2716) phased (#{@options[:fork_worker] ? "\u2714" : "\u2716"}) refork"
         log "* Preloading application"
         load_and_bind
 
@@ -384,7 +393,7 @@ module Puma
           end
         end
       else
-        log "*     Restarts: (\u2714) hot (\u2714) phased"
+        log "*     Restarts: (\u2714) hot (\u2714) phased (#{@options[:fork_worker] ? "\u2714" : "\u2716"}) refork"
 
         unless @config.app_configured?
           error "No application configured, nothing to run"
@@ -411,6 +420,7 @@ module Puma
 
       log "Use Ctrl-C to stop"
 
+      warn_ruby_mn_threads
       single_worker_warning
 
       redirect_io
@@ -440,30 +450,37 @@ module Puma
 
         while @status == :run
           begin
-            if all_workers_idle_timed_out?
+            if @options[:idle_timeout] && all_workers_idle_timed_out?
               log "- All workers reached idle timeout"
               break
             end
 
-            if @phased_restart
-              start_phased_restart
-              @phased_restart = false
-              in_phased_restart = true
+            if @pending_phased_restart
+              start_phased_restart(@pending_phased_restart == :refork)
+
+              in_phased_restart = @pending_phased_restart
+              @pending_phased_restart = false
+
               workers_not_booted = @options[:workers]
+              # worker 0 is not restarted on refork
+              workers_not_booted -= 1 if in_phased_restart == :refork
             end
 
-            check_workers
+            check_workers(in_phased_restart == :refork)
 
             if read.wait_readable([0, @next_check - Time.now].max)
               req = read.read_nonblock(1)
+              next unless req
 
-              @next_check = Time.now if req == "!"
-              next if !req || req == "!"
+              if req == PIPE_WAKEUP
+                @next_check = Time.now
+                next
+              end
 
               result = read.gets
               pid = result.to_i
 
-              if req == "b" || req == "f"
+              if req == PIPE_BOOT || req == PIPE_FORK
                 pid, idx = result.split(':').map(&:to_i)
                 w = worker_at idx
                 w.pid = pid if w.pid.nil?
@@ -471,36 +488,36 @@ module Puma
 
               if w = @workers.find { |x| x.pid == pid }
                 case req
-                when "b"
+                when PIPE_BOOT
                   w.boot!
                   log "- Worker #{w.index} (PID: #{pid}) booted in #{w.uptime.round(2)}s, phase: #{w.phase}"
                   @next_check = Time.now
                   workers_not_booted -= 1
-                when "e"
+                when PIPE_EXTERNAL_TERM
                   # external term, see worker method, Signal.trap "SIGTERM"
                   w.term!
-                when "t"
+                when PIPE_TERM
                   w.term unless w.term?
-                when "p"
+                when PIPE_PING
                   status = result.sub(/^\d+/,'').chomp
                   w.ping!(status)
                   @events.fire(:ping!, w)
 
-                  if in_phased_restart && workers_not_booted.positive? && w0 = worker_at(0)
+                  if in_phased_restart && @options[:fork_worker] && workers_not_booted.positive? && w0 = worker_at(0)
                     w0.ping!(status)
                     @events.fire(:ping!, w0)
                   end
 
                   if !booted && @workers.none? {|worker| worker.last_status.empty?}
-                    @events.fire_on_booted!
+                    @events.fire_after_booted!
                     debug_loaded_extensions("Loaded Extensions - master:") if @log_writer.debug?
                     booted = true
                   end
-                when "i"
-                  if @idle_workers[pid]
-                    @idle_workers.delete pid
+                when PIPE_IDLE
+                  if idle_workers[pid]
+                    idle_workers.delete pid
                   else
-                    @idle_workers[pid] = true
+                    idle_workers[pid] = true
                   end
                 end
               else
@@ -509,7 +526,7 @@ module Puma
             end
 
             if in_phased_restart && workers_not_booted.zero?
-              @events.fire_on_booted!
+              @events.fire_after_booted!
               debug_loaded_extensions("Loaded Extensions - master:") if @log_writer.debug?
               in_phased_restart = false
             end
@@ -560,9 +577,14 @@ module Puma
       @workers.reject! do |w|
         next false if w.pid.nil?
         begin
-          # When `fork_worker` is enabled, some worker may not be direct children, but grand children.
-          # Because of this they won't be reaped by `Process.wait2(-1)`, so we need to check them individually)
-          if reaped_children.delete(w.pid) || (@options[:fork_worker] && Process.wait(w.pid, Process::WNOHANG))
+          # We may need to check the PID individually because:
+          # 1. From Ruby versions 2.6 to 3.2, `Process.detach` can prevent or delay
+          #    `Process.wait2(-1)` from detecting a terminated process: https://bugs.ruby-lang.org/issues/19837.
+          # 2. When `fork_worker` is enabled, some worker may not be direct children,
+          #    but grand children.  Because of this they won't be reaped by `Process.wait2(-1)`.
+          if (status = reaped_children.delete(w.pid) || Process.wait2(w.pid, Process::WNOHANG)&.last)
+            w.process_status = status
+            @config.run_hooks(:after_worker_shutdown, w, @log_writer)
             true
           else
             w.term if w.term?
@@ -602,7 +624,11 @@ module Puma
     end
 
     def idle_timed_out_worker_pids
-      @idle_workers.keys
+      idle_workers.keys
+    end
+
+    def idle_workers
+      @idle_workers ||= {}
     end
   end
 end