puma 5.6.8 → 6.4.2

data/lib/puma/client.rb

@@ -8,9 +8,9 @@ class IO
   end
 end
 
-require 'puma/detect'
+require_relative 'detect'
+require_relative 'io_buffer'
 require 'tempfile'
-require 'forwardable'
 
 if Puma::IS_JRUBY
   # We have to work around some OpenSSL buffer/io-readiness bugs
@@ -25,6 +25,9 @@ module Puma
 
   class HttpParserError501 < IOError; end
 
+  #———————————————————————— DO NOT USE — this class is for internal use only ———
+
+
   # An instance of this class represents a unique request from a client.
   # For example, this could be a web request from a browser or from CURL.
   #
@@ -38,7 +41,7 @@ module Puma
   # the header and body are fully buffered via the `try_to_finish` method.
   # They can be used to "time out" a response via the `timeout_at` reader.
   #
-  class Client
+  class Client # :nodoc:
 
     # this tests all values but the last, which must be chunked
     ALLOWED_TRANSFER_ENCODING = %w[compress deflate gzip].freeze
@@ -66,17 +69,13 @@ module Puma
     EmptyBody = NullIO.new
 
     include Puma::Const
-    extend Forwardable
 
     def initialize(io, env=nil)
       @io = io
       @to_io = io.to_io
+      @io_buffer = IOBuffer.new
       @proto_env = env
-      if !env
-        @env = nil
-      else
-        @env = env.dup
-      end
+      @env = env&.dup
 
       @parser = HttpParser.new
       @parsed_bytes = 0
@@ -94,7 +93,11 @@ module Puma
       @requests_served = 0
       @hijacked = false
 
+      @http_content_length_limit = nil
+      @http_content_length_limit_exceeded = false
+
       @peerip = nil
+      @peer_family = nil
       @listener = nil
       @remote_addr_header = nil
       @expect_proxy_proto = false
@@ -102,16 +105,22 @@ module Puma
       @body_remain = 0
 
       @in_last_chunk = false
+
+      # need unfrozen ASCII-8BIT, +'' is UTF-8
+      @read_buffer = String.new # rubocop: disable Performance/UnfreezeString
     end
 
     attr_reader :env, :to_io, :body, :io, :timeout_at, :ready, :hijacked,
-                :tempfile
+                :tempfile, :io_buffer, :http_content_length_limit_exceeded
 
-    attr_writer :peerip
+    attr_writer :peerip, :http_content_length_limit
 
     attr_accessor :remote_addr_header, :listener
 
-    def_delegators :@io, :closed?
+    # Remove in Puma 7?
+    def closed?
+      @to_io.closed?
+    end
 
     # Test to see if io meets a bare minimum of functioning, @to_io needs to be
     # used for MiniSSL::Socket
@@ -147,6 +156,7 @@ module Puma
 
     def reset(fast_check=true)
       @parser.reset
+      @io_buffer.reset
       @read_header = true
       @read_proxy = !!@expect_proxy_proto
       @env = @proto_env.dup
@@ -157,6 +167,7 @@ module Puma
       @body_remain = 0
       @peerip = nil if @remote_addr_header
       @in_last_chunk = false
+      @http_content_length_limit_exceeded = false
 
       if @buffer
         return false unless try_to_parse_proxy_protocol
@@ -216,6 +227,17 @@ module Puma
     end
 
     def try_to_finish
+      if env[CONTENT_LENGTH] && above_http_content_limit(env[CONTENT_LENGTH].to_i)
+        @http_content_length_limit_exceeded = true
+      end
+
+      if @http_content_length_limit_exceeded
+        @buffer = nil
+        @body = EmptyBody
+        set_ready
+        return true
+      end
+
       return read_body if in_data_phase
 
       begin
@@ -245,6 +267,10 @@ module Puma
 
       @parsed_bytes = @parser.execute(@env, @buffer, @parsed_bytes)
 
+      if @parser.finished? && above_http_content_limit(@parser.body.bytesize)
+        @http_content_length_limit_exceeded = true
+      end
+
       if @parser.finished?
         return setup_body
       elsif @parsed_bytes >= MAX_HEADER
@@ -282,7 +308,7 @@ module Puma
       return @peerip if @peerip
 
       if @remote_addr_header
-        hdr = (@env[@remote_addr_header] || LOCALHOST_IP).split(/[\s,]/).first
+        hdr = (@env[@remote_addr_header] || @io.peeraddr.last).split(/[\s,]/).first
         @peerip = hdr
         return hdr
       end
@@ -290,6 +316,16 @@ module Puma
       @peerip ||= @io.peeraddr.last
     end
 
+    def peer_family
+      return @peer_family if @peer_family
+
+      @peer_family ||= begin
+                         @io.local_address.afamily
+                       rescue
+                         Socket::AF_INET
+                       end
+    end
+
     # Returns true if the persistent connection can be closed immediately
     # without waiting for the configured idle/shutdown timeout.
     # @version 5.0.0
@@ -313,7 +349,7 @@ module Puma
     private
 
     def setup_body
-      @body_read_start = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+      @body_read_start = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond)
 
       if @env[HTTP_EXPECT] == CONTINUE
         # TODO allow a hook here to check the headers before
@@ -357,7 +393,7 @@ module Puma
 
       if cl
         # cannot contain characters that are not \d, or be empty
-        if cl =~ CONTENT_LENGTH_VALUE_INVALID || cl.empty?
+        if CONTENT_LENGTH_VALUE_INVALID.match?(cl) || cl.empty?
           raise HttpParserError, "Invalid Content-Length: #{cl.inspect}"
         end
       else
@@ -410,7 +446,7 @@ module Puma
       end
 
       begin
-        chunk = @io.read_nonblock(want)
+        chunk = @io.read_nonblock(want, @read_buffer)
       rescue IO::WaitReadable
         return false
       rescue SystemCallError, IOError
@@ -442,7 +478,7 @@ module Puma
     def read_chunked_body
       while true
         begin
-          chunk = @io.read_nonblock(4096)
+          chunk = @io.read_nonblock(4096, @read_buffer)
         rescue IO::WaitReadable
           return false
         rescue SystemCallError, IOError
@@ -523,7 +559,7 @@ module Puma
           # Puma doesn't process chunk extensions, but should parse if they're
           # present, which is the reason for the semicolon regex
           chunk_hex = line.strip[/\A[^;]+/]
-          if chunk_hex =~ CHUNK_SIZE_INVALID
+          if CHUNK_SIZE_INVALID.match? chunk_hex
             raise HttpParserError, "Invalid chunk size: '#{chunk_hex}'"
           end
           len = chunk_hex.to_i(16)
@@ -610,10 +646,14 @@ module Puma
 
     def set_ready
       if @body_read_start
-        @env['puma.request_body_wait'] = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond) - @body_read_start
+        @env['puma.request_body_wait'] = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond) - @body_read_start
       end
       @requests_served += 1
       @ready = true
     end
+
+    def above_http_content_limit(value)
+      @http_content_length_limit&.< value
+    end
   end
 end

data/lib/puma/cluster/worker.rb

@@ -2,27 +2,29 @@
 
 module Puma
   class Cluster < Puma::Runner
+    #—————————————————————— DO NOT USE — this class is for internal use only ———
+
+
     # This class is instantiated by the `Puma::Cluster` and represents a single
     # worker process.
     #
     # At the core of this class is running an instance of `Puma::Server` which
     # gets created via the `start_server` method from the `Puma::Runner` class
     # that this inherits from.
-    class Worker < Puma::Runner
+    class Worker < Puma::Runner # :nodoc:
       attr_reader :index, :master
 
       def initialize(index:, master:, launcher:, pipes:, server: nil)
-        super launcher, launcher.events
+        super(launcher)
 
         @index = index
         @master = master
-        @launcher = launcher
-        @options = launcher.options
         @check_pipe = pipes[:check_pipe]
         @worker_write = pipes[:worker_write]
         @fork_pipe = pipes[:fork_pipe]
         @wakeup = pipes[:wakeup]
         @server = server
+        @hook_data = {}
       end
 
       def run
@@ -52,13 +54,14 @@ module Puma
 
         # Invoke any worker boot hooks so they can get
         # things in shape before booting the app.
-        @launcher.config.run_hooks :before_worker_boot, index, @launcher.events
+        @config.run_hooks(:before_worker_boot, index, @log_writer, @hook_data)
 
         begin
         server = @server ||= start_server
         rescue Exception => e
           log "! Unable to start worker"
-          log e.backtrace[0]
+          log e
+          log e.backtrace.join("\n    ")
           exit 1
         end
 
@@ -83,8 +86,7 @@ module Puma
                 if restart_server.length > 0
                   restart_server.clear
                   server.begin_restart(true)
-                  @launcher.config.run_hooks :before_refork, nil, @launcher.events
-                  Puma::Util.nakayoshi_gc @events if @options[:nakayoshi_fork]
+                  @config.run_hooks(:before_refork, nil, @log_writer, @hook_data)
                 end
               elsif idx == 0 # restart server
                 restart_server << true << false
@@ -113,6 +115,11 @@ module Puma
 
         while restart_server.pop
           server_thread = server.run
+
+          if @log_writer.debug? && index == 0
+            debug_loaded_extensions "Loaded Extensions - worker 0:"
+          end
+
           stat_thread ||= Thread.new(@worker_write) do |io|
             Puma.set_thread_name "stat pld"
             base_payload = "p#{Process.pid}"
@@ -138,7 +145,7 @@ module Puma
 
         # Invoke any worker shutdown hooks so they can prevent the worker
         # exiting until any background operations are completed
-        @launcher.config.run_hooks :before_worker_shutdown, index, @launcher.events
+        @config.run_hooks(:before_worker_shutdown, index, @log_writer, @hook_data)
       ensure
         @worker_write << "t#{Process.pid}\n" rescue nil
         @worker_write.close
@@ -147,7 +154,7 @@ module Puma
       private
 
       def spawn_worker(idx)
-        @launcher.config.run_hooks :before_worker_fork, idx, @launcher.events
+        @config.run_hooks(:before_worker_fork, idx, @log_writer, @hook_data)
 
         pid = fork do
           new_worker = Worker.new index: idx,
@@ -165,7 +172,7 @@ module Puma
           exit! 1
         end
 
-        @launcher.config.run_hooks :after_worker_fork, idx, @launcher.events
+        @config.run_hooks(:after_worker_fork, idx, @log_writer, @hook_data)
         pid
       end
     end

data/lib/puma/cluster/worker_handle.rb

@@ -2,12 +2,15 @@
 
 module Puma
   class Cluster < Runner
+    #—————————————————————— DO NOT USE — this class is for internal use only ———
+
+
     # This class represents a worker process from the perspective of the puma
     # master process. It contains information about the process and its health
     # and it exposes methods to control the process via IPC. It does not
     # include the actual logic executed by the worker process itself. For that,
     # see Puma::Cluster::Worker.
-    class WorkerHandle
+    class WorkerHandle # :nodoc:
       def initialize(idx, pid, phase, options)
         @index = idx
         @pid = pid

data/lib/puma/cluster.rb

@@ -1,12 +1,10 @@
 # frozen_string_literal: true
 
-require 'puma/runner'
-require 'puma/util'
-require 'puma/plugin'
-require 'puma/cluster/worker_handle'
-require 'puma/cluster/worker'
-
-require 'time'
+require_relative 'runner'
+require_relative 'util'
+require_relative 'plugin'
+require_relative 'cluster/worker_handle'
+require_relative 'cluster/worker'
 
 module Puma
   # This class is instantiated by the `Puma::Launcher` and used
@@ -17,8 +15,8 @@ module Puma
   # via the `spawn_workers` method call. Each worker will have it's own
   # instance of a `Puma::Server`.
   class Cluster < Runner
-    def initialize(cli, events)
-      super cli, events
+    def initialize(launcher)
+      super(launcher)
 
       @phase = 0
       @workers = []
@@ -27,6 +25,10 @@ module Puma
       @phased_restart = false
     end
 
+    # Returns the list of cluster worker handles.
+    # @return [Array<Puma::Cluster::WorkerHandle>]
+    attr_reader :workers
+
     def stop_workers
       log "- Gracefully shutting down workers..."
       @workers.each { |x| x.term }
@@ -83,16 +85,14 @@ module Puma
         @workers << WorkerHandle.new(idx, pid, @phase, @options)
       end
 
-      if @options[:fork_worker] &&
-        @workers.all? {|x| x.phase == @phase}
-
+      if @options[:fork_worker] && all_workers_in_phase?
         @fork_writer << "0\n"
       end
     end
 
     # @version 5.0.0
     def spawn_worker(idx, master)
-      @launcher.config.run_hooks :before_worker_fork, idx, @launcher.events
+      @config.run_hooks(:before_worker_fork, idx, @log_writer)
 
       pid = fork { worker(idx, master) }
       if !pid
@@ -101,7 +101,7 @@ module Puma
         exit! 1
       end
 
-      @launcher.config.run_hooks :after_worker_fork, idx, @launcher.events
+      @config.run_hooks(:after_worker_fork, idx, @log_writer)
       pid
     end
 
@@ -146,10 +146,22 @@ module Puma
       idx
     end
 
+    def worker_at(idx)
+      @workers.find { |w| w.index == idx }
+    end
+
     def all_workers_booted?
       @workers.count { |w| !w.booted? } == 0
     end
 
+    def all_workers_in_phase?
+      @workers.all? { |w| w.phase == @phase }
+    end
+
+    def all_workers_idle_timed_out?
+      (@workers.map(&:pid) - idle_timed_out_worker_pids).empty?
+    end
+
     def check_workers
       return if @next_check >= Time.now
 
@@ -176,10 +188,10 @@ module Puma
         end
       end
 
-      @next_check = [
-        @workers.reject(&:term?).map(&:ping_timeout).min,
-        @next_check
-      ].compact.min
+      t = @workers.reject(&:term?)
+      t.map!(&:ping_timeout)
+
+      @next_check = [t.min, @next_check].compact.min
     end
 
     def worker(index, master)
@@ -209,8 +221,8 @@ module Puma
       stop
     end
 
-    def phased_restart
-      return false if @options[:preload_app]
+    def phased_restart(refork = false)
+      return false if @options[:preload_app] && !refork
 
       @phased_restart = true
       wakeup!
@@ -226,7 +238,7 @@ module Puma
     def stop_blocked
       @status = :stop if @status == :run
       wakeup!
-      @control.stop(true) if @control
+      @control&.stop true
       Process.waitall
     end
 
@@ -248,24 +260,24 @@ module Puma
       old_worker_count = @workers.count { |w| w.phase != @phase }
       worker_status = @workers.map do |w|
         {
-          started_at: w.started_at.utc.iso8601,
+          started_at: utc_iso8601(w.started_at),
           pid: w.pid,
           index: w.index,
           phase: w.phase,
           booted: w.booted?,
-          last_checkin: w.last_checkin.utc.iso8601,
+          last_checkin: utc_iso8601(w.last_checkin),
           last_status: w.last_status,
         }
       end
 
       {
-        started_at: @started_at.utc.iso8601,
+        started_at: utc_iso8601(@started_at),
         workers: @workers.size,
         phase: @phase,
         booted_workers: worker_status.count { |w| w[:booted] },
         old_workers: old_worker_count,
         worker_status: worker_status,
-      }
+      }.merge(super)
     end
 
     def preload?
@@ -274,10 +286,10 @@ module Puma
 
     # @version 5.0.0
     def fork_worker!
-      if (worker = @workers.find { |w| w.index == 0 })
+      if (worker = worker_at 0)
         worker.phase += 1
       end
-      phased_restart
+      phased_restart(true)
     end
 
     # We do this in a separate method to keep the lambda scope
@@ -290,7 +302,7 @@ module Puma
 
         # Auto-fork after the specified number of requests.
         if (fork_requests = @options[:fork_worker].to_i) > 0
-          @launcher.events.register(:ping!) do |w|
+          @events.register(:ping!) do |w|
             fork_worker! if w.index == 0 &&
               w.phase == 0 &&
               w.last_status[:requests_count] >= fork_requests
@@ -336,6 +348,8 @@ module Puma
     def run
       @status = :run
 
+      @idle_workers = {}
+
       output_header "cluster"
 
       # This is aligned with the output from Runner, see Runner#output_header
@@ -372,12 +386,12 @@ module Puma
       else
         log "*     Restarts: (\u2714) hot (\u2714) phased"
 
-        unless @launcher.config.app_configured?
+        unless @config.app_configured?
           error "No application configured, nothing to run"
           exit 1
         end
 
-        @launcher.binder.parse @options[:binds], self
+        @launcher.binder.parse @options[:binds]
       end
 
       read, @wakeup = Puma::Util.pipe
@@ -409,8 +423,9 @@ module Puma
 
       @master_read, @worker_write = read, @wakeup
 
-      @launcher.config.run_hooks :before_fork, nil, @launcher.events
-      Puma::Util.nakayoshi_gc @events if @options[:nakayoshi_fork]
+      @options[:worker_write] = @worker_write
+
+      @config.run_hooks(:before_fork, nil, @log_writer)
 
       spawn_workers
 
@@ -425,6 +440,11 @@ module Puma
 
         while @status == :run
           begin
+            if all_workers_idle_timed_out?
+              log "- All workers reached idle timeout"
+              break
+            end
+
             if @phased_restart
               start_phased_restart
               @phased_restart = false
@@ -445,7 +465,7 @@ module Puma
 
               if req == "b" || req == "f"
                 pid, idx = result.split(':').map(&:to_i)
-                w = @workers.find {|x| x.index == idx}
+                w = worker_at idx
                 w.pid = pid if w.pid.nil?
               end
 
@@ -462,22 +482,37 @@ module Puma
                 when "t"
                   w.term unless w.term?
                 when "p"
-                  w.ping!(result.sub(/^\d+/,'').chomp)
-                  @launcher.events.fire(:ping!, w)
+                  status = result.sub(/^\d+/,'').chomp
+                  w.ping!(status)
+                  @events.fire(:ping!, w)
+
+                  if in_phased_restart && workers_not_booted.positive? && w0 = worker_at(0)
+                    w0.ping!(status)
+                    @events.fire(:ping!, w0)
+                  end
+
                   if !booted && @workers.none? {|worker| worker.last_status.empty?}
-                    @launcher.events.fire_on_booted!
+                    @events.fire_on_booted!
+                    debug_loaded_extensions("Loaded Extensions - master:") if @log_writer.debug?
                     booted = true
                   end
+                when "i"
+                  if @idle_workers[pid]
+                    @idle_workers.delete pid
+                  else
+                    @idle_workers[pid] = true
+                  end
                 end
               else
                 log "! Out-of-sync worker list, no #{pid} worker"
               end
             end
+
             if in_phased_restart && workers_not_booted.zero?
               @events.fire_on_booted!
+              debug_loaded_extensions("Loaded Extensions - master:") if @log_writer.debug?
               in_phased_restart = false
             end
-
           rescue Interrupt
             @status = :stop
           end
@@ -506,10 +541,28 @@ module Puma
     # loops thru @workers, removing workers that exited, and calling
     # `#term` if needed
     def wait_workers
+      # Reap all children, known workers or otherwise.
+      # If puma has PID 1, as it's common in containerized environments,
+      # then it's responsible for reaping orphaned processes, so we must reap
+      # all our dead children, regardless of whether they are workers we spawned
+      # or some reattached processes.
+      reaped_children = {}
+      loop do
+        begin
+          pid, status = Process.wait2(-1, Process::WNOHANG)
+          break unless pid
+          reaped_children[pid] = status
+        rescue Errno::ECHILD
+          break
+        end
+      end
+
       @workers.reject! do |w|
         next false if w.pid.nil?
         begin
-          if Process.wait(w.pid, Process::WNOHANG)
+          # When `fork_worker` is enabled, some worker may not be direct children, but grand children.
+          # Because of this they won't be reaped by `Process.wait2(-1)`, so we need to check them individually)
+          if reaped_children.delete(w.pid) || (@options[:fork_worker] && Process.wait(w.pid, Process::WNOHANG))
             true
           else
             w.term if w.term?
@@ -526,6 +579,11 @@ module Puma
           end
         end
       end
+
+      # Log unknown children
+      reaped_children.each do |pid, status|
+        log "! reaped unknown child process pid=#{pid} status=#{status}"
+      end
     end
 
     # @version 5.0.0
@@ -533,14 +591,18 @@ module Puma
       @workers.each do |w|
         if !w.term? && w.ping_timeout <= Time.now
           details = if w.booted?
-                      "(worker failed to check in within #{@options[:worker_timeout]} seconds)"
+                      "(Worker #{w.index} failed to check in within #{@options[:worker_timeout]} seconds)"
                     else
-                      "(worker failed to boot within #{@options[:worker_boot_timeout]} seconds)"
+                      "(Worker #{w.index} failed to boot within #{@options[:worker_boot_timeout]} seconds)"
                     end
           log "! Terminating timed out worker #{details}: #{w.pid}"
           w.kill
         end
       end
     end
+
+    def idle_timed_out_worker_pids
+      @idle_workers.keys
+    end
   end
 end