puma 5.3.2 → 5.6.8

data/lib/puma/client.rb

@@ -23,6 +23,8 @@ module Puma
 
   class ConnectionError < RuntimeError; end
 
+  class HttpParserError501 < IOError; end
+
   # An instance of this class represents a unique request from a client.
   # For example, this could be a web request from a browser or from CURL.
   #
@@ -35,7 +37,30 @@ module Puma
   # Instances of this class are responsible for knowing if
   # the header and body are fully buffered via the `try_to_finish` method.
   # They can be used to "time out" a response via the `timeout_at` reader.
+  #
   class Client
+
+    # this tests all values but the last, which must be chunked
+    ALLOWED_TRANSFER_ENCODING = %w[compress deflate gzip].freeze
+
+    # chunked body validation
+    CHUNK_SIZE_INVALID = /[^\h]/.freeze
+    CHUNK_VALID_ENDING = Const::LINE_END
+    CHUNK_VALID_ENDING_SIZE = CHUNK_VALID_ENDING.bytesize
+
+    # The maximum number of bytes we'll buffer looking for a valid
+    # chunk header.
+    MAX_CHUNK_HEADER_SIZE = 4096
+
+    # The maximum amount of excess data the client sends
+    # using chunk size extensions before we abort the connection.
+    MAX_CHUNK_EXCESS = 16 * 1024
+
+    # Content-Length header value validation
+    CONTENT_LENGTH_VALUE_INVALID = /[^\d]/.freeze
+
+    TE_ERR_MSG = 'Invalid Transfer-Encoding'
+
     # The object used for a request with no body. All requests with
     # no body share this one object since it has no state.
     EmptyBody = NullIO.new
@@ -56,6 +81,7 @@ module Puma
       @parser = HttpParser.new
       @parsed_bytes = 0
       @read_header = true
+      @read_proxy = false
       @ready = false
 
       @body = nil
@@ -71,6 +97,7 @@ module Puma
       @peerip = nil
       @listener = nil
       @remote_addr_header = nil
+      @expect_proxy_proto = false
 
       @body_remain = 0
 
@@ -106,7 +133,7 @@ module Puma
 
     # @!attribute [r] in_data_phase
     def in_data_phase
-      !@read_header
+      !(@read_header || @read_proxy)
     end
 
     def set_timeout(val)
@@ -121,6 +148,7 @@ module Puma
     def reset(fast_check=true)
       @parser.reset
       @read_header = true
+      @read_proxy = !!@expect_proxy_proto
       @env = @proto_env.dup
       @body = nil
       @tempfile = nil
@@ -131,6 +159,8 @@ module Puma
       @in_last_chunk = false
 
       if @buffer
+        return false unless try_to_parse_proxy_protocol
+
         @parsed_bytes = @parser.execute(@env, @buffer, @parsed_bytes)
 
         if @parser.finished?
@@ -143,8 +173,7 @@ module Puma
         return false
       else
         begin
-          if fast_check &&
-              IO.select([@to_io], nil, nil, FAST_TRACK_KA_TIMEOUT)
+          if fast_check && @to_io.wait_readable(FAST_TRACK_KA_TIMEOUT)
             return try_to_finish
           end
         rescue IOError
@@ -157,13 +186,37 @@ module Puma
     def close
       begin
         @io.close
-      rescue IOError
-        Thread.current.purge_interrupt_queue if Thread.current.respond_to? :purge_interrupt_queue
+      rescue IOError, Errno::EBADF
+        Puma::Util.purge_interrupt_queue
       end
     end
 
+    # If necessary, read the PROXY protocol from the buffer. Returns
+    # false if more data is needed.
+    def try_to_parse_proxy_protocol
+      if @read_proxy
+        if @expect_proxy_proto == :v1
+          if @buffer.include? "\r\n"
+            if md = PROXY_PROTOCOL_V1_REGEX.match(@buffer)
+              if md[1]
+                @peerip = md[1].split(" ")[0]
+              end
+              @buffer = md.post_match
+            end
+            # if the buffer has a \r\n but doesn't have a PROXY protocol
+            # request, this is just HTTP from a non-PROXY client; move on
+            @read_proxy = false
+            return @buffer.size > 0
+          else
+            return false
+          end
+        end
+      end
+      true
+    end
+
     def try_to_finish
-      return read_body unless @read_header
+      return read_body if in_data_phase
 
       begin
         data = @io.read_nonblock(CHUNK_SIZE)
@@ -188,6 +241,8 @@ module Puma
         @buffer = data
       end
 
+      return false unless try_to_parse_proxy_protocol
+
       @parsed_bytes = @parser.execute(@env, @buffer, @parsed_bytes)
 
       if @parser.finished?
@@ -202,13 +257,13 @@ module Puma
 
     def eagerly_finish
       return true if @ready
-      return false unless IO.select([@to_io], nil, nil, 0)
+      return false unless @to_io.wait_readable(0)
       try_to_finish
     end
 
     def finish(timeout)
       return if @ready
-      IO.select([@to_io], nil, nil, timeout) || timeout! until try_to_finish
+      @to_io.wait_readable(timeout) || timeout! until try_to_finish
     end
 
     def timeout!
@@ -244,6 +299,17 @@ module Puma
       @parsed_bytes == 0
     end
 
+    def expect_proxy_proto=(val)
+      if val
+        if @read_header
+          @read_proxy = true
+        end
+      else
+        @read_proxy = false
+      end
+      @expect_proxy_proto = val
+    end
+
     private
 
     def setup_body
@@ -261,16 +327,27 @@ module Puma
       body = @parser.body
 
       te = @env[TRANSFER_ENCODING2]
-
       if te
-        if te.include?(",")
-          te.split(",").each do |part|
-            if CHUNKED.casecmp(part.strip) == 0
-              return setup_chunked_body(body)
-            end
+        te_lwr = te.downcase
+        if te.include? ','
+          te_ary = te_lwr.split ','
+          te_count = te_ary.count CHUNKED
+          te_valid = te_ary[0..-2].all? { |e| ALLOWED_TRANSFER_ENCODING.include? e }
+          if te_ary.last == CHUNKED && te_count == 1 && te_valid
+            @env.delete TRANSFER_ENCODING2
+            return setup_chunked_body body
+          elsif te_count >= 1
+            raise HttpParserError   , "#{TE_ERR_MSG}, multiple chunked: '#{te}'"
+          elsif !te_valid
+            raise HttpParserError501, "#{TE_ERR_MSG}, unknown value: '#{te}'"
           end
-        elsif CHUNKED.casecmp(te) == 0
-          return setup_chunked_body(body)
+        elsif te_lwr == CHUNKED
+          @env.delete TRANSFER_ENCODING2
+          return setup_chunked_body body
+        elsif ALLOWED_TRANSFER_ENCODING.include? te_lwr
+          raise HttpParserError     , "#{TE_ERR_MSG}, single value must be chunked: '#{te}'"
+        else
+          raise HttpParserError501  , "#{TE_ERR_MSG}, unknown value: '#{te}'"
         end
       end
 
@@ -278,7 +355,12 @@ module Puma
 
       cl = @env[CONTENT_LENGTH]
 
-      unless cl
+      if cl
+        # cannot contain characters that are not \d, or be empty
+        if cl =~ CONTENT_LENGTH_VALUE_INVALID || cl.empty?
+          raise HttpParserError, "Invalid Content-Length: #{cl.inspect}"
+        end
+      else
         @buffer = body.empty? ? nil : body
         @body = EmptyBody
         set_ready
@@ -309,7 +391,7 @@ module Puma
 
       @body_remain = remain
 
-      return false
+      false
     end
 
     def read_body
@@ -386,6 +468,7 @@ module Puma
       @chunked_body = true
       @partial_part_left = 0
       @prev_chunk = ""
+      @excess_cr = 0
 
       @body = Tempfile.new(Const::PUMA_TMP_BASE)
       @body.unlink
@@ -436,25 +519,51 @@ module Puma
 
       while !io.eof?
         line = io.gets
-        if line.end_with?("\r\n")
-          len = line.strip.to_i(16)
+        if line.end_with?(CHUNK_VALID_ENDING)
+          # Puma doesn't process chunk extensions, but should parse if they're
+          # present, which is the reason for the semicolon regex
+          chunk_hex = line.strip[/\A[^;]+/]
+          if chunk_hex =~ CHUNK_SIZE_INVALID
+            raise HttpParserError, "Invalid chunk size: '#{chunk_hex}'"
+          end
+          len = chunk_hex.to_i(16)
           if len == 0
             @in_last_chunk = true
             @body.rewind
             rest = io.read
-            last_crlf_size = "\r\n".bytesize
-            if rest.bytesize < last_crlf_size
+            if rest.bytesize < CHUNK_VALID_ENDING_SIZE
               @buffer = nil
-              @partial_part_left = last_crlf_size - rest.bytesize
+              @partial_part_left = CHUNK_VALID_ENDING_SIZE - rest.bytesize
               return false
             else
-              @buffer = rest[last_crlf_size..-1]
+              # if the next character is a CRLF, set buffer to everything after that CRLF
+              start_of_rest = if rest.start_with?(CHUNK_VALID_ENDING)
+                CHUNK_VALID_ENDING_SIZE
+              else # we have started a trailer section, which we do not support. skip it!
+                rest.index(CHUNK_VALID_ENDING*2) + CHUNK_VALID_ENDING_SIZE*2
+              end
+
+              @buffer = rest[start_of_rest..-1]
               @buffer = nil if @buffer.empty?
               set_ready
               return true
             end
           end
 
+          # Track the excess as a function of the size of the
+          # header vs the size of the actual data. Excess can
+          # go negative (and is expected to) when the body is
+          # significant.
+          # The additional of chunk_hex.size and 2 compensates
+          # for a client sending 1 byte in a chunked body over
+          # a long period of time, making sure that that client
+          # isn't accidentally eventually punished.
+          @excess_cr += (line.size - len - chunk_hex.size - 2)
+
+          if @excess_cr >= MAX_CHUNK_EXCESS
+            raise HttpParserError, "Maximum chunk excess detected"
+          end
+
           len += 2
 
           part = io.read(len)
@@ -468,7 +577,12 @@ module Puma
 
           case
           when got == len
-            write_chunk(part[0..-3]) # to skip the ending \r\n
+            # proper chunked segment must end with "\r\n"
+            if part.end_with? CHUNK_VALID_ENDING
+              write_chunk(part[0..-3]) # to skip the ending \r\n
+            else
+              raise HttpParserError, "Chunk size mismatch"
+            end
           when got <= len - 2
             write_chunk(part)
             @partial_part_left = len - part.size
@@ -477,6 +591,10 @@ module Puma
             @partial_part_left = len - part.size
           end
         else
+          if @prev_chunk.size + chunk.size >= MAX_CHUNK_HEADER_SIZE
+            raise HttpParserError, "maximum size of chunk header exceeded"
+          end
+
           @prev_chunk = line
           return false
         end

data/lib/puma/cluster/worker.rb

@@ -33,9 +33,9 @@ module Puma
         Signal.trap "SIGINT", "IGNORE"
         Signal.trap "SIGCHLD", "DEFAULT"
 
-       Thread.new do
-          Puma.set_thread_name "worker check pipe"
-          IO.select [@check_pipe]
+        Thread.new do
+          Puma.set_thread_name "wrkr check"
+          @check_pipe.wait_readable
           log "! Detected parent died, dying"
           exit! 1
         end
@@ -76,7 +76,7 @@ module Puma
           end
 
           Thread.new do
-            Puma.set_thread_name "worker fork pipe"
+            Puma.set_thread_name "wrkr fork"
             while (idx = @fork_pipe.gets)
               idx = idx.to_i
               if idx == -1 # stop server
@@ -106,7 +106,7 @@ module Puma
         begin
           @worker_write << "b#{Process.pid}:#{index}\n"
         rescue SystemCallError, IOError
-          Thread.current.purge_interrupt_queue if Thread.current.respond_to? :purge_interrupt_queue
+          Puma::Util.purge_interrupt_queue
           STDERR.puts "Master seems to have exited, exiting."
           return
         end
@@ -114,7 +114,7 @@ module Puma
         while restart_server.pop
           server_thread = server.run
           stat_thread ||= Thread.new(@worker_write) do |io|
-            Puma.set_thread_name "stat payload"
+            Puma.set_thread_name "stat pld"
             base_payload = "p#{Process.pid}"
 
             while true
@@ -127,10 +127,10 @@ module Puma
                 payload = %Q!#{base_payload}{ "backlog":#{b}, "running":#{r}, "pool_capacity":#{t}, "max_threads": #{m}, "requests_count": #{rc} }\n!
                 io << payload
               rescue IOError
-                Thread.current.purge_interrupt_queue if Thread.current.respond_to? :purge_interrupt_queue
+                Puma::Util.purge_interrupt_queue
                 break
               end
-              sleep Const::WORKER_CHECK_INTERVAL
+              sleep @options[:worker_check_interval]
             end
           end
           server_thread.join
@@ -168,16 +168,6 @@ module Puma
         @launcher.config.run_hooks :after_worker_fork, idx, @launcher.events
         pid
       end
-
-      def wakeup!
-        return unless @wakeup
-
-        begin
-          @wakeup.write "!" unless @wakeup.closed?
-        rescue SystemCallError, IOError
-          Thread.current.purge_interrupt_queue if Thread.current.respond_to? :purge_interrupt_queue
-        end
-      end
     end
   end
 end

data/lib/puma/cluster/worker_handle.rb

@@ -40,6 +40,10 @@ module Puma
         @stage = :booted
       end
 
+      def term!
+        @term = true
+      end
+
       def term?
         @term
       end

data/lib/puma/cluster.rb

@@ -108,24 +108,42 @@ module Puma
     def cull_workers
       diff = @workers.size - @options[:workers]
       return if diff < 1
+      debug "Culling #{diff} workers"
 
-      debug "Culling #{diff.inspect} workers"
+      workers = workers_to_cull(diff)
+      debug "Workers to cull: #{workers.inspect}"
 
-      workers_to_cull = @workers[-diff,diff]
-      debug "Workers to cull: #{workers_to_cull.inspect}"
-
-      workers_to_cull.each do |worker|
+      workers.each do |worker|
         log "- Worker #{worker.index} (PID: #{worker.pid}) terminating"
         worker.term
       end
     end
 
+    def workers_to_cull(diff)
+      workers = @workers.sort_by(&:started_at)
+
+      # In fork_worker mode, worker 0 acts as our master process.
+      # We should avoid culling it to preserve copy-on-write memory gains.
+      workers.reject! { |w| w.index == 0 } if @options[:fork_worker]
+
+      workers[cull_start_index(diff), diff]
+    end
+
+    def cull_start_index(diff)
+      case @options[:worker_culling_strategy]
+      when :oldest
+        0
+      else # :youngest
+        -diff
+      end
+    end
+
     # @!attribute [r] next_worker_index
     def next_worker_index
-      all_positions =  0...@options[:workers]
-      occupied_positions = @workers.map { |w| w.index }
-      available_positions = all_positions.to_a - occupied_positions
-      available_positions.first
+      occupied_positions = @workers.map(&:index)
+      idx = 0
+      idx += 1 until !occupied_positions.include?(idx)
+      idx
     end
 
     def all_workers_booted?
@@ -135,7 +153,7 @@ module Puma
     def check_workers
       return if @next_check >= Time.now
 
-      @next_check = Time.now + Const::WORKER_CHECK_INTERVAL
+      @next_check = Time.now + @options[:worker_check_interval]
 
       timeout_workers
       wait_workers
@@ -164,16 +182,6 @@ module Puma
       ].compact.min
     end
 
-    def wakeup!
-      return unless @wakeup
-
-      begin
-        @wakeup.write "!" unless @wakeup.closed?
-      rescue SystemCallError, IOError
-        Thread.current.purge_interrupt_queue if Thread.current.respond_to? :purge_interrupt_queue
-      end
-    end
-
     def worker(index, master)
       @workers = []
 
@@ -426,9 +434,7 @@ module Puma
 
             check_workers
 
-            res = IO.select([read], nil, nil, [0, @next_check - Time.now].max)
-
-            if res
+            if read.wait_readable([0, @next_check - Time.now].max)
               req = read.read_nonblock(1)
 
               @next_check = Time.now if req == "!"
@@ -452,7 +458,7 @@ module Puma
                   workers_not_booted -= 1
                 when "e"
                   # external term, see worker method, Signal.trap "SIGTERM"
-                  w.instance_variable_set :@term, true
+                  w.term!
                 when "t"
                   w.term unless w.term?
                 when "p"

data/lib/puma/configuration.rb

@@ -11,6 +11,7 @@ module Puma
 
     DefaultTCPHost = "0.0.0.0"
     DefaultTCPPort = 9292
+    DefaultWorkerCheckInterval = 5
     DefaultWorkerTimeout = 60
     DefaultWorkerShutdownTimeout = 30
   end
@@ -195,12 +196,14 @@ module Puma
         :workers => Integer(ENV['WEB_CONCURRENCY'] || 0),
         :silence_single_worker_warning => false,
         :mode => :http,
+        :worker_check_interval => DefaultWorkerCheckInterval,
         :worker_timeout => DefaultWorkerTimeout,
         :worker_boot_timeout => DefaultWorkerTimeout,
         :worker_shutdown_timeout => DefaultWorkerShutdownTimeout,
+        :worker_culling_strategy => :youngest,
         :remote_address => :socket,
         :tag => method(:infer_tag),
-        :environment => -> { ENV['RACK_ENV'] || ENV['RAILS_ENV'] || "development" },
+        :environment => -> { ENV['APP_ENV'] || ENV['RACK_ENV'] || ENV['RAILS_ENV'] || 'development' },
         :rackup => DefaultRackup,
         :logger => STDOUT,
         :persistent_timeout => Const::PERSISTENT_TIMEOUT,

data/lib/puma/const.rb

@@ -76,7 +76,7 @@ module Puma
     508 => 'Loop Detected',
     510 => 'Not Extended',
     511 => 'Network Authentication Required'
-  }
+  }.freeze
 
   # For some HTTP status codes the client only expects headers.
   #
@@ -85,7 +85,7 @@ module Puma
     204 => true,
     205 => true,
     304 => true
-  }
+  }.freeze
 
   # Frequently used constants when constructing requests or responses.  Many times
   # the constant just refers to a string with the same contents.  Using these constants
@@ -100,8 +100,8 @@ module Puma
   # too taxing on performance.
   module Const
 
-    PUMA_VERSION = VERSION = "5.3.2".freeze
-    CODE_NAME = "Sweetnighter".freeze
+    PUMA_VERSION = VERSION = "5.6.8".freeze
+    CODE_NAME = "Birdie's Version".freeze
 
     PUMA_SERVER_STRING = ['puma', PUMA_VERSION, CODE_NAME].join(' ').freeze
 
@@ -145,9 +145,11 @@ module Puma
       408 => "HTTP/1.1 408 Request Timeout\r\nConnection: close\r\nServer: Puma #{PUMA_VERSION}\r\n\r\n".freeze,
       # Indicate that there was an internal error, obviously.
       500 => "HTTP/1.1 500 Internal Server Error\r\n\r\n".freeze,
+      # Incorrect or invalid header value
+      501 => "HTTP/1.1 501 Not Implemented\r\n\r\n".freeze,
       # A common header for indicating the server is too busy.  Not used yet.
       503 => "HTTP/1.1 503 Service Unavailable\r\n\r\nBUSY".freeze
-    }
+    }.freeze
 
     # The basic max request size we'll try to read.
     CHUNK_SIZE = 16 * 1024
@@ -235,9 +237,6 @@ module Puma
 
     EARLY_HINTS = "rack.early_hints".freeze
 
-    # Minimum interval to checks worker health
-    WORKER_CHECK_INTERVAL = 5
-
     # Illegal character in the key or value of response header
     DQUOTE = "\"".freeze
     HTTP_HEADER_DELIMITER = Regexp.escape("(),/:;<=>?@[]{}\\").freeze
@@ -247,5 +246,7 @@ module Puma
 
     # Banned keys of response header
     BANNED_HEADER_KEY = /\A(rack\.|status\z)/.freeze
+
+    PROXY_PROTOCOL_V1_REGEX = /^PROXY (?:TCP4|TCP6|UNKNOWN) ([^\r]+)\r\n/.freeze
   end
 end

data/lib/puma/control_cli.rb

@@ -17,26 +17,30 @@ module Puma
     CMD_PATH_SIG_MAP = {
       'gc'       => nil,
       'gc-stats' => nil,
-      'halt'     => 'SIGQUIT',
-      'phased-restart' => 'SIGUSR1',
-      'refork'   => 'SIGURG',
+      'halt'              => 'SIGQUIT',
+      'info'              => 'SIGINFO',
+      'phased-restart'    => 'SIGUSR1',
+      'refork'            => 'SIGURG',
       'reload-worker-directory' => nil,
-      'restart'  => 'SIGUSR2',
+      'reopen-log'        => 'SIGHUP',
+      'restart'           => 'SIGUSR2',
       'start'    => nil,
       'stats'    => nil,
       'status'   => '',
-      'stop'     => 'SIGTERM',
-      'thread-backtraces' => nil
+      'stop'              => 'SIGTERM',
+      'thread-backtraces' => nil,
+      'worker-count-down' => 'SIGTTOU',
+      'worker-count-up'   => 'SIGTTIN'
     }.freeze
 
     # @deprecated 6.0.0
     COMMANDS = CMD_PATH_SIG_MAP.keys.freeze
 
     # commands that cannot be used in a request
-    NO_REQ_COMMANDS = %w{refork}.freeze
+    NO_REQ_COMMANDS = %w[info reopen-log worker-count-down worker-count-up].freeze
 
     # @version 5.0.0
-    PRINTABLE_COMMANDS = %w{gc-stats stats thread-backtraces}.freeze
+    PRINTABLE_COMMANDS = %w[gc-stats stats thread-backtraces].freeze
 
     def initialize(argv, stdout=STDOUT, stderr=STDERR)
       @state = nil
@@ -47,7 +51,7 @@ module Puma
       @control_auth_token = nil
       @config_file = nil
       @command = nil
-      @environment = ENV['RACK_ENV'] || ENV['RAILS_ENV']
+      @environment = ENV['APP_ENV'] || ENV['RACK_ENV'] || ENV['RAILS_ENV']
 
       @argv = argv.dup
       @stdout = stdout
@@ -185,8 +189,6 @@ module Puma
 
       if @command == 'status'
         message 'Puma is started'
-      elsif NO_REQ_COMMANDS.include? @command
-        raise "Invalid request command: #{@command}"
       else
         url = "/#{@command}"
 
@@ -242,7 +244,11 @@ module Puma
           @stdout.flush unless @stdout.sync
           return
         elsif sig.start_with? 'SIG'
-          Process.kill sig, @pid
+          if Signal.list.key? sig.sub(/\ASIG/, '')
+            Process.kill sig, @pid
+          else
+            raise "Signal '#{sig}' not available'"
+          end
         elsif @command == 'status'
           begin
             Process.kill 0, @pid
@@ -268,7 +274,7 @@ module Puma
       return start if @command == 'start'
       prepare_configuration
 
-      if Puma.windows? || @control_url
+      if Puma.windows? || @control_url && !NO_REQ_COMMANDS.include?(@command)
         send_request
       else
         send_signal

data/lib/puma/detect.rb

@@ -10,8 +10,10 @@ module Puma
 
   IS_JRUBY = Object.const_defined? :JRUBY_VERSION
 
-  IS_WINDOWS = !!(RUBY_PLATFORM =~ /mswin|ming|cygwin/ ||
-    IS_JRUBY && RUBY_DESCRIPTION =~ /mswin/)
+  IS_OSX = RUBY_PLATFORM.include? 'darwin'
+
+  IS_WINDOWS = !!(RUBY_PLATFORM =~ /mswin|ming|cygwin/) ||
+    IS_JRUBY && RUBY_DESCRIPTION.include?('mswin')
 
   # @version 5.2.0
   IS_MRI = (RUBY_ENGINE == 'ruby' || RUBY_ENGINE.nil?)
@@ -20,6 +22,10 @@ module Puma
     IS_JRUBY
   end
 
+  def self.osx?
+    IS_OSX
+  end
+
   def self.windows?
     IS_WINDOWS
   end