rack 3.0.0 → 3.1.0

data/lib/rack/multipart/parser.rb

@@ -3,49 +3,46 @@
 require 'strscan'
 
 require_relative '../utils'
+require_relative '../bad_request'
 
 module Rack
   module Multipart
-    class MultipartPartLimitError < Errno::EMFILE; end
+    class MultipartPartLimitError < Errno::EMFILE
+      include BadRequest
+    end
+
+    class MultipartTotalPartLimitError < StandardError
+      include BadRequest
+    end
 
     # Use specific error class when parsing multipart request
     # that ends early.
-    class EmptyContentError < ::EOFError; end
+    class EmptyContentError < ::EOFError
+      include BadRequest
+    end
 
     # Base class for multipart exceptions that do not subclass from
     # other exception classes for backwards compatibility.
-    class Error < StandardError; end
+    class BoundaryTooLongError < StandardError
+      include BadRequest
+    end
+
+    # Prefer to use the BoundaryTooLongError class or Rack::BadRequest.
+    Error = BoundaryTooLongError
 
     EOL = "\r\n"
     MULTIPART = %r|\Amultipart/.*boundary=\"?([^\";,]+)\"?|ni
-    TOKEN = /[^\s()<>,;:\\"\/\[\]?=]+/
-    CONDISP = /Content-Disposition:\s*#{TOKEN}\s*/i
-    VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/
-    BROKEN = /^#{CONDISP}.*;\s*filename=(#{VALUE})/i
     MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{EOL}/ni
-    MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni
+    MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:(.*)(?=#{EOL}(\S|\z))/ni
     MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{EOL}]*)/ni
-    # Updated definitions from RFC 2231
-    ATTRIBUTE_CHAR = %r{[^ \t\v\n\r)(><@,;:\\"/\[\]?='*%]}
-    ATTRIBUTE = /#{ATTRIBUTE_CHAR}+/
-    SECTION = /\*[0-9]+/
-    REGULAR_PARAMETER_NAME = /#{ATTRIBUTE}#{SECTION}?/
-    REGULAR_PARAMETER = /(#{REGULAR_PARAMETER_NAME})=(#{VALUE})/
-    EXTENDED_OTHER_NAME = /#{ATTRIBUTE}\*[1-9][0-9]*\*/
-    EXTENDED_OTHER_VALUE = /%[0-9a-fA-F]{2}|#{ATTRIBUTE_CHAR}/
-    EXTENDED_OTHER_PARAMETER = /(#{EXTENDED_OTHER_NAME})=(#{EXTENDED_OTHER_VALUE}*)/
-    EXTENDED_INITIAL_NAME = /#{ATTRIBUTE}(?:\*0)?\*/
-    EXTENDED_INITIAL_VALUE = /[a-zA-Z0-9\-]*'[a-zA-Z0-9\-]*'#{EXTENDED_OTHER_VALUE}*/
-    EXTENDED_INITIAL_PARAMETER = /(#{EXTENDED_INITIAL_NAME})=(#{EXTENDED_INITIAL_VALUE})/
-    EXTENDED_PARAMETER = /#{EXTENDED_INITIAL_PARAMETER}|#{EXTENDED_OTHER_PARAMETER}/
-    DISPPARM = /;\s*(?:#{REGULAR_PARAMETER}|#{EXTENDED_PARAMETER})\s*/
-    RFC2183 = /^#{CONDISP}(#{DISPPARM})+$/i
 
     class Parser
       BUFSIZE = 1_048_576
       TEXT_PLAIN = "text/plain"
       TEMPFILE_FACTORY = lambda { |filename, content_type|
-        Tempfile.new(["RackMultipart", ::File.extname(filename.gsub("\0", '%00'))])
+        extension = ::File.extname(filename.gsub("\0", '%00'))[0, 129]
+
+        Tempfile.new(["RackMultipart", extension])
       }
 
       class BoundedIO # :nodoc:
@@ -96,7 +93,7 @@ module Rack
         if boundary.length > 70
           # RFC 1521 Section 7.2.1 imposes a 70 character maximum for the boundary.
           # Most clients use no more than 55 characters.
-          raise Error, "multipart boundary size too large (#{boundary.length} characters)"
+          raise BoundaryTooLongError, "multipart boundary size too large (#{boundary.length} characters)"
         end
 
         io = BoundedIO.new(io, content_length) if content_length
@@ -166,7 +163,7 @@ module Rack
 
           @mime_parts[mime_index] = klass.new(body, head, filename, content_type, name)
 
-          check_open_files
+          check_part_limits
         end
 
         def on_mime_body(mime_index, content)
@@ -178,13 +175,23 @@ module Rack
 
         private
 
-        def check_open_files
-          if Utils.multipart_part_limit > 0
-            if @open_files >= Utils.multipart_part_limit
+        def check_part_limits
+          file_limit = Utils.multipart_file_limit
+          part_limit = Utils.multipart_total_part_limit
+
+          if file_limit && file_limit > 0
+            if @open_files >= file_limit
               @mime_parts.each(&:close)
               raise MultipartPartLimitError, 'Maximum file multiparts in content reached'
             end
           end
+
+          if part_limit && part_limit > 0
+            if @mime_parts.size >= part_limit
+              @mime_parts.each(&:close)
+              raise MultipartTotalPartLimitError, 'Maximum total multiparts in content reached'
+            end
+          end
         end
       end
 
@@ -201,6 +208,8 @@ module Rack
 
         @sbuf = StringScanner.new("".dup)
         @body_regex = /(?:#{EOL}|\A)--#{Regexp.quote(boundary)}(?:#{EOL}|--)/m
+        @body_regex_at_end = /#{@body_regex}\z/m
+        @end_boundary_size = boundary.bytesize + 4 # (-- at start, -- at finish)
         @rx_max_size = boundary.bytesize + 6 # (\r\n-- at start, either \r\n or -- at finish)
         @head_regex = /(.*?#{EOL})#{EOL}/m
       end
@@ -267,7 +276,14 @@ module Rack
             @state = :MIME_HEAD
             return
           when :END_BOUNDARY
-            # invalid multipart upload, but retry for opening boundary
+            # invalid multipart upload
+            if @sbuf.pos == @end_boundary_size && @sbuf.rest == EOL
+              # stop parsing a buffer if a buffer is only an end boundary.
+              @state = :DONE
+              return
+            end
+
+            # retry for opening boundary
           else
             # no boundary found, keep reading data
             return :want_read
@@ -285,17 +301,102 @@ module Rack
         end
       end
 
+      CONTENT_DISPOSITION_MAX_PARAMS = 16
+      CONTENT_DISPOSITION_MAX_BYTES = 1536
       def handle_mime_head
         if @sbuf.scan_until(@head_regex)
           head = @sbuf[1]
           content_type = head[MULTIPART_CONTENT_TYPE, 1]
-          if name = head[MULTIPART_CONTENT_DISPOSITION, 1]
-            name = dequote(name)
+          if (disposition = head[MULTIPART_CONTENT_DISPOSITION, 1]) &&
+              disposition.bytesize <= CONTENT_DISPOSITION_MAX_BYTES
+
+            # ignore actual content-disposition value (should always be form-data)
+            i = disposition.index(';')
+            disposition.slice!(0, i+1)
+            param = nil
+            num_params = 0
+
+            # Parse parameter list
+            while i = disposition.index('=')
+              # Only parse up to max parameters, to avoid potential denial of service
+              num_params += 1
+              break if num_params > CONTENT_DISPOSITION_MAX_PARAMS
+
+              # Found end of parameter name, ensure forward progress in loop
+              param = disposition.slice!(0, i+1)
+
+              # Remove ending equals and preceding whitespace from parameter name
+              param.chomp!('=')
+              param.lstrip!
+
+              if disposition[0] == '"'
+                # Parameter value is quoted, parse it, handling backslash escapes
+                disposition.slice!(0, 1)
+                value = String.new
+
+                while i = disposition.index(/(["\\])/)
+                  c = $1
+
+                  # Append all content until ending quote or escape
+                  value << disposition.slice!(0, i)
+
+                  # Remove either backslash or ending quote,
+                  # ensures forward progress in loop
+                  disposition.slice!(0, 1)
+
+                  # stop parsing parameter value if found ending quote
+                  break if c == '"'
+
+                  escaped_char = disposition.slice!(0, 1)
+                  if param == 'filename' && escaped_char != '"'
+                    # Possible IE uploaded filename, append both escape backslash and value
+                    value << c << escaped_char
+                  else
+                    # Other only append escaped value
+                    value << escaped_char
+                  end
+                end
+              else
+                if i = disposition.index(';')
+                  # Parameter value unquoted (which may be invalid), value ends at semicolon
+                  value = disposition.slice!(0, i)
+                else
+                  # If no ending semicolon, assume remainder of line is value and stop
+                  # parsing
+                  disposition.strip!
+                  value = disposition
+                  disposition = ''
+                end
+              end
+
+              case param
+              when 'name'
+                name = value
+              when 'filename'
+                filename = value
+              when 'filename*'
+                filename_star = value
+              # else
+              # ignore other parameters
+              end
+
+              # skip trailing semicolon, to proceed to next parameter
+              if i = disposition.index(';')
+                disposition.slice!(0, i+1)
+              end
+            end
           else
             name = head[MULTIPART_CONTENT_ID, 1]
           end
 
-          filename = get_filename(head)
+          if filename_star
+            encoding, _, filename = filename_star.split("'", 3)
+            filename = normalize_filename(filename || '')
+            filename.force_encoding(find_encoding(encoding))
+          elsif filename
+            filename = $1 if filename =~ /^"(.*)"$/
+            filename = normalize_filename(filename)
+          end
 
           if name.nil? || name.empty?
             name = filename || "#{content_type || TEXT_PLAIN}[]".dup
@@ -310,7 +411,7 @@ module Rack
 
       def handle_mime_body
         if (body_with_boundary = @sbuf.check_until(@body_regex)) # check but do not advance the pointer yet
-          body = body_with_boundary.sub(/#{@body_regex}\z/m, '') # remove the boundary from the string
+          body = body_with_boundary.sub(@body_regex_at_end, '') # remove the boundary from the string
           @collector.on_mime_body @mime_index, body
           @sbuf.pos += body.length + 2 # skip \r\n after the content
           @state = :CONSUME_TOKEN
@@ -340,39 +441,14 @@ module Rack
         end
       end
 
-      def get_filename(head)
-        filename = nil
-        case head
-        when RFC2183
-          params = Hash[*head.scan(DISPPARM).flat_map(&:compact)]
-
-          if filename = params['filename*']
-            encoding, _, filename = filename.split("'", 3)
-          elsif filename = params['filename']
-            filename = $1 if filename =~ /^"(.*)"$/
-          end
-        when BROKEN
-          filename = $1
-          filename = $1 if filename =~ /^"(.*)"$/
-        end
-
-        return unless filename
-
+      def normalize_filename(filename)
         if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
           filename = Utils.unescape_path(filename)
         end
 
         filename.scrub!
 
-        if filename !~ /\\[^\\"]/
-          filename = filename.gsub(/\\(.)/, '\1')
-        end
-
-        if encoding
-          filename.force_encoding ::Encoding.find(encoding)
-        end
-
-        filename
+        filename.split(/[\/\\]/).last || String.new
       end
 
       CHARSET = "charset"
@@ -398,11 +474,7 @@ module Rack
               v.strip!
               v = v[1..-2] if v.start_with?('"') && v.end_with?('"')
               if k == "charset"
-                encoding = begin
-                  Encoding.find v
-                rescue ArgumentError
-                  Encoding::BINARY
-                end
+                encoding = find_encoding(v)
               end
             end
           end
@@ -412,6 +484,15 @@ module Rack
         body.force_encoding(encoding)
       end
 
+      # Return the related Encoding object. However, because
+      # enc is submitted by the user, it may be invalid, so
+      # use a binary encoding in that case.
+      def find_encoding(enc)
+        Encoding.find enc
+      rescue ArgumentError
+        Encoding::BINARY
+      end
+
       def handle_empty_content!(content)
         if content.nil? || content.empty?
           raise EmptyContentError

data/lib/rack/multipart.rb

@@ -6,6 +6,8 @@ require_relative 'utils'
 require_relative 'multipart/parser'
 require_relative 'multipart/generator'
 
+require_relative 'bad_request'
+
 module Rack
   # A multipart form data parser, adapted from IOWA.
   #
@@ -13,9 +15,40 @@ module Rack
   module Multipart
     MULTIPART_BOUNDARY = "AaB03x"
 
+    class MissingInputError < StandardError
+      include BadRequest
+    end
+
+    # Accumulator for multipart form data, conforming to the QueryParser API.
+    # In future, the Parser could return the pair list directly, but that would
+    # change its API.
+    class ParamList # :nodoc:
+      def self.make_params
+        new
+      end
+
+      def self.normalize_params(params, key, value)
+        params << [key, value]
+      end
+
+      def initialize
+        @pairs = []
+      end
+
+      def <<(pair)
+        @pairs << pair
+      end
+
+      def to_params_hash
+        @pairs
+      end
+    end
+
     class << self
       def parse_multipart(env, params = Rack::Utils.default_query_parser)
-        io = env[RACK_INPUT]
+        unless io = env[RACK_INPUT]
+          raise MissingInputError, "Missing input stream!"
+        end
 
         if content_length = env['CONTENT_LENGTH']
           content_length = content_length.to_i

data/lib/rack/query_parser.rb

@@ -1,38 +1,39 @@
 # frozen_string_literal: true
 
+require_relative 'bad_request'
+require 'uri'
+
 module Rack
   class QueryParser
-    DEFAULT_SEP = /[&] */n
-    COMMON_SEP = { ";" => /[;] */n, ";," => /[;,] */n, "&" => /[&] */n }
+    DEFAULT_SEP = /& */n
+    COMMON_SEP = { ";" => /; */n, ";," => /[;,] */n, "&" => /& */n }
 
     # ParameterTypeError is the error that is raised when incoming structural
     # parameters (parsed by parse_nested_query) contain conflicting types.
-    class ParameterTypeError < TypeError; end
+    class ParameterTypeError < TypeError
+      include BadRequest
+    end
 
     # InvalidParameterError is the error that is raised when incoming structural
     # parameters (parsed by parse_nested_query) contain invalid format or byte
     # sequence.
-    class InvalidParameterError < ArgumentError; end
+    class InvalidParameterError < ArgumentError
+      include BadRequest
+    end
 
     # ParamsTooDeepError is the error that is raised when params are recursively
     # nested over the specified limit.
-    class ParamsTooDeepError < RangeError; end
-
-    def self.make_default(_key_space_limit=(not_deprecated = true; nil), param_depth_limit)
-      unless not_deprecated
-        warn("`first argument `key_space limit` is deprecated and no longer has an effect. Please call with only one argument, which will be required in a future version of Rack", uplevel: 1)
-      end
+    class ParamsTooDeepError < RangeError
+      include BadRequest
+    end
 
+    def self.make_default(param_depth_limit)
       new Params, param_depth_limit
     end
 
     attr_reader :param_depth_limit
 
-    def initialize(params_class, _key_space_limit=(not_deprecated = true; nil), param_depth_limit)
-      unless not_deprecated
-        warn("`second argument `key_space limit` is deprecated and no longer has an effect. Please call with only two arguments, which will be required in a future version of Rack", uplevel: 1)
-      end
-
+    def initialize(params_class, param_depth_limit)
       @params_class = params_class
       @param_depth_limit = param_depth_limit
     end
@@ -128,8 +129,6 @@ module Rack
 
       return if k.empty?
 
-      v ||= String.new
-
       if after == ''
         if k == '[]' && depth != 0
           return [v]
@@ -190,63 +189,11 @@ module Rack
       true
     end
 
-    def unescape(s)
-      Utils.unescape(s)
+    def unescape(string, encoding = Encoding::UTF_8)
+      URI.decode_www_form_component(string, encoding)
     end
 
-    class Params
-      def initialize
-        @size   = 0
-        @params = {}
-      end
-
-      def [](key)
-        @params[key]
-      end
-
-      def []=(key, value)
-        @params[key] = value
-      end
-
-      def key?(key)
-        @params.key?(key)
-      end
-
-      # Recursively unwraps nested `Params` objects and constructs an object
-      # of the same shape, but using the objects' internal representations
-      # (Ruby hashes) in place of the objects. The result is a hash consisting
-      # purely of Ruby primitives.
-      #
-      #   Mutation warning!
-      #
-      #   1. This method mutates the internal representation of the `Params`
-      #      objects in order to save object allocations.
-      #
-      #   2. The value you get back is a reference to the internal hash
-      #      representation, not a copy.
-      #
-      #   3. Because the `Params` object's internal representation is mutable
-      #      through the `#[]=` method, it is not thread safe. The result of
-      #      getting the hash representation while another thread is adding a
-      #      key to it is non-deterministic.
-      #
-      def to_h
-        @params.each do |key, value|
-          case value
-          when self
-            # Handle circular references gracefully.
-            @params[key] = @params
-          when Params
-            @params[key] = value.to_h
-          when Array
-            value.map! { |v| v.kind_of?(Params) ? v.to_h : v }
-          else
-            # Ignore anything that is not a `Params` object or
-            # a collection that can contain one.
-          end
-        end
-        @params
-      end
+    class Params < Hash
       alias_method :to_params_hash, :to_h
     end
   end

data/lib/rack/request.rb

@@ -44,7 +44,7 @@ module Rack
     @x_forwarded_proto_priority = [:proto, :scheme]
 
     valid_ipv4_octet = /\.(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])/
-    
+
     trusted_proxies = Regexp.union(
       /\A127#{valid_ipv4_octet}{3}\z/,                          # localhost IPv4 range 127.x.x.x, per RFC-3330
       /\A::1\z/,                                                # localhost IPv6 ::1
@@ -54,7 +54,7 @@ module Rack
       /\A192\.168#{valid_ipv4_octet}{2}\z/,                     # private IPv4 range 192.168.x.x
       /\Alocalhost\z|\Aunix(\z|:)/i,                            # localhost hostname, and unix domain sockets
     )
-    
+
     self.ip_filter = lambda { |ip| trusted_proxies.match?(ip) }
 
     ALLOWED_SCHEMES = %w(https http wss ws).freeze
@@ -482,9 +482,14 @@ module Rack
 
       # Returns the data received in the query string.
       def GET
-        if get_header(RACK_REQUEST_QUERY_STRING) == query_string
+        rr_query_string = get_header(RACK_REQUEST_QUERY_STRING)
+        query_string = self.query_string
+        if rr_query_string == query_string
           get_header(RACK_REQUEST_QUERY_HASH)
         else
+          if rr_query_string
+            warn "query string used for GET parsing different from current query string. Starting in Rack 3.2, Rack will used the cached GET value instead of parsing the current query string.", uplevel: 1
+          end
           query_hash = parse_query(query_string, '&')
           set_header(RACK_REQUEST_QUERY_STRING, query_string)
           set_header(RACK_REQUEST_QUERY_HASH, query_hash)
@@ -496,26 +501,52 @@ module Rack
       # This method support both application/x-www-form-urlencoded and
       # multipart/form-data.
       def POST
-        if get_header(RACK_INPUT).nil?
-          raise "Missing rack.input"
-        elsif get_header(RACK_REQUEST_FORM_INPUT) == get_header(RACK_INPUT)
-          get_header(RACK_REQUEST_FORM_HASH)
-        elsif form_data? || parseable_data?
-          unless set_header(RACK_REQUEST_FORM_HASH, parse_multipart)
-            form_vars = get_header(RACK_INPUT).read
-
-            # Fix for Safari Ajax postings that always append \0
-            # form_vars.sub!(/\0\z/, '') # performance replacement:
-            form_vars.slice!(-1) if form_vars.end_with?("\0")
-
-            set_header RACK_REQUEST_FORM_VARS, form_vars
-            set_header RACK_REQUEST_FORM_HASH, parse_query(form_vars, '&')
+        if error = get_header(RACK_REQUEST_FORM_ERROR)
+          raise error.class, error.message, cause: error.cause
+        end
+
+        begin
+          rack_input = get_header(RACK_INPUT)
+
+          # If the form hash was already memoized:
+          if form_hash = get_header(RACK_REQUEST_FORM_HASH)
+            form_input = get_header(RACK_REQUEST_FORM_INPUT)
+            # And it was memoized from the same input:
+            if form_input.equal?(rack_input)
+              return form_hash
+            elsif form_input
+              warn "input stream used for POST parsing different from current input stream. Starting in Rack 3.2, Rack will used the cached POST value instead of parsing the current input stream.", uplevel: 1
+            end
           end
-          set_header RACK_REQUEST_FORM_INPUT, get_header(RACK_INPUT)
-          get_header RACK_REQUEST_FORM_HASH
-        else
-          set_header RACK_REQUEST_FORM_INPUT, get_header(RACK_INPUT)
-          set_header(RACK_REQUEST_FORM_HASH, {})
+
+          # Otherwise, figure out how to parse the input:
+          if rack_input.nil?
+            set_header RACK_REQUEST_FORM_INPUT, nil
+            set_header(RACK_REQUEST_FORM_HASH, {})
+          elsif form_data? || parseable_data?
+            if pairs = Rack::Multipart.parse_multipart(env, Rack::Multipart::ParamList)
+              set_header RACK_REQUEST_FORM_PAIRS, pairs
+              set_header RACK_REQUEST_FORM_HASH, expand_param_pairs(pairs)
+            else
+              form_vars = get_header(RACK_INPUT).read
+
+              # Fix for Safari Ajax postings that always append \0
+              # form_vars.sub!(/\0\z/, '') # performance replacement:
+              form_vars.slice!(-1) if form_vars.end_with?("\0")
+
+              set_header RACK_REQUEST_FORM_VARS, form_vars
+              set_header RACK_REQUEST_FORM_HASH, parse_query(form_vars, '&')
+            end
+
+            set_header RACK_REQUEST_FORM_INPUT, get_header(RACK_INPUT)
+            get_header RACK_REQUEST_FORM_HASH
+          else
+            set_header RACK_REQUEST_FORM_INPUT, get_header(RACK_INPUT)
+            set_header(RACK_REQUEST_FORM_HASH, {})
+          end
+        rescue => error
+          set_header(RACK_REQUEST_FORM_ERROR, error)
+          raise
         end
       end
 
@@ -585,24 +616,10 @@ module Rack
         Rack::Request.ip_filter.call(ip)
       end
 
-      # shortcut for <tt>request.params[key]</tt>
-      def [](key)
-        warn("Request#[] is deprecated and will be removed in a future version of Rack. Please use request.params[] instead", uplevel: 1)
-
-        params[key.to_s]
-      end
-
-      # shortcut for <tt>request.params[key] = value</tt>
-      #
-      # Note that modifications will not be persisted in the env. Use update_param or delete_param if you want to destructively modify params.
-      def []=(key, value)
-        warn("Request#[]= is deprecated and will be removed in a future version of Rack. Please use request.params[]= instead", uplevel: 1)
-
-        params[key.to_s] = value
-      end
-
       # like Hash#values_at
       def values_at(*keys)
+        warn("Request#values_at is deprecated and will be removed in a future version of Rack. Please use request.params.values_at instead", uplevel: 1)
+        
         keys.map { |key| params[key] }
       end
 
@@ -652,6 +669,16 @@ module Rack
         Rack::Multipart.extract_multipart(self, query_parser)
       end
 
+      def expand_param_pairs(pairs, query_parser = query_parser())
+        params = query_parser.make_params
+
+        pairs.each do |k, v|
+          query_parser.normalize_params(params, k, v)
+        end
+
+        params.to_params_hash
+      end
+
       def split_header(value)
         value ? value.strip.split(/[,\s]+/) : []
       end