http 5.3.1 → 6.0.0

data/lib/http/redirector.rb

@@ -1,10 +1,9 @@
 # frozen_string_literal: true
 
-require "set"
-
 require "http/headers"
 
 module HTTP
+  # Follows HTTP redirects according to configured policy
   class Redirector
     # Notifies that we reached max allowed redirect hops
     class TooManyRedirectsError < ResponseError; end
@@ -26,117 +25,123 @@ module HTTP
     # Verbs which will remain unchanged upon See Other response.
     SEE_OTHER_ALLOWED_VERBS = %i[get head].to_set.freeze
 
-    # @!attribute [r] strict
-    #   Returns redirector policy.
-    #   @return [Boolean]
+    # Returns redirector policy
+    #
+    # @example
+    #   redirector.strict # => true
+    #
+    # @return [Boolean]
+    # @api public
     attr_reader :strict
 
-    # @!attribute [r] max_hops
-    #   Returns maximum allowed hops.
-    #   @return [Fixnum]
+    # Returns maximum allowed hops
+    #
+    # @example
+    #   redirector.max_hops # => 5
+    #
+    # @return [Fixnum]
+    # @api public
     attr_reader :max_hops
 
-    # @param [Hash] opts
-    # @option opts [Boolean] :strict (true) redirector hops policy
-    # @option opts [#to_i] :max_hops (5) maximum allowed amount of hops
-    def initialize(opts = {})
-      @strict      = opts.fetch(:strict, true)
-      @max_hops    = opts.fetch(:max_hops, 5).to_i
-      @on_redirect = opts.fetch(:on_redirect, nil)
+    # Initializes a new Redirector
+    #
+    # @example
+    #   HTTP::Redirector.new(strict: true, max_hops: 5)
+    #
+    # @param [Boolean] strict (true) redirector hops policy
+    # @param [#to_i] max_hops (5) maximum allowed amount of hops
+    # @param [#call, nil] on_redirect optional redirect callback
+    # @api public
+    # @return [HTTP::Redirector]
+    def initialize(strict: true, max_hops: 5, on_redirect: nil)
+      @strict      = strict
+      @max_hops    = Integer(max_hops)
+      @on_redirect = on_redirect
     end
 
     # Follows redirects until non-redirect response found
-    def perform(request, response)
+    #
+    # @example
+    #   redirector.perform(request, response) { |req| client.perform(req) }
+    #
+    # @param [HTTP::Request] request
+    # @param [HTTP::Response] response
+    # @api public
+    # @return [HTTP::Response]
+    def perform(request, response, &)
       @request  = request
       @response = response
       @visited  = []
-      collect_cookies_from_request
-      collect_cookies_from_response
-
-      while REDIRECT_CODES.include? @response.status.code
-        @visited << "#{@request.verb} #{@request.uri}"
-
-        raise TooManyRedirectsError if too_many_hops?
-        raise EndlessRedirectError  if endless_loop?
 
-        @response.flush
-
-        # XXX(ixti): using `Array#inject` to return `nil` if no Location header.
-        @request = redirect_to(@response.headers.get(Headers::LOCATION).inject(:+))
-        unless cookie_jar.empty?
-          @request.headers.set(Headers::COOKIE, cookie_jar.cookies.map { |c| "#{c.name}=#{c.value}" }.join("; "))
-        end
-        @on_redirect.call @response, @request if @on_redirect.respond_to?(:call)
-        @response = yield @request
-        collect_cookies_from_response
-      end
+      follow_redirects(&) while REDIRECT_CODES.include?(@response.code)
 
       @response
     end
 
     private
 
-    # All known cookies. On the original request, this is only the original cookies, but after that,
-    # Set-Cookie headers can add, set or delete cookies.
-    def cookie_jar
-      # it seems that @response.cookies instance is reused between responses, so we have to "clone"
-      @cookie_jar ||= HTTP::CookieJar.new
-    end
+    # Perform a single redirect step
+    #
+    # @api private
+    # @return [void]
+    def follow_redirects
+      @visited << visit_key
 
-    def collect_cookies_from_request
-      request_cookie_header = @request.headers["Cookie"]
-      cookies =
-        if request_cookie_header
-          HTTP::Cookie.cookie_value_to_hash(request_cookie_header)
-        else
-          {}
-        end
-
-      cookies.each do |key, value|
-        cookie_jar.add(HTTP::Cookie.new(key, value, :path => @request.uri.path, :domain => @request.host))
-      end
+      raise TooManyRedirectsError if too_many_hops?
+      raise EndlessRedirectError  if endless_loop?
+
+      @response.flush
+
+      @request = redirect_to(redirect_uri)
+      @on_redirect&.call @response, @request
+      @response = yield @request
     end
 
-    # Carry cookies from one response to the next. Carrying cookies to the next response ends up
-    # carrying them to the next request as well.
+    # Extracts the redirect URI from the Location header
     #
-    # Note that this isn't part of the IETF standard, but all major browsers support setting cookies
-    # on redirect: https://blog.dubbelboer.com/2012/11/25/302-cookie.html
-    def collect_cookies_from_response
-      # Overwrite previous cookies
-      @response.cookies.each do |cookie|
-        if cookie.value == ""
-          cookie_jar.delete(cookie)
-        else
-          cookie_jar.add(cookie)
-        end
-      end
-
-      # I wish we could just do @response.cookes = cookie_jar
-      cookie_jar.each do |cookie|
-        @response.cookies.add(cookie)
-      end
+    # @api private
+    # @return [String, nil] URI string or nil if no Location header
+    def redirect_uri
+      location = @response.headers.get(Headers::LOCATION)
+      location.join unless location.empty?
     end
 
     # Check if we reached max amount of redirect hops
+    #
+    # @api private
     # @return [Boolean]
     def too_many_hops?
-      1 <= @max_hops && @max_hops < @visited.count
+      @max_hops.positive? && @visited.length > @max_hops
     end
 
     # Check if we got into an endless loop
+    #
+    # @api private
     # @return [Boolean]
     def endless_loop?
-      2 <= @visited.count(@visited.last)
+      @visited.count(@visited.last) > 1
+    end
+
+    # Build a visit key for the current request
+    #
+    # Includes verb, URI, and Cookie header so that requests to the same URL
+    # with different cookies are not falsely detected as an endless loop.
+    #
+    # @api private
+    # @return [String]
+    def visit_key
+      "#{@request.verb} #{@request.uri} #{@request.headers[Headers::COOKIE]}"
     end
 
     # Redirect policy for follow
+    #
+    # @api private
     # @return [Request]
     def redirect_to(uri)
       raise StateError, "no Location header in redirect" unless uri
 
       verb = @request.verb
-      code = @response.status.code
+      code = @response.code
 
       if UNSAFE_VERBS.include?(verb) && STRICT_SENSITIVE_CODES.include?(code)
         raise StateError, "can't follow #{@response.status} redirect" if @strict
@@ -144,7 +149,7 @@ module HTTP
         verb = :get
       end
 
-      verb = :get if !SEE_OTHER_ALLOWED_VERBS.include?(verb) && 303 == code
+      verb = :get if !SEE_OTHER_ALLOWED_VERBS.include?(verb) && code.eql?(303)
 
       @request.redirect(uri, verb)
     end

data/lib/http/request/body.rb

@@ -2,18 +2,66 @@
 
 module HTTP
   class Request
+    # Represents an HTTP request body with streaming support
     class Body
+      # The source data for this body
+      #
+      # @example
+      #   body.source # => "hello world"
+      #
+      # @return [String, Enumerable, IO, nil]
+      # @api public
       attr_reader :source
 
+      # Initialize a new request body
+      #
+      # @example
+      #   Body.new("hello world")
+      #
+      # @return [HTTP::Request::Body]
+      # @api public
       def initialize(source)
         @source = source
 
         validate_source_type!
       end
 
-      # Returns size which should be used for the "Content-Length" header.
+      # Whether the body is empty
+      #
+      # @example
+      #   body.empty? # => true
+      #
+      # @return [Boolean]
+      # @api public
+      def empty?
+        @source.nil?
+      end
+
+      # Whether the body content can be accessed for logging
+      #
+      # Returns true for String sources (the content can be inspected).
+      # Returns false for IO streams and Enumerables (which cannot be
+      # read without consuming them), and for nil bodies.
+      #
+      # The logging feature checks the string encoding separately to
+      # decide whether to log the content as text or format it as binary.
+      #
+      # @example
+      #   body.loggable? # => true
+      #
+      # @return [Boolean]
+      # @api public
+      def loggable?
+        @source.is_a?(String)
+      end
+
+      # Returns size for the "Content-Length" header
+      #
+      # @example
+      #   body.size
       #
       # @return [Integer]
+      # @api public
       def size
         if @source.is_a?(String)
           @source.bytesize
@@ -24,33 +72,49 @@ module HTTP
         elsif @source.nil?
           0
         else
-          raise RequestError, "cannot determine size of body: #{@source.inspect}"
+          raise RequestError,
+                "cannot determine size of body: #{@source.inspect}; " \
+                "set the Content-Length header explicitly or use chunked Transfer-Encoding"
         end
       end
 
-      # Yields chunks of content to be streamed to the request body.
+      # Yields chunks of content to be streamed
+      #
+      # @example
+      #   body.each { |chunk| socket.write(chunk) }
       #
       # @yieldparam [String]
+      # @return [self]
+      # @api public
       def each(&block)
         if @source.is_a?(String)
           yield @source
         elsif @source.respond_to?(:read)
           IO.copy_stream(@source, ProcIO.new(block))
           rewind(@source)
-        elsif @source.is_a?(Enumerable)
+        elsif @source
           @source.each(&block)
         end
 
         self
       end
 
-      # Request bodies are equivalent when they have the same source.
+      # Check equality based on source
+      #
+      # @example
+      #   body == other_body
+      #
+      # @return [Boolean]
+      # @api public
       def ==(other)
-        self.class == other.class && self.source == other.source # rubocop:disable Style/RedundantSelf
+        other.is_a?(self.class) && source == other.source
       end
 
       private
 
+      # Rewind an IO source if possible
+      # @return [void]
+      # @api private
       def rewind(io)
         io.rewind if io.respond_to? :rewind
       rescue Errno::ESPIPE, Errno::EPIPE
@@ -73,6 +137,9 @@ module HTTP
         nil
       end
 
+      # Validate that source is a supported type
+      # @return [void]
+      # @api private
       def validate_source_type!
         return if @source.is_a?(String)
         return if @source.respond_to?(:read)
@@ -86,10 +153,24 @@ module HTTP
       # #write simply calling the proc, which we can pass in as the
       # "destination IO" in IO.copy_stream.
       class ProcIO
+        # Initialize a new ProcIO wrapper
+        #
+        # @example
+        #   ProcIO.new(block)
+        #
+        # @return [ProcIO]
+        # @api public
         def initialize(block)
           @block = block
         end
 
+        # Write data by calling the wrapped proc
+        #
+        # @example
+        #   proc_io.write("hello")
+        #
+        # @return [Integer]
+        # @api public
         def write(data)
           @block.call(data)
           data.bytesize

data/lib/http/request/builder.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require "uri"
+
+require "http/form_data"
+require "http/headers"
+require "http/connection"
+require "http/uri"
+
+module HTTP
+  class Request
+    # Builds HTTP::Request objects from resolved options
+    #
+    # @example Build a request from options
+    #   options = HTTP::Options.new(headers: {"Accept" => "application/json"})
+    #   builder = HTTP::Request::Builder.new(options)
+    #   request = builder.build(:get, "https://example.com")
+    #
+    # @see Options
+    class Builder
+      # Pattern matching HTTP or HTTPS URI schemes
+      HTTP_OR_HTTPS_RE = %r{\Ahttps?://}i
+
+      # Initialize a new Request Builder
+      #
+      # @example
+      #   HTTP::Request::Builder.new(HTTP::Options.new)
+      #
+      # @param options [HTTP::Options] resolved request options
+      # @return [HTTP::Request::Builder]
+      # @api public
+      def initialize(options)
+        @options = options
+      end
+
+      # Build an HTTP request
+      #
+      # @example
+      #   builder.build(:get, "https://example.com")
+      #
+      # @param verb [Symbol] the HTTP method
+      # @param uri [#to_s] the URI to request
+      # @return [HTTP::Request] the built request object
+      # @api public
+      def build(verb, uri)
+        uri     = make_request_uri(uri)
+        headers = make_request_headers
+        body    = make_request_body(headers)
+
+        req = HTTP::Request.new(
+          verb:           verb,
+          uri:            uri,
+          uri_normalizer: @options.feature(:normalize_uri)&.normalizer,
+          proxy:          @options.proxy,
+          headers:        headers,
+          body:           body
+        )
+
+        wrap(req)
+      end
+
+      # Wrap a request through feature middleware
+      #
+      # @example
+      #   builder.wrap(redirect_request)
+      #
+      # @param request [HTTP::Request] the request to wrap
+      # @return [HTTP::Request] the wrapped request
+      # @api public
+      def wrap(request)
+        @options.features.inject(request) do |req, (_name, feature)|
+          feature.wrap_request(req)
+        end
+      end
+
+      private
+
+      # Merges query params if needed
+      #
+      # @param uri [#to_s] the URI to process
+      # @return [HTTP::URI] the constructed URI
+      # @api private
+      def make_request_uri(uri)
+        uri = uri.to_s
+
+        if @options.base_uri? && uri !~ HTTP_OR_HTTPS_RE
+          uri = resolve_against_base(uri)
+        elsif @options.persistent? && uri !~ HTTP_OR_HTTPS_RE
+          uri = "#{@options.persistent}#{uri}"
+        end
+
+        uri = HTTP::URI.parse uri
+
+        merge_query_params!(uri)
+
+        # Some proxies (seen on WEBrick) fail if URL has
+        # empty path (e.g. `http://example.com`) while it's RFC-compliant:
+        # http://tools.ietf.org/html/rfc1738#section-3.1
+        uri.path = "/" if uri.path.empty?
+
+        uri
+      end
+
+      # Resolve a relative URI against the configured base URI
+      #
+      # Ensures the base URI path has a trailing slash so that relative
+      # paths are appended rather than replacing the last path segment,
+      # per the convention described in RFC 3986 Section 5.
+      #
+      # @param uri [String] the relative URI to resolve
+      # @return [String] the resolved absolute URI
+      # @api private
+      def resolve_against_base(uri)
+        base = @options.base_uri or raise Error, "base_uri is not set"
+
+        unless base.path.end_with?("/")
+          base = base.dup
+          base.path = "#{base.path}/"
+        end
+
+        String(base.join(uri))
+      end
+
+      # Merge query parameters into URI
+      #
+      # @return [void]
+      # @api private
+      def merge_query_params!(uri)
+        return unless @options.params && !@options.params.empty?
+
+        existing = ::URI.decode_www_form(uri.query || "")
+        uri.query = ::URI.encode_www_form(existing.concat(@options.params.to_a))
+      end
+
+      # Creates request headers
+      #
+      # @return [HTTP::Headers] the constructed headers
+      # @api private
+      def make_request_headers
+        headers = @options.headers
+
+        # Tell the server to keep the conn open
+        headers[Headers::CONNECTION] = @options.persistent? ? Connection::KEEP_ALIVE : Connection::CLOSE
+
+        headers
+      end
+
+      # Create the request body object to send
+      #
+      # @return [String, HTTP::FormData, nil] the request body
+      # @api private
+      def make_request_body(headers)
+        if @options.body
+          @options.body
+        elsif @options.form
+          form = make_form_data(@options.form)
+          headers[Headers::CONTENT_TYPE] ||= form.content_type
+          form
+        elsif @options.json
+          make_json_body(@options.json, headers)
+        end
+      end
+
+      # Encode JSON body and set content type header
+      # @return [String] the encoded JSON body
+      # @api private
+      def make_json_body(data, headers)
+        body = MimeType[:json].encode data
+        headers[Headers::CONTENT_TYPE] ||= "application/json; charset=#{body.encoding.name.downcase}"
+        body
+      end
+
+      # Coerce form data into an HTTP::FormData object
+      # @return [HTTP::FormData::Multipart, HTTP::FormData::Urlencoded] form data
+      # @api private
+      def make_form_data(form)
+        return form if form.is_a? FormData::Multipart
+        return form if form.is_a? FormData::Urlencoded
+
+        FormData.create(form)
+      end
+    end
+  end
+end

data/lib/http/request/proxy.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module HTTP
+  class Request
+    # Proxy-related methods for HTTP requests
+    module Proxy
+      # Merges proxy headers into the request headers
+      #
+      # @example
+      #   request.include_proxy_headers
+      #
+      # @return [void]
+      # @api public
+      def include_proxy_headers
+        headers.merge!(proxy[:proxy_headers]) if proxy.key?(:proxy_headers)
+        include_proxy_authorization_header if using_authenticated_proxy?
+      end
+
+      # Compute and add the Proxy-Authorization header
+      #
+      # @example
+      #   request.include_proxy_authorization_header
+      #
+      # @return [void]
+      # @api public
+      def include_proxy_authorization_header
+        headers[Headers::PROXY_AUTHORIZATION] = proxy_authorization_header
+      end
+
+      # Build the Proxy-Authorization header value
+      #
+      # @example
+      #   request.proxy_authorization_header
+      #
+      # @return [String]
+      # @api public
+      def proxy_authorization_header
+        digest = encode64(format("%s:%s", proxy.fetch(:proxy_username), proxy.fetch(:proxy_password)))
+        "Basic #{digest}"
+      end
+
+      # Setup tunnel through proxy for SSL request
+      #
+      # @example
+      #   request.connect_using_proxy(socket)
+      #
+      # @return [void]
+      # @api public
+      def connect_using_proxy(socket)
+        Writer.new(socket, nil, proxy_connect_headers, proxy_connect_header).connect_through_proxy
+      end
+
+      # Compute HTTP request header SSL proxy connection
+      #
+      # @example
+      #   request.proxy_connect_header
+      #
+      # @return [String]
+      # @api public
+      def proxy_connect_header
+        "CONNECT #{host}:#{port} HTTP/#{version}"
+      end
+
+      # Headers to send with proxy connect request
+      #
+      # @example
+      #   request.proxy_connect_headers
+      #
+      # @return [HTTP::Headers]
+      # @api public
+      def proxy_connect_headers
+        connect_headers = Headers.coerce(
+          Headers::HOST       => headers[Headers::HOST],
+          Headers::USER_AGENT => headers[Headers::USER_AGENT]
+        )
+
+        connect_headers[Headers::PROXY_AUTHORIZATION] = proxy_authorization_header if using_authenticated_proxy?
+        connect_headers.merge!(proxy[:proxy_headers]) if proxy.key?(:proxy_headers)
+        connect_headers
+      end
+    end
+  end
+end