http 5.3.1 → 6.0.0

data/lib/http/client.rb

@@ -3,11 +3,13 @@
 require "forwardable"
 
 require "http/form_data"
+require "http/retriable/performer"
 require "http/options"
 require "http/feature"
 require "http/headers"
 require "http/connection"
 require "http/redirector"
+require "http/request/builder"
 require "http/uri"
 
 module HTTP
@@ -16,76 +18,106 @@ module HTTP
     extend Forwardable
     include Chainable
 
-    HTTP_OR_HTTPS_RE = %r{^https?://}i.freeze
-
-    def initialize(default_options = {})
-      @default_options = HTTP::Options.new(default_options)
+    # Initialize a new HTTP Client
+    #
+    # @example
+    #   client = HTTP::Client.new(headers: {"Accept" => "application/json"})
+    #
+    # @param default_options [HTTP::Options, nil] existing options instance
+    # @param options [Hash] keyword options (see HTTP::Options#initialize)
+    # @return [HTTP::Client] a new client instance
+    # @api public
+    def initialize(default_options = nil, **)
+      @default_options = HTTP::Options.new(default_options, **)
       @connection = nil
       @state = :clean
     end
 
     # Make an HTTP request
-    def request(verb, uri, opts = {})
+    #
+    # @example
+    #   client.request(:get, "https://example.com")
+    #
+    # @param verb [Symbol] the HTTP method
+    # @param uri [#to_s] the URI to request
+    # @return [HTTP::Response] the response
+    # @api public
+    def request(verb, uri,
+                headers: nil, params: nil, form: nil, json: nil, body: nil,
+                response: nil, encoding: nil, follow: nil, ssl: nil, ssl_context: nil,
+                proxy: nil, nodelay: nil, features: nil, retriable: nil,
+                socket_class: nil, ssl_socket_class: nil, timeout_class: nil,
+                timeout_options: nil, keep_alive_timeout: nil, base_uri: nil, persistent: nil)
+      opts = { headers: headers, params: params, form: form, json: json, body: body,
+               response: response, encoding: encoding, follow: follow, ssl: ssl,
+               ssl_context: ssl_context, proxy: proxy, nodelay: nodelay, features: features,
+               retriable: retriable, socket_class: socket_class, ssl_socket_class: ssl_socket_class,
+               timeout_class: timeout_class, timeout_options: timeout_options,
+               keep_alive_timeout: keep_alive_timeout, base_uri: base_uri, persistent: persistent }.compact
       opts = @default_options.merge(opts)
-      req = build_request(verb, uri, opts)
-      res = perform(req, opts)
+      builder = Request::Builder.new(opts)
+      req     = builder.build(verb, uri)
+      res     = perform(req, opts)
       return res unless opts.follow
 
-      Redirector.new(opts.follow).perform(req, res) do |request|
-        perform(wrap_request(request, opts), opts)
+      Redirector.new(**opts.follow).perform(req, res) do |request|
+        perform(builder.wrap(request), opts)
       end
     end
 
-    # Prepare an HTTP request
-    def build_request(verb, uri, opts = {})
-      opts    = @default_options.merge(opts)
-      uri     = make_request_uri(uri, opts)
-      headers = make_request_headers(opts)
-      body    = make_request_body(opts, headers)
-
-      req = HTTP::Request.new(
-        :verb           => verb,
-        :uri            => uri,
-        :uri_normalizer => opts.feature(:normalize_uri)&.normalizer,
-        :proxy          => opts.proxy,
-        :headers        => headers,
-        :body           => body
-      )
-
-      wrap_request(req, opts)
-    end
-
     # @!method persistent?
+    #   Indicate whether the client has persistent connections
+    #
+    #   @example
+    #     client.persistent?
+    #
     #   @see Options#persistent?
     #   @return [Boolean] whenever client is persistent
+    #   @api public
     def_delegator :default_options, :persistent?
 
     # Perform a single (no follow) HTTP request
+    #
+    # @example
+    #   client.perform(request, options)
+    #
+    # @param req [HTTP::Request] the request to perform
+    # @param options [HTTP::Options] request options
+    # @return [HTTP::Response] the response
+    # @api public
     def perform(req, options)
-      verify_connection!(req.uri)
-
-      @state = :dirty
-
-      begin
-        @connection ||= HTTP::Connection.new(req, options)
-
-        unless @connection.failed_proxy_connect?
-          @connection.send_request(req)
-          @connection.read_headers!
-        end
-      rescue Error => e
-        options.features.each_value do |feature|
-          feature.on_error(req, e)
-        end
-        raise
+      if options.retriable
+        perform_with_retry(req, options)
+      else
+        perform_once(req, options)
       end
-      res = build_response(req, options)
+    end
 
-      res = options.features.inject(res) do |response, (_name, feature)|
-        feature.wrap_response(response)
-      end
+    # Close the connection and reset state
+    #
+    # @example
+    #   client.close
+    #
+    # @return [void]
+    # @api public
+    def close
+      @connection&.close
+      @connection = nil
+      @state = :clean
+    end
 
-      @connection.finish_response if req.verb == :head
+    private
+
+    # Execute a single HTTP request without retry logic
+    #
+    # @param req [HTTP::Request] the request to perform
+    # @param options [HTTP::Options] request options
+    # @return [HTTP::Response] the response
+    # @api private
+    def perform_once(req, options)
+      res = perform_exchange(req, options)
+
+      @connection.finish_response if res.request.verb == :head
       @state = :clean
 
       res
@@ -94,33 +126,93 @@ module HTTP
       raise
     end
 
-    def close
-      @connection&.close
-      @connection = nil
-      @state = :clean
+    # Execute a request with retry logic
+    #
+    # @param req [HTTP::Request] the request to perform
+    # @param options [HTTP::Options] request options
+    # @return [HTTP::Response] the response
+    # @api private
+    def perform_with_retry(req, options)
+      Retriable::Performer.new(**options.retriable).perform(self, req) do
+        perform_once(req, options)
+      end
     end
 
-    private
+    # Send request over the connection, handling proxy and errors
+    # @return [void]
+    # @api private
+    def send_request(req, options)
+      notify_features(req, options)
+
+      @connection ||= HTTP::Connection.new(req, options)
 
-    def wrap_request(req, opts)
-      opts.features.inject(req) do |request, (_name, feature)|
-        feature.wrap_request(request)
+      unless @connection.failed_proxy_connect?
+        @connection.send_request(req)
+        @connection.read_headers!
       end
+    rescue Error => e
+      options.features.each_value { |feature| feature.on_error(req, e) }
+      raise
     end
 
+    # Build response and apply feature wrapping
+    # @return [HTTP::Response] the wrapped response
+    # @api private
+    def build_wrapped_response(req, options)
+      res = build_response(req, options)
+
+      options.features.values.reverse.inject(res) do |response, feature|
+        feature.wrap_response(response)
+      end
+    end
+
+    # Notify features of an upcoming request attempt
+    # @return [void]
+    # @api private
+    def notify_features(req, options)
+      options.features.each_value { |feature| feature.on_request(req) }
+    end
+
+    # Execute the HTTP exchange wrapped by feature around_request hooks
+    # @return [HTTP::Response] the response
+    # @api private
+    def perform_exchange(req, options)
+      around_request(req, options) do |request|
+        verify_connection!(request.uri)
+        @state = :dirty
+        send_request(request, options)
+        build_wrapped_response(request, options)
+      end
+    end
+
+    # Compose around_request chains from all features
+    # @return [HTTP::Response] the response
+    # @api private
+    def around_request(request, options, &block)
+      options.features.values.reverse.reduce(block) do |inner, feature|
+        ->(req) { feature.around_request(req) { |r| inner.call(r) } }
+      end.call(request)
+    end
+
+    # Build a response from the current connection
+    # @return [HTTP::Response] the built response
+    # @api private
     def build_response(req, options)
       Response.new(
-        :status        => @connection.status_code,
-        :version       => @connection.http_version,
-        :headers       => @connection.headers,
-        :proxy_headers => @connection.proxy_response_headers,
-        :connection    => @connection,
-        :encoding      => options.encoding,
-        :request       => req
-      )
+        status:        @connection.status_code,
+        version:       @connection.http_version,
+        headers:       @connection.headers,
+        proxy_headers: @connection.proxy_response_headers,
+        connection:    @connection,
+        encoding:      options.encoding,
+        request:       req
+      ).tap { |res| @connection.pending_response = res }
     end
 
     # Verify our request isn't going to be made against another URI
+    #
+    # @return [void]
+    # @api private
     def verify_connection!(uri)
       if default_options.persistent? && uri.origin != default_options.persistent
         raise StateError, "Persistence is enabled for #{default_options.persistent}, but we got #{uri.origin}"
@@ -132,68 +224,7 @@ module HTTP
 
       # If we get into a bad state (eg, Timeout.timeout ensure being killed)
       # close the connection to prevent potential for mixed responses.
-      return close if @state == :dirty
-    end
-
-    # Merges query params if needed
-    #
-    # @param [#to_s] uri
-    # @return [URI]
-    def make_request_uri(uri, opts)
-      uri = uri.to_s
-
-      uri = "#{default_options.persistent}#{uri}" if default_options.persistent? && uri !~ HTTP_OR_HTTPS_RE
-
-      uri = HTTP::URI.parse uri
-
-      uri.query_values = uri.query_values(Array).to_a.concat(opts.params.to_a) if opts.params && !opts.params.empty?
-
-      # Some proxies (seen on WEBRick) fail if URL has
-      # empty path (e.g. `http://example.com`) while it's RFC-complaint:
-      # http://tools.ietf.org/html/rfc1738#section-3.1
-      uri.path = "/" if uri.path.empty?
-
-      uri
-    end
-
-    # Creates request headers with cookies (if any) merged in
-    def make_request_headers(opts)
-      headers = opts.headers
-
-      # Tell the server to keep the conn open
-      headers[Headers::CONNECTION] = default_options.persistent? ? Connection::KEEP_ALIVE : Connection::CLOSE
-
-      cookies = opts.cookies.values
-
-      unless cookies.empty?
-        cookies = opts.headers.get(Headers::COOKIE).concat(cookies).join("; ")
-        headers[Headers::COOKIE] = cookies
-      end
-
-      headers
-    end
-
-    # Create the request body object to send
-    def make_request_body(opts, headers)
-      case
-      when opts.body
-        opts.body
-      when opts.form
-        form = make_form_data(opts.form)
-        headers[Headers::CONTENT_TYPE] ||= form.content_type
-        form
-      when opts.json
-        body = MimeType[:json].encode opts.json
-        headers[Headers::CONTENT_TYPE] ||= "application/json; charset=#{body.encoding.name.downcase}"
-        body
-      end
-    end
-
-    def make_form_data(form)
-      return form if form.is_a? HTTP::FormData::Multipart
-      return form if form.is_a? HTTP::FormData::Urlencoded
-
-      HTTP::FormData.create(form)
+      close if @state == :dirty
     end
   end
 end

data/lib/http/connection/internals.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module HTTP
+  class Connection
+    # Internal private methods for Connection
+    module Internals
+      private
+
+      # Flush the pending response body so the connection can be reused
+      # @return [void]
+      # @api private
+      def flush_pending_response
+        response = @pending_response
+        unless response.respond_to?(:flush)
+          close
+          return
+        end
+
+        flush_or_close_response(response)
+      rescue
+        close
+      end
+
+      # Flush the response or close if the body exceeds the size limit
+      # @param response [HTTP::Response] the response to flush
+      # @return [void]
+      # @api private
+      def flush_or_close_response(response)
+        content_length = response.content_length
+        if content_length && content_length > MAX_FLUSH_SIZE
+          close
+        else
+          response.flush
+        end
+      end
+
+      # Sets up SSL context and starts TLS if needed
+      # @param (see Connection#initialize)
+      # @return [void]
+      # @api private
+      def start_tls(req, options)
+        return unless req.uri.https? && !failed_proxy_connect?
+
+        ssl_context = options.ssl_context
+
+        unless ssl_context
+          ssl_context = OpenSSL::SSL::SSLContext.new
+          ssl_context.set_params(options.ssl || {})
+        end
+
+        @socket.start_tls(req.uri.host, options.ssl_socket_class, ssl_context)
+      end
+
+      # Open tunnel through proxy
+      # @return [void]
+      # @api private
+      def send_proxy_connect_request(req)
+        return unless req.uri.https? && req.using_proxy?
+
+        @pending_request = true
+
+        req.connect_using_proxy @socket
+
+        @pending_request  = false
+        @pending_response = true
+
+        read_headers!
+        handle_proxy_connect_response
+      end
+
+      # Process the proxy connect response
+      # @return [void]
+      # @api private
+      def handle_proxy_connect_response
+        @proxy_response_headers = @parser.headers
+
+        if @parser.status_code != 200
+          @failed_proxy_connect = true
+          return
+        end
+
+        @parser.reset
+        @pending_response = false
+      end
+
+      # Resets expiration of persistent connection
+      # @return [void]
+      # @api private
+      def reset_timer
+        @conn_expires_at = Time.now + @keep_alive_timeout if @persistent
+      end
+
+      # Store keep-alive state from parser
+      # @return [void]
+      # @api private
+      def set_keep_alive
+        return @keep_alive = false unless @persistent
+
+        @keep_alive =
+          case @parser.http_version
+          when HTTP_1_0 # HTTP/1.0 requires opt in for Keep Alive
+            @parser.headers[Headers::CONNECTION] == KEEP_ALIVE
+          when HTTP_1_1 # HTTP/1.1 is opt-out
+            @parser.headers[Headers::CONNECTION] != CLOSE
+          else # Anything else we assume doesn't support it
+            false
+          end
+      end
+
+      # Check if the response body has a known framing mechanism
+      #
+      # @example
+      #   body_framed?
+      #
+      # @return [Boolean]
+      # @api private
+      def body_framed?
+        @parser.headers.include?(Headers::TRANSFER_ENCODING) ||
+          @parser.headers.include?(Headers::CONTENT_LENGTH)
+      end
+
+      # Feeds some more data into parser
+      # @return [void]
+      # @raise [SocketReadError] when unable to read from socket
+      # @api private
+      def read_more(size)
+        return if @parser.finished?
+
+        value = @socket.readpartial(size, @buffer)
+        if value == :eof
+          @parser << ""
+          :eof
+        elsif value
+          @parser << value
+        end
+      rescue IOError, SocketError, SystemCallError => e
+        raise SocketReadError, "error reading from socket: #{e}", e.backtrace
+      end
+    end
+  end
+end