httpx 1.1.5 → 1.2.1

data/lib/httpx/connection.rb

@@ -47,11 +47,11 @@ module HTTPX
 
     attr_accessor :family
 
-    def initialize(type, uri, options)
-      @type = type
+    def initialize(uri, options)
       @origins = [uri.origin]
       @origin = Utils.to_uri(uri.origin)
       @options = Options.new(options)
+      @type = initialize_type(uri, @options)
       @window_size = @options.window_size
       @read_buffer = Buffer.new(@options.buffer_size)
       @write_buffer = Buffer.new(@options.buffer_size)
@@ -92,18 +92,14 @@ module HTTPX
     def match?(uri, options)
       return false if !used? && (@state == :closing || @state == :closed)
 
-      return false if exhausted?
-
       (
-        (
-          @origins.include?(uri.origin) &&
-          # if there is more than one origin to match, it means that this connection
-          # was the result of coalescing. To prevent blind trust in the case where the
-          # origin came from an ORIGIN frame, we're going to verify the hostname with the
-          # SSL certificate
-          (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host)))
-        ) && @options == options
-      ) || (match_altsvcs?(uri) && match_altsvc_options?(uri, options))
+        @origins.include?(uri.origin) &&
+        # if there is more than one origin to match, it means that this connection
+        # was the result of coalescing. To prevent blind trust in the case where the
+        # origin came from an ORIGIN frame, we're going to verify the hostname with the
+        # SSL certificate
+        (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host)))
+      ) && @options == options
     end
 
     def expired?
@@ -115,8 +111,6 @@ module HTTPX
     def mergeable?(connection)
       return false if @state == :closing || @state == :closed || !@io
 
-      return false if exhausted?
-
       return false unless connection.addresses
 
       (
@@ -139,7 +133,7 @@ module HTTPX
     end
 
     def create_idle(options = {})
-      self.class.new(@type, @origin, @options.merge(options))
+      self.class.new(@origin, @options.merge(options))
     end
 
     def merge(connection)
@@ -167,24 +161,6 @@ module HTTPX
       end
     end
 
-    # checks if this is connection is an alternative service of
-    # +uri+
-    def match_altsvcs?(uri)
-      @origins.any? { |origin| uri.altsvc_match?(origin) } ||
-        AltSvc.cached_altsvc(@origin).any? do |altsvc|
-          origin = altsvc["origin"]
-          origin.altsvc_match?(uri.origin)
-        end
-    end
-
-    def match_altsvc_options?(uri, options)
-      return @options == options unless @options.ssl[:hostname] == uri.host
-
-      dup_options = @options.merge(ssl: { hostname: nil })
-      dup_options.ssl.delete(:hostname)
-      dup_options == options
-    end
-
     def connecting?
       @state == :idle
     end
@@ -223,7 +199,6 @@ module HTTPX
       when :closing
         consume
         transition(:closed)
-        emit(:close)
       when :open
         consume
       end
@@ -236,24 +211,29 @@ module HTTPX
       @parser.close if @parser
     end
 
+    def terminate
+      @connected_at = nil if @state == :closed
+
+      close
+    end
+
     # bypasses the state machine to force closing of connections still connecting.
     # **only** used for Happy Eyeballs v2.
     def force_reset
       @state = :closing
       transition(:closed)
-      emit(:close)
     end
 
     def reset
+      return if @state == :closing || @state == :closed
+
       transition(:closing)
+
       transition(:closed)
-      emit(:close)
     end
 
     def send(request)
       if @parser && !@write_buffer.full?
-        request.headers["alt-used"] = @origin.authority if match_altsvcs?(request.uri)
-
         if @response_received_at && @keep_alive_timeout &&
            Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
           # when pushing a request into an existing connection, we have to check whether there
@@ -319,10 +299,6 @@ module HTTPX
       transition(:open)
     end
 
-    def exhausted?
-      @parser && parser.exhausted?
-    end
-
     def consume
       return unless @io
 
@@ -497,34 +473,25 @@ module HTTPX
         request.emit(:promise, parser, stream)
       end
       parser.on(:exhausted) do
+        @pending.concat(parser.pending)
         emit(:exhausted)
       end
       parser.on(:origin) do |origin|
         @origins |= [origin]
       end
       parser.on(:close) do |force|
-        if @state != :closed
-          transition(:closing)
-          if force || @state == :idle
-            transition(:closed)
-            emit(:close)
-          end
+        if force
+          reset
+          emit(:terminate)
         end
       end
       parser.on(:close_handshake) do
         consume
       end
       parser.on(:reset) do
-        if parser.empty?
-          reset
-        else
-          transition(:closing)
-          transition(:closed)
-
-          @parser.reset if @parser
-          transition(:idle)
-          transition(:open)
-        end
+        @pending.concat(parser.pending) unless parser.empty?
+        reset
+        idling unless @pending.empty?
       end
       parser.on(:current_timeout) do
         @current_timeout = @timeout = parser.timeout
@@ -593,13 +560,23 @@ module HTTPX
       when :inactive
         return unless @state == :open
       when :closing
-        return unless @state == :open
-
+        return unless @state == :idle || @state == :open
+
+        unless @write_buffer.empty?
+          # preset state before handshake, as error callbacks
+          # may take it back here.
+          @state = nextstate
+          # handshakes, try sending
+          consume
+          @write_buffer.clear
+          return
+        end
       when :closed
         return unless @state == :closing
         return unless @write_buffer.empty?
 
         purge_after_closed
+        emit(:close) if @pending.empty?
       when :already_open
         nextstate = :open
         # the first check for given io readiness must still use a timeout.
@@ -621,6 +598,19 @@ module HTTPX
       @timeout = nil
     end
 
+    def initialize_type(uri, options)
+      options.transport || begin
+        case uri.scheme
+        when "http"
+          "tcp"
+        when "https"
+          "ssl"
+        else
+          raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
+        end
+      end
+    end
+
     def build_socket(addrs = nil)
       case @type
       when "tcp"
@@ -715,7 +705,7 @@ module HTTPX
         interval = @timers.after(timeout, callback)
 
         Array(finish_events).each do |event|
-          # clean up reques timeouts if the connection errors out
+          # clean up request timeouts if the connection errors out
           request.once(event) do
             if @intervals.include?(interval)
               interval.delete(callback)

data/lib/httpx/extensions.rb

@@ -54,21 +54,6 @@ module HTTPX
       def origin
         "#{scheme}://#{authority}"
       end unless URI::HTTP.method_defined?(:origin)
-
-      def altsvc_match?(uri)
-        uri = URI.parse(uri)
-
-        origin == uri.origin || begin
-          case scheme
-          when "h2"
-            (uri.scheme == "https" || uri.scheme == "h2") &&
-            host == uri.host &&
-            (port || default_port) == (uri.port || uri.default_port)
-          else
-            false
-          end
-        end
-      end
     end
   end
 end

data/lib/httpx/options.rb

@@ -11,6 +11,7 @@ module HTTPX
     MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
     KEEP_ALIVE_TIMEOUT = 20
     SETTINGS_TIMEOUT = 10
+    CLOSE_HANDSHAKE_TIMEOUT = 10
     CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
     REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
 
@@ -39,6 +40,7 @@ module HTTPX
       :timeout => {
         connect_timeout: CONNECT_TIMEOUT,
         settings_timeout: SETTINGS_TIMEOUT,
+        close_handshake_timeout: CLOSE_HANDSHAKE_TIMEOUT,
         operation_timeout: OPERATION_TIMEOUT,
         keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
         read_timeout: READ_TIMEOUT,
@@ -97,7 +99,7 @@ module HTTPX
     # :compress_request_body :: whether to auto-decompress response body (defaults to <tt>true</tt>)
     # :timeout :: hash of timeout configurations (supports <tt>:connect_timeout</tt>, <tt>:settings_timeout</tt>,
     #             <tt>:operation_timeout</tt>, <tt>:keep_alive_timeout</tt>,  <tt>:read_timeout</tt>,  <tt>:write_timeout</tt>
-    #            and <tt>:request_timeout</tt>
+    #             and <tt>:request_timeout</tt>
     # :headers :: hash of HTTP headers (ex: <tt>{ "x-custom-foo" => "bar" }</tt>)
     # :window_size :: number of bytes to read from a socket
     # :buffer_size :: internal read and write buffer size in bytes
@@ -226,44 +228,69 @@ module HTTPX
       OUT
     end
 
-    REQUEST_IVARS = %i[@params @form @xml @json @body].freeze
-    private_constant :REQUEST_IVARS
+    REQUEST_BODY_IVARS = %i[@headers @params @form @xml @json @body].freeze
 
     def ==(other)
-      ivars = instance_variables | other.instance_variables
+      super || options_equals?(other)
+    end
+
+    def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
+      # headers and other request options do not play a role, as they are
+      # relevant only for the request.
+      ivars = instance_variables - ignore_ivars
+      other_ivars = other.instance_variables - ignore_ivars
+
+      return false if ivars.size != other_ivars.size
+
+      return false if ivars.sort != other_ivars.sort
+
       ivars.all? do |ivar|
-        case ivar
-        when :@headers
-          # currently, this is used to pick up an available matching connection.
-          # the headers do not play a role, as they are relevant only for the request.
-          true
-        when *REQUEST_IVARS
-          true
-        else
-          instance_variable_get(ivar) == other.instance_variable_get(ivar)
-        end
+        instance_variable_get(ivar) == other.instance_variable_get(ivar)
       end
     end
 
+    OTHER_LOOKUP = ->(obj, k, ivar_map) {
+      case obj
+      when Hash
+        obj[ivar_map[k]]
+      else
+        obj.instance_variable_get(k)
+      end
+    }
     def merge(other)
-      raise ArgumentError, "#{other} is not a valid set of options" unless other.respond_to?(:to_hash)
+      ivar_map = nil
+      other_ivars = case other
+                    when Hash
+                      ivar_map = other.keys.to_h { |k| [:"@#{k}", k] }
+                      ivar_map.keys
+                    else
+                      other.instance_variables
+      end
+
+      return self if other_ivars.empty?
 
-      h2 = other.to_hash
-      return self if h2.empty?
+      return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == OTHER_LOOKUP[other, ivar, ivar_map] }
 
-      h1 = to_hash
+      opts = dup
 
-      return self if h1 >= h2
+      other_ivars.each do |ivar|
+        v = OTHER_LOOKUP[other, ivar, ivar_map]
 
-      merged = h1.merge(h2) do |_k, v1, v2|
-        if v1.respond_to?(:merge) && v2.respond_to?(:merge)
-          v1.merge(v2)
-        else
-          v2
+        unless v
+          opts.instance_variable_set(ivar, v)
+          next
         end
+
+        v = opts.__send__(:"option_#{ivar[1..-1]}", v)
+
+        orig_v = instance_variable_get(ivar)
+
+        v = orig_v.merge(v) if orig_v.respond_to?(:merge) && v.respond_to?(:merge)
+
+        opts.instance_variable_set(ivar, v)
       end
 
-      self.class.new(merged)
+      opts
     end
 
     def to_hash
@@ -272,10 +299,40 @@ module HTTPX
       end
     end
 
-    def initialize_dup(other)
-      instance_variables.each do |ivar|
-        instance_variable_set(ivar, other.instance_variable_get(ivar).dup)
+    def extend_with_plugin_classes(pl)
+      if defined?(pl::RequestMethods) || defined?(pl::RequestClassMethods)
+        @request_class = @request_class.dup
+        @request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
+        @request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
+      end
+      if defined?(pl::ResponseMethods) || defined?(pl::ResponseClassMethods)
+        @response_class = @response_class.dup
+        @response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
+        @response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
       end
+      if defined?(pl::HeadersMethods) || defined?(pl::HeadersClassMethods)
+        @headers_class = @headers_class.dup
+        @headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
+        @headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
+      end
+      if defined?(pl::RequestBodyMethods) || defined?(pl::RequestBodyClassMethods)
+        @request_body_class = @request_body_class.dup
+        @request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
+        @request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
+      end
+      if defined?(pl::ResponseBodyMethods) || defined?(pl::ResponseBodyClassMethods)
+        @response_body_class = @response_body_class.dup
+        @response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
+        @response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
+      end
+      if defined?(pl::ConnectionMethods)
+        @connection_class = @connection_class.dup
+        @connection_class.__send__(:include, pl::ConnectionMethods)
+      end
+      return unless defined?(pl::OptionsMethods)
+
+      @options_class = @options_class.dup
+      @options_class.__send__(:include, pl::OptionsMethods)
     end
 
     private

data/lib/httpx/plugins/aws_sigv4.rb

@@ -5,7 +5,7 @@ module HTTPX
     #
     # This plugin adds AWS Sigv4 authentication.
     #
-    # https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
+    # https://docs.aws.amazon.com/IAM/latest/UserGuide/signing-elements.html
     #
     # https://gitlab.com/os85/httpx/wikis/AWS-SigV4
     #
@@ -185,7 +185,7 @@ module HTTPX
         def canonical_query
           params = query.split("&")
           # params = params.map { |p| p.match(/=/) ? p : p + '=' }
-          # From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+          # From: https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html#create-canonical-request
           # Sort the parameter names by character code point in ascending order.
           # Parameters with duplicate names should be sorted by value.
           #

data/lib/httpx/plugins/basic_auth.rb

@@ -3,7 +3,7 @@
 module HTTPX
   module Plugins
     #
-    # This plugin adds helper methods to implement HTTP Basic Auth (https://tools.ietf.org/html/rfc7617)
+    # This plugin adds helper methods to implement HTTP Basic Auth (https://datatracker.ietf.org/doc/html/rfc7617)
     #
     # https://gitlab.com/os85/httpx/wikis/Auth#basic-auth
     #

data/lib/httpx/plugins/callbacks.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds suppoort for callbacks around the request/response lifecycle.
+    #
+    # https://gitlab.com/os85/httpx/-/wikis/Events
+    #
+    module Callbacks
+      # connection closed user-space errors happen after errors can be surfaced to requests,
+      # so they need to pierce through the scheduler, which is only possible by simulating an
+      # interrupt.
+      class CallbackError < Exception; end # rubocop:disable Lint/InheritException
+
+      module InstanceMethods
+        include HTTPX::Callbacks
+
+        %i[
+          connection_opened connection_closed
+          request_error
+          request_started request_body_chunk request_completed
+          response_started response_body_chunk response_completed
+        ].each do |meth|
+          class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+            def on_#{meth}(&blk)   # def on_connection_opened(&blk)
+              on(:#{meth}, &blk)   #   on(:connection_opened, &blk)
+            end                    # end
+          MOD
+        end
+
+        private
+
+        def init_connection(uri, options)
+          connection = super
+          connection.on(:open) do
+            emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
+          end
+          connection.on(:close) do
+            emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
+          end
+
+          connection
+        end
+
+        def set_request_callbacks(request)
+          super
+
+          request.on(:headers) do
+            emit_or_callback_error(:request_started, request)
+          end
+          request.on(:body_chunk) do |chunk|
+            emit_or_callback_error(:request_body_chunk, request, chunk)
+          end
+          request.on(:done) do
+            emit_or_callback_error(:request_completed, request)
+          end
+
+          request.on(:response_started) do |res|
+            if res.is_a?(Response)
+              emit_or_callback_error(:response_started, request, res)
+              res.on(:chunk_received) do |chunk|
+                emit_or_callback_error(:response_body_chunk, request, res, chunk)
+              end
+            else
+              emit_or_callback_error(:request_error, request, res.error)
+            end
+          end
+          request.on(:response) do |res|
+            emit_or_callback_error(:response_completed, request, res)
+          end
+        end
+
+        def emit_or_callback_error(*args)
+          emit(*args)
+        rescue StandardError => e
+          ex = CallbackError.new(e.message)
+          ex.set_backtrace(e.backtrace)
+          raise ex
+        end
+
+        def receive_requests(*)
+          super
+        rescue CallbackError => e
+          raise e.cause
+        end
+      end
+    end
+    register_plugin :callbacks, Callbacks
+  end
+end

data/lib/httpx/plugins/circuit_breaker.rb

@@ -25,6 +25,8 @@ module HTTPX
       end
 
       module InstanceMethods
+        include HTTPX::Callbacks
+
         def initialize(*)
           super
           @circuit_store = CircuitStore.new(@options)

data/lib/httpx/plugins/cookies.rb

@@ -71,22 +71,32 @@ module HTTPX
       end
 
       module OptionsMethods
-        def do_initialize(*)
-          super
+        def option_headers(*)
+          value = super
+
+          merge_cookie_in_jar(value.delete("cookie"), @cookies) if defined?(@cookies) && value.key?("cookie")
 
-          return unless @headers.key?("cookie")
+          value
+        end
 
-          @headers.delete("cookie").each do |ck|
+        def option_cookies(value)
+          jar = value.is_a?(Jar) ? value : Jar.new(value)
+
+          merge_cookie_in_jar(@headers.delete("cookie"), jar) if defined?(@headers) && @headers.key?("cookie")
+
+          jar
+        end
+
+        private
+
+        def merge_cookie_in_jar(cookies, jar)
+          cookies.each do |ck|
             ck.split(/ *; */).each do |cookie|
               name, value = cookie.split("=", 2)
-              @cookies.add(Cookie.new(name, value))
+              jar.add(Cookie.new(name, value))
             end
           end
         end
-
-        def option_cookies(value)
-          value.is_a?(Jar) ? value : Jar.new(value)
-        end
       end
     end
     register_plugin :cookies, Cookies

data/lib/httpx/plugins/digest_auth.rb

@@ -3,7 +3,7 @@
 module HTTPX
   module Plugins
     #
-    # This plugin adds helper methods to implement HTTP Digest Auth (https://tools.ietf.org/html/rfc7616)
+    # This plugin adds helper methods to implement HTTP Digest Auth (https://datatracker.ietf.org/doc/html/rfc7616)
     #
     # https://gitlab.com/os85/httpx/wikis/Auth#digest-auth
     #

data/lib/httpx/plugins/follow_redirects.rb

@@ -34,6 +34,12 @@ module HTTPX
         def option_allow_auth_to_other_origins(value)
           value
         end
+
+        def option_redirect_on(value)
+          raise TypeError, ":redirect_on must be callable" unless value.respond_to?(:call)
+
+          value
+        end
       end
 
       module InstanceMethods
@@ -57,6 +63,11 @@ module HTTPX
           # build redirect request
           redirect_uri = __get_location_from_response(response)
 
+          if options.redirect_on
+            redirect_allowed = options.redirect_on.call(redirect_uri)
+            return response unless redirect_allowed
+          end
+
           if response.status == 305 && options.respond_to?(:proxy)
             # The requested resource MUST be accessed through the proxy given by
             # the Location field. The Location field gives the URI of the proxy.

data/lib/httpx/plugins/grpc.rb

@@ -261,14 +261,14 @@ module HTTPX
             headers["grpc-timeout"] = "#{deadline}m"
           end
 
-          headers = headers.merge(metadata) if metadata
+          headers = headers.merge(metadata.transform_keys(&:to_s)) if metadata
 
           # prepare compressor
           compression = @options.grpc_compression == true ? "gzip" : @options.grpc_compression
 
           headers["grpc-encoding"] = compression if compression
 
-          headers.merge!(@options.call_credentials.call) if @options.call_credentials
+          headers.merge!(@options.call_credentials.call.transform_keys(&:to_s)) if @options.call_credentials
 
           build_request("POST", uri, headers: headers, body: input)
         end