httpx 1.1.5 → 1.2.0

data/lib/httpx/options.rb

@@ -11,6 +11,7 @@ module HTTPX
     MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
     KEEP_ALIVE_TIMEOUT = 20
     SETTINGS_TIMEOUT = 10
+    CLOSE_HANDSHAKE_TIMEOUT = 10
     CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
     REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
 
@@ -39,6 +40,7 @@ module HTTPX
       :timeout => {
         connect_timeout: CONNECT_TIMEOUT,
         settings_timeout: SETTINGS_TIMEOUT,
+        close_handshake_timeout: CLOSE_HANDSHAKE_TIMEOUT,
         operation_timeout: OPERATION_TIMEOUT,
         keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
         read_timeout: READ_TIMEOUT,
@@ -226,44 +228,69 @@ module HTTPX
       OUT
     end
 
-    REQUEST_IVARS = %i[@params @form @xml @json @body].freeze
-    private_constant :REQUEST_IVARS
+    REQUEST_BODY_IVARS = %i[@headers @params @form @xml @json @body].freeze
 
     def ==(other)
-      ivars = instance_variables | other.instance_variables
+      super || options_equals?(other)
+    end
+
+    def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
+      # headers and other request options do not play a role, as they are
+      # relevant only for the request.
+      ivars = instance_variables - ignore_ivars
+      other_ivars = other.instance_variables - ignore_ivars
+
+      return false if ivars.size != other_ivars.size
+
+      return false if ivars.sort != other_ivars.sort
+
       ivars.all? do |ivar|
-        case ivar
-        when :@headers
-          # currently, this is used to pick up an available matching connection.
-          # the headers do not play a role, as they are relevant only for the request.
-          true
-        when *REQUEST_IVARS
-          true
-        else
-          instance_variable_get(ivar) == other.instance_variable_get(ivar)
-        end
+        instance_variable_get(ivar) == other.instance_variable_get(ivar)
       end
     end
 
+    OTHER_LOOKUP = ->(obj, k, ivar_map) {
+      case obj
+      when Hash
+        obj[ivar_map[k]]
+      else
+        obj.instance_variable_get(k)
+      end
+    }
     def merge(other)
-      raise ArgumentError, "#{other} is not a valid set of options" unless other.respond_to?(:to_hash)
+      ivar_map = nil
+      other_ivars = case other
+                    when Hash
+                      ivar_map = other.keys.to_h { |k| [:"@#{k}", k] }
+                      ivar_map.keys
+                    else
+                      other.instance_variables
+      end
+
+      return self if other_ivars.empty?
 
-      h2 = other.to_hash
-      return self if h2.empty?
+      return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == OTHER_LOOKUP[other, ivar, ivar_map] }
 
-      h1 = to_hash
+      opts = dup
 
-      return self if h1 >= h2
+      other_ivars.each do |ivar|
+        v = OTHER_LOOKUP[other, ivar, ivar_map]
 
-      merged = h1.merge(h2) do |_k, v1, v2|
-        if v1.respond_to?(:merge) && v2.respond_to?(:merge)
-          v1.merge(v2)
-        else
-          v2
+        unless v
+          opts.instance_variable_set(ivar, v)
+          next
         end
+
+        v = opts.__send__(:"option_#{ivar[1..-1]}", v)
+
+        orig_v = instance_variable_get(ivar)
+
+        v = orig_v.merge(v) if orig_v.respond_to?(:merge) && v.respond_to?(:merge)
+
+        opts.instance_variable_set(ivar, v)
       end
 
-      self.class.new(merged)
+      opts
     end
 
     def to_hash
@@ -272,10 +299,40 @@ module HTTPX
       end
     end
 
-    def initialize_dup(other)
-      instance_variables.each do |ivar|
-        instance_variable_set(ivar, other.instance_variable_get(ivar).dup)
+    def extend_with_plugin_classes(pl)
+      if defined?(pl::RequestMethods) || defined?(pl::RequestClassMethods)
+        @request_class = @request_class.dup
+        @request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
+        @request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
+      end
+      if defined?(pl::ResponseMethods) || defined?(pl::ResponseClassMethods)
+        @response_class = @response_class.dup
+        @response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
+        @response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
       end
+      if defined?(pl::HeadersMethods) || defined?(pl::HeadersClassMethods)
+        @headers_class = @headers_class.dup
+        @headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
+        @headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
+      end
+      if defined?(pl::RequestBodyMethods) || defined?(pl::RequestBodyClassMethods)
+        @request_body_class = @request_body_class.dup
+        @request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
+        @request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
+      end
+      if defined?(pl::ResponseBodyMethods) || defined?(pl::ResponseBodyClassMethods)
+        @response_body_class = @response_body_class.dup
+        @response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
+        @response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
+      end
+      if defined?(pl::ConnectionMethods)
+        @connection_class = @connection_class.dup
+        @connection_class.__send__(:include, pl::ConnectionMethods)
+      end
+      return unless defined?(pl::OptionsMethods)
+
+      @options_class = @options_class.dup
+      @options_class.__send__(:include, pl::OptionsMethods)
     end
 
     private

data/lib/httpx/plugins/aws_sigv4.rb

@@ -5,7 +5,7 @@ module HTTPX
     #
     # This plugin adds AWS Sigv4 authentication.
     #
-    # https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
+    # https://docs.aws.amazon.com/IAM/latest/UserGuide/signing-elements.html
     #
     # https://gitlab.com/os85/httpx/wikis/AWS-SigV4
     #
@@ -185,7 +185,7 @@ module HTTPX
         def canonical_query
           params = query.split("&")
           # params = params.map { |p| p.match(/=/) ? p : p + '=' }
-          # From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+          # From: https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html#create-canonical-request
           # Sort the parameter names by character code point in ascending order.
           # Parameters with duplicate names should be sorted by value.
           #

data/lib/httpx/plugins/basic_auth.rb

@@ -3,7 +3,7 @@
 module HTTPX
   module Plugins
     #
-    # This plugin adds helper methods to implement HTTP Basic Auth (https://tools.ietf.org/html/rfc7617)
+    # This plugin adds helper methods to implement HTTP Basic Auth (https://datatracker.ietf.org/doc/html/rfc7617)
     #
     # https://gitlab.com/os85/httpx/wikis/Auth#basic-auth
     #

data/lib/httpx/plugins/callbacks.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds suppoort for callbacks around the request/response lifecycle.
+    #
+    # https://gitlab.com/os85/httpx/-/wikis/Events
+    #
+    module Callbacks
+      # connection closed user-space errors happen after errors can be surfaced to requests,
+      # so they need to pierce through the scheduler, which is only possible by simulating an
+      # interrupt.
+      class CallbackError < Exception; end # rubocop:disable Lint/InheritException
+
+      module InstanceMethods
+        include HTTPX::Callbacks
+
+        %i[
+          connection_opened connection_closed
+          request_error
+          request_started request_body_chunk request_completed
+          response_started response_body_chunk response_completed
+        ].each do |meth|
+          class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+            def on_#{meth}(&blk)   # def on_connection_opened(&blk)
+              on(:#{meth}, &blk)   #   on(:connection_opened, &blk)
+            end                    # end
+          MOD
+        end
+
+        private
+
+        def init_connection(uri, options)
+          connection = super
+          connection.on(:open) do
+            emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
+          end
+          connection.on(:close) do
+            emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
+          end
+
+          connection
+        end
+
+        def set_request_callbacks(request)
+          super
+
+          request.on(:headers) do
+            emit_or_callback_error(:request_started, request)
+          end
+          request.on(:body_chunk) do |chunk|
+            emit_or_callback_error(:request_body_chunk, request, chunk)
+          end
+          request.on(:done) do
+            emit_or_callback_error(:request_completed, request)
+          end
+
+          request.on(:response_started) do |res|
+            if res.is_a?(Response)
+              emit_or_callback_error(:response_started, request, res)
+              res.on(:chunk_received) do |chunk|
+                emit_or_callback_error(:response_body_chunk, request, res, chunk)
+              end
+            else
+              emit_or_callback_error(:request_error, request, res.error)
+            end
+          end
+          request.on(:response) do |res|
+            emit_or_callback_error(:response_completed, request, res)
+          end
+        end
+
+        def emit_or_callback_error(*args)
+          emit(*args)
+        rescue StandardError => e
+          ex = CallbackError.new(e.message)
+          ex.set_backtrace(e.backtrace)
+          raise ex
+        end
+
+        def receive_requests(*)
+          super
+        rescue CallbackError => e
+          raise e.cause
+        end
+      end
+    end
+    register_plugin :callbacks, Callbacks
+  end
+end

data/lib/httpx/plugins/circuit_breaker.rb

@@ -25,6 +25,8 @@ module HTTPX
       end
 
       module InstanceMethods
+        include HTTPX::Callbacks
+
         def initialize(*)
           super
           @circuit_store = CircuitStore.new(@options)

data/lib/httpx/plugins/cookies.rb

@@ -71,22 +71,32 @@ module HTTPX
       end
 
       module OptionsMethods
-        def do_initialize(*)
-          super
+        def option_headers(*)
+          value = super
+
+          merge_cookie_in_jar(value.delete("cookie"), @cookies) if defined?(@cookies) && value.key?("cookie")
 
-          return unless @headers.key?("cookie")
+          value
+        end
 
-          @headers.delete("cookie").each do |ck|
+        def option_cookies(value)
+          jar = value.is_a?(Jar) ? value : Jar.new(value)
+
+          merge_cookie_in_jar(@headers.delete("cookie"), jar) if defined?(@headers) && @headers.key?("cookie")
+
+          jar
+        end
+
+        private
+
+        def merge_cookie_in_jar(cookies, jar)
+          cookies.each do |ck|
             ck.split(/ *; */).each do |cookie|
               name, value = cookie.split("=", 2)
-              @cookies.add(Cookie.new(name, value))
+              jar.add(Cookie.new(name, value))
             end
           end
         end
-
-        def option_cookies(value)
-          value.is_a?(Jar) ? value : Jar.new(value)
-        end
       end
     end
     register_plugin :cookies, Cookies

data/lib/httpx/plugins/digest_auth.rb

@@ -3,7 +3,7 @@
 module HTTPX
   module Plugins
     #
-    # This plugin adds helper methods to implement HTTP Digest Auth (https://tools.ietf.org/html/rfc7616)
+    # This plugin adds helper methods to implement HTTP Digest Auth (https://datatracker.ietf.org/doc/html/rfc7616)
     #
     # https://gitlab.com/os85/httpx/wikis/Auth#digest-auth
     #

data/lib/httpx/plugins/follow_redirects.rb

@@ -34,6 +34,12 @@ module HTTPX
         def option_allow_auth_to_other_origins(value)
           value
         end
+
+        def option_redirect_on(value)
+          raise TypeError, ":redirect_on must be callable" unless value.respond_to?(:call)
+
+          value
+        end
       end
 
       module InstanceMethods
@@ -57,6 +63,11 @@ module HTTPX
           # build redirect request
           redirect_uri = __get_location_from_response(response)
 
+          if options.redirect_on
+            redirect_allowed = options.redirect_on.call(redirect_uri)
+            return response unless redirect_allowed
+          end
+
           if response.status == 305 && options.respond_to?(:proxy)
             # The requested resource MUST be accessed through the proxy given by
             # the Location field. The Location field gives the URI of the proxy.

data/lib/httpx/plugins/grpc.rb

@@ -261,14 +261,14 @@ module HTTPX
             headers["grpc-timeout"] = "#{deadline}m"
           end
 
-          headers = headers.merge(metadata) if metadata
+          headers = headers.merge(metadata.transform_keys(&:to_s)) if metadata
 
           # prepare compressor
           compression = @options.grpc_compression == true ? "gzip" : @options.grpc_compression
 
           headers["grpc-encoding"] = compression if compression
 
-          headers.merge!(@options.call_credentials.call) if @options.call_credentials
+          headers.merge!(@options.call_credentials.call.transform_keys(&:to_s)) if @options.call_credentials
 
           build_request("POST", uri, headers: headers, body: input)
         end

data/lib/httpx/plugins/h2c.rb

@@ -4,9 +4,9 @@ module HTTPX
   module Plugins
     #
     # This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
-    # (https://tools.ietf.org/html/rfc7540#section-3.2)
+    # (https://datatracker.ietf.org/doc/html/rfc7540#section-3.2)
     #
-    # https://gitlab.com/os85/httpx/wikis/Upgrade#h2c
+    # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2c
     #
     module H2C
       VALID_H2C_VERBS = %w[GET OPTIONS HEAD].freeze
@@ -73,22 +73,34 @@ module HTTPX
             @inflight -= prev_parser.requests.size
           end
 
-          parser_options = @options.merge(max_concurrent_requests: request.options.max_concurrent_requests)
-          @parser = H2CParser.new(@write_buffer, parser_options)
+          @parser = H2CParser.new(@write_buffer, @options)
           set_parser_callbacks(@parser)
           @inflight += 1
           @parser.upgrade(request, response)
           @upgrade_protocol = "h2c"
 
-          if request.options.max_concurrent_requests != @options.max_concurrent_requests
-            @options = @options.merge(max_concurrent_requests: nil)
-          end
-
           prev_parser.requests.each do |req|
             req.transition(:idle)
             send(req)
           end
         end
+
+        private
+
+        def send_request_to_parser(request)
+          super
+
+          return unless request.headers["upgrade"] == "h2c" && parser.is_a?(Connection::HTTP1)
+
+          max_concurrent_requests = parser.max_concurrent_requests
+
+          return if max_concurrent_requests == 1
+
+          parser.max_concurrent_requests = 1
+          request.once(:response) do
+            parser.max_concurrent_requests = max_concurrent_requests
+          end
+        end
       end
     end
     register_plugin(:h2c, H2C)

data/lib/httpx/plugins/proxy/socks4.rb

@@ -4,7 +4,7 @@ require "resolv"
 require "ipaddr"
 
 module HTTPX
-  class Socks4Error < Error; end
+  class Socks4Error < HTTPProxyError; end
 
   module Plugins
     module Proxy
@@ -85,7 +85,7 @@ module HTTPX
         end
 
         class SocksParser
-          include Callbacks
+          include HTTPX::Callbacks
 
           def initialize(buffer, options)
             @buffer = buffer

data/lib/httpx/plugins/proxy/socks5.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  class Socks5Error < Error; end
+  class Socks5Error < HTTPProxyError; end
 
   module Plugins
     module Proxy
@@ -137,7 +137,7 @@ module HTTPX
         end
 
         class SocksParser
-          include Callbacks
+          include HTTPX::Callbacks
 
           def initialize(buffer, options)
             @buffer = buffer

data/lib/httpx/plugins/proxy.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  class HTTPProxyError < Error; end
+  class HTTPProxyError < ConnectionError; end
 
   module Plugins
     #
@@ -143,7 +143,7 @@ module HTTPX
           proxy = Parameters.new(**proxy_opts)
 
           proxy_options = options.merge(proxy: proxy)
-          connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options)
+          connection = pool.find_connection(uri, proxy_options) || init_connection(uri, proxy_options)
           unless connections.nil? || connections.include?(connection)
             connections << connection
             set_connection_callbacks(connection, connections, options)
@@ -151,19 +151,15 @@ module HTTPX
           connection
         end
 
-        def build_connection(uri, options)
-          proxy = options.proxy
-          return super unless proxy
-
-          init_connection("tcp", uri, options)
-        end
-
         def fetch_response(request, connections, options)
           response = super
 
-          if response.is_a?(ErrorResponse) &&
-             __proxy_error?(response) && !@_proxy_uris.empty?
+          if response.is_a?(ErrorResponse) && proxy_error?(request, response)
             @_proxy_uris.shift
+
+            # return last error response if no more proxies to try
+            return response if @_proxy_uris.empty?
+
             log { "failed connecting to proxy, trying next..." }
             request.transition(:idle)
             send_request(request, connections, options)
@@ -172,13 +168,7 @@ module HTTPX
           response
         end
 
-        def build_altsvc_connection(_, _, _, _, _, options)
-          return if options.proxy
-
-          super
-        end
-
-        def __proxy_error?(response)
+        def proxy_error?(_request, response)
           error = response.error
           case error
           when NativeResolveError
@@ -235,14 +225,6 @@ module HTTPX
           end
         end
 
-        def send(request)
-          return super unless (
-            @options.proxy && @state != :idle && connecting?
-          )
-
-          (@proxy_pending ||= []) << request
-        end
-
         def connecting?
           return super unless @options.proxy
 
@@ -271,6 +253,12 @@ module HTTPX
 
         private
 
+        def initialize_type(uri, options)
+          return super unless options.proxy
+
+          "tcp"
+        end
+
         def connect
           return super unless @options.proxy
 
@@ -278,12 +266,6 @@ module HTTPX
           when :idle
             transition(:connecting)
           when :connected
-            if @proxy_pending
-              while (req = @proxy_pendind.shift)
-                send(req)
-              end
-            end
-
             transition(:open)
           end
         end

data/lib/httpx/plugins/rate_limiter.rb

@@ -9,7 +9,7 @@ module HTTPX
     # * when the server is unavailable (503);
     # * when a 3xx request comes with a "retry-after" value
     #
-    # https://gitlab.com/os85/httpx/wikis/RateLimiter
+    # https://gitlab.com/os85/httpx/wikis/Rate-Limiter
     #
     module RateLimiter
       class << self

data/lib/httpx/plugins/retries.rb

@@ -132,6 +132,10 @@ module HTTPX
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
         end
 
+        def proxy_error?(request, response)
+          super && !request.retries.positive?
+        end
+
         #
         # Atttempt to set the request to perform a partial range request.
         # This happens if the peer server accepts byte-range requests, and