httpx 1.6.3 → 1.7.1

data/lib/httpx/options.rb

@@ -12,6 +12,7 @@ module HTTPX
     CLOSE_HANDSHAKE_TIMEOUT = 10
     CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
     REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
+    RESOLVER_TYPES = %i[memory file].freeze
 
     # default value used for "user-agent" header, when not overridden.
     USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
@@ -44,7 +45,7 @@ module HTTPX
 
         return unless meth =~ /^option_(.+)$/
 
-        optname = Regexp.last_match(1)
+        optname = Regexp.last_match(1) #: String
 
         if optname =~ /^(.+[^_])_+with/
           # ignore alias method chain generated methods.
@@ -52,14 +53,14 @@ module HTTPX
           # it relies on the "_with/_without" separator, which is the most used convention,
           # however it shouldn't be used in practice in httpx given the plugin architecture
           # as the main extension API.
-          orig_name = Regexp.last_match(1)
+          orig_name = Regexp.last_match(1) #: String
 
           return if @options_names.include?(orig_name.to_sym)
         end
 
         optname = optname.to_sym
 
-        attr_reader(optname)
+        attr_reader(optname) unless method_defined?(optname)
 
         @options_names << optname unless @options_names.include?(optname)
       end
@@ -103,7 +104,10 @@ module HTTPX
     # :io :: open socket, or domain/ip-to-socket hash, which requests should be sent to
     # :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
     # :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
-    #                    using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class)
+    #                    using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class inheriting
+    #                    from HTTPX::Resolver::Resolver)
+    # :resolver_cache :: strategy to cache DNS results, ignored by the <tt>:system</tt> resolver, can be set to <tt>:memory<tt>
+    #                    or an instance of a custom class inheriting from HTTPX::Resolver::Cache::Base
     # :resolver_options :: hash of options passed to the resolver. Accepted keys depend on the resolver type.
     # :pool_options :: hash of options passed to the connection pool (See Pool#initialize).
     # :ip_families :: which socket families are supported (system-dependent)
@@ -151,6 +155,36 @@ module HTTPX
       freeze
     end
 
+    # returns the class with which to instantiate the DNS resolver.
+    def resolver_class
+      case @resolver_class
+      when Symbol
+        public_send(:"resolver_#{@resolver_class}_class")
+      else
+        @resolver_class
+      end
+    end
+
+    def resolver_cache
+      cache_type = @resolver_cache
+
+      case cache_type
+      when :memory
+        Resolver::Cache::Memory.cache(cache_type)
+      when :file
+        Resolver::Cache::File.cache(cache_type)
+      else
+        unless cache_type.respond_to?(:resolve) &&
+               cache_type.respond_to?(:get) &&
+               cache_type.respond_to?(:set) &&
+               cache_type.respond_to?(:evict)
+          raise TypeError, ":resolver_cache must be a compatible resolver cache and implement #get, #set and #evict"
+        end
+
+        cache_type #: Object & Resolver::_Cache
+      end
+    end
+
     def freeze
       self.class.options_names.each do |ivar|
         # avoid freezing debug option, as when it's set, it's usually an
@@ -172,6 +206,7 @@ module HTTPX
       super || options_equals?(other)
     end
 
+    # checks whether +other+ is equal by comparing the session options
     def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
       # headers and other request options do not play a role, as they are
       # relevant only for the request.
@@ -187,34 +222,43 @@ module HTTPX
       end
     end
 
+    # returns a HTTPX::Options instance resulting of the merging of +other+ with self.
+    # it may return self if +other+ is self or equal to self.
     def merge(other)
-      ivar_map = nil
-      other_ivars = case other
-                    when Options
-                      other.instance_variables
-                    else
-                      other = Hash[other] unless other.is_a?(Hash)
-                      ivar_map = other.keys.to_h { |k| [:"@#{k}", k] }
-                      ivar_map.keys
-      end
+      if (is_options = other.is_a?(Options))
+
+        return self if eql?(other)
+
+        opts_names = other.class.options_names
+
+        return self if opts_names.all? { |opt| public_send(opt) == other.public_send(opt) }
+
+        other_opts = opts_names
+      else
+        other_opts = other # : Hash[Symbol, untyped]
+        other_opts = Hash[other] unless other.is_a?(Hash)
 
-      return self if other_ivars.empty?
+        return self if other_opts.empty?
 
-      return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == access_option(other, ivar, ivar_map) }
+        return self if other_opts.all? { |opt, v| !respond_to?(opt) || public_send(opt) == v }
+      end
 
       opts = dup
 
-      other_ivars.each do |ivar|
-        v = access_option(other, ivar, ivar_map)
+      other_opts.each do |opt, v|
+        next unless respond_to?(opt)
+
+        v = other.public_send(opt) if is_options
+        ivar = :"@#{opt}"
 
         unless v
           opts.instance_variable_set(ivar, v)
           next
         end
 
-        v = opts.__send__(:"option_#{ivar[1..-1]}", v)
+        v = opts.__send__(:"option_#{opt}", v)
 
-        orig_v = instance_variable_get(ivar)
+        orig_v = public_send(opt)
 
         v = orig_v.merge(v) if orig_v.respond_to?(:merge) && v.respond_to?(:merge)
 
@@ -354,7 +398,7 @@ module HTTPX
       request_class response_class headers_class request_body_class
       response_body_class connection_class http1_class http2_class
       resolver_native_class resolver_system_class resolver_https_class options_class pool_class
-      io fallback_protocol debug debug_redact resolver_class
+      io fallback_protocol debug debug_redact
       compress_request_body decompress_response_body
       persistent close_on_fork
     ].each do |method_name|
@@ -379,7 +423,20 @@ module HTTPX
     end
 
     def option_timeout(value)
-      Hash[value]
+      timeout_hash = Hash[value]
+
+      default_timeouts = DEFAULT_OPTIONS[:timeout]
+
+      # Validate keys and values
+      timeout_hash.each do |key, val|
+        raise TypeError, "invalid timeout: :#{key}" unless default_timeouts.key?(key)
+
+        next if val.nil?
+
+        raise TypeError, ":#{key} must be numeric" unless val.is_a?(Numeric)
+      end
+
+      timeout_hash
     end
 
     def option_supported_compression_formats(value)
@@ -401,6 +458,41 @@ module HTTPX
       Array(value)
     end
 
+    def option_resolver_class(resolver_type)
+      case resolver_type
+      when Symbol
+        meth = :"resolver_#{resolver_type}_class"
+
+        raise TypeError, ":resolver_class must be a supported type" unless respond_to?(meth)
+
+        resolver_type
+      when Class
+        raise TypeError, ":resolver_class must be a subclass of `#{Resolver::Resolver}`" unless resolver_type < Resolver::Resolver
+
+        resolver_type
+      else
+        raise TypeError, ":resolver_class must be a supported type"
+      end
+    end
+
+    def option_resolver_cache(cache_type)
+      if cache_type.is_a?(Symbol)
+        raise TypeError, ":resolver_cache: #{cache_type} is invalid" unless RESOLVER_TYPES.include?(cache_type)
+
+        require "httpx/resolver/cache/file" if cache_type == :file
+
+      else
+        unless cache_type.respond_to?(:resolve) &&
+               cache_type.respond_to?(:get) &&
+               cache_type.respond_to?(:set) &&
+               cache_type.respond_to?(:evict)
+          raise TypeError, ":resolver_cache must be a compatible resolver cache and implement #resolve, #get, #set and #evict"
+        end
+      end
+
+      cache_type
+    end
+
     # called after all options are initialized
     def do_initialize
       hs = @headers
@@ -424,6 +516,8 @@ module HTTPX
       end
     end
 
+    # rubocop:disable Lint/UselessConstantScoping
+    # these really need to be defined at the end of the class
     SET_TEMPORARY_NAME = ->(klass, pl = nil) do
       if klass.respond_to?(:set_temporary_name) # ruby 3.4 only
         name = klass.name || "#{klass.superclass.name}(plugin)"
@@ -474,10 +568,12 @@ module HTTPX
       :addresses => nil,
       :persistent => false,
       :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
+      :resolver_cache => (ENV["HTTPX_RESOLVER_CACHE"] || :memory).to_sym,
       :resolver_options => { cache: true }.freeze,
       :pool_options => EMPTY_HASH,
       :ip_families => nil,
       :close_on_fork => false,
-    }.freeze
+    }.each_value(&:freeze).freeze
+    # rubocop:enable Lint/UselessConstantScoping
   end
 end

data/lib/httpx/parser/http1.rb

@@ -26,6 +26,7 @@ module HTTPX
         @headers = {}
         @content_length = nil
         @_has_trailers = nil
+        @buffer.clear
       end
 
       def upgrade?

data/lib/httpx/plugins/auth/digest.rb

@@ -8,6 +8,8 @@ module HTTPX
   module Plugins
     module Authentication
       class Digest
+        Error = Class.new(Error)
+
         def initialize(user, password, hashed: false, **)
           @user = user
           @password = password
@@ -29,19 +31,53 @@ module HTTPX
           # discard first token, it's Digest
           auth_info = authenticate[/^(\w+) (.*)/, 2]
 
-          params = auth_info.split(/ *, */)
-                            .to_h { |val| val.split("=", 2) }
-                            .transform_values { |v| v.delete("\"") }
+          raise_format_error unless auth_info
+
+          s = StringScanner.new(auth_info)
+
+          params = {}
+          until s.eos?
+            k = s.scan_until(/=/)
+            raise_format_error unless k&.end_with?("=")
+
+            if s.peek(1) == "\""
+              s.skip("\"")
+              v = s.scan_until(/"/)
+              raise_format_error unless v&.end_with?("\"")
+
+              v = v[0..-2]
+              s.skip_until(/,/)
+            else
+              v = s.scan_until(/,|$/)
+
+              if v&.end_with?(",")
+                v = v[0..-2]
+              else
+                raise_format_error unless s.eos?
+              end
+
+              v = v[0..-2] if v&.end_with?(",")
+            end
+            params[k[0..-2]] = v
+            s.skip(/\s/)
+          end
+
           nonce = params["nonce"]
           nc = next_nonce
 
           # verify qop
           qop = params["qop"]
 
+          if qop
+            # some servers send multiple values wrapped in parentheses (i.e. "(qauth,)")
+            qop = qop[/\(?([^)]+)\)?/, 1]
+            qop = qop.split(",").map { |s| s.delete_prefix("'").delete_suffix("'") }.delete_if(&:empty?).map.first
+          end
+
           if params["algorithm"] =~ /(.*?)(-sess)?$/
             alg = Regexp.last_match(1)
             algorithm = ::Digest.const_get(alg)
-            raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
+            raise Error, "unknown algorithm \"#{alg}\"" unless algorithm
 
             sess = Regexp.last_match(2)
           else
@@ -96,6 +132,10 @@ module HTTPX
         def next_nonce
           @nonce += 1
         end
+
+        def raise_format_error
+          raise Error, "unsupported digest header format"
+        end
       end
     end
   end

data/lib/httpx/plugins/auth.rb

@@ -10,13 +10,122 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Auth#auth
     #
     module Auth
+      def self.subplugins
+        {
+          retries: AuthRetries,
+        }
+      end
+
+      # adds support for the following options:
+      #
+      # :auth_header_value :: the token to use as a string, or a callable which returns a string when called.
+      # :auth_header_type :: the authentication type to use in the "authorization" header value (i.e. "Bearer", "Digest"...)
+      # :generate_auth_value_on_retry :: callable which returns whether the request should regenerate the auth_header_value
+      #                                  when the request is retried (this option will only work if the session also loads the
+      #                                  <tt>:retries</tt> plugin).
+      module OptionsMethods
+        def option_auth_header_value(value)
+          value
+        end
+
+        def option_auth_header_type(value)
+          value
+        end
+
+        def option_generate_auth_value_on_retry(value)
+          raise TypeError, "`:generate_auth_value_on_retry` must be a callable" unless value.respond_to?(:call)
+
+          value
+        end
+      end
+
       module InstanceMethods
-        def authorization(token)
-          with(headers: { "authorization" => token })
+        def initialize(*)
+          super
+
+          @auth_header_value = nil
+          @skip_auth_header_value = false
+        end
+
+        def authorization(token = nil, auth_header_type: nil, &blk)
+          with(auth_header_type: auth_header_type, auth_header_value: token || blk)
+        end
+
+        def bearer_auth(token = nil, &blk)
+          authorization(token, auth_header_type: "Bearer", &blk)
+        end
+
+        def skip_auth_header
+          @skip_auth_header_value = true
+          yield
+        ensure
+          @skip_auth_header_value = false
+        end
+
+        def reset_auth_header_value!
+          @auth_header_value = nil
+        end
+
+        private
+
+        def send_request(request, *)
+          return super if @skip_auth_header_value
+
+          @auth_header_value ||= generate_auth_token
+
+          request.authorize(@auth_header_value) if @auth_header_value
+
+          super
         end
 
-        def bearer_auth(token)
-          authorization("Bearer #{token}")
+        def generate_auth_token
+          return unless (auth_value = @options.auth_header_value)
+
+          auth_value = auth_value.call(self) if dynamic_auth_token?(auth_value)
+
+          auth_value
+        end
+
+        def dynamic_auth_token?(auth_header_value)
+          auth_header_value&.respond_to?(:call)
+        end
+      end
+
+      module RequestMethods
+        def authorize(auth_value)
+          if (auth_type = @options.auth_header_type)
+            auth_value = "#{auth_type} #{auth_value}"
+          end
+
+          @headers.add("authorization", auth_value)
+        end
+      end
+
+      module AuthRetries
+        module InstanceMethods
+          private
+
+          def retryable_request?(request, response, options)
+            super || auth_error?(response, options)
+          end
+
+          def retryable_response?(response, options)
+            auth_error?(response, options) || super
+          end
+
+          def prepare_to_retry(request, response)
+            super
+
+            return unless auth_error?(response, request.options) ||
+                          (@options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response))
+
+            request.headers.get("authorization").pop
+            @auth_header_value = generate_auth_token
+          end
+
+          def auth_error?(response, options)
+            response.is_a?(Response) && response.status == 401 && dynamic_auth_token?(options.auth_header_value)
+          end
         end
       end
     end

data/lib/httpx/plugins/aws_sdk_authentication.rb

@@ -94,7 +94,6 @@ module HTTPX
           region: AwsSdkAuthentication.region(@options.aws_profile),
           **options
         )
-
           aws_sigv4_authentication(
             credentials: credentials,
             region: region,

data/lib/httpx/plugins/cookies/cookie.rb

@@ -7,6 +7,7 @@ module HTTPX
     # Contains the single cookie info: name, value and attributes.
     class Cookie
       include Comparable
+
       # Maximum number of bytes per cookie (RFC 6265 6.1 requires 4096 at
       # least)
       MAX_LENGTH = 4096

data/lib/httpx/plugins/digest_auth.rb

@@ -8,15 +8,14 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Auth#digest-auth
     #
     module DigestAuth
-      DigestError = Class.new(Error)
-
       class << self
         def extra_options(options)
           options.merge(max_concurrent_requests: 1)
         end
 
-        def load_dependencies(*)
+        def load_dependencies(klass)
           require_relative "auth/digest"
+          klass.plugin(:auth)
         end
       end
 
@@ -48,11 +47,11 @@ module HTTPX
 
             probe_response = wrap { super(request).first }
 
-            return ([probe_response] * requests.size) unless probe_response.is_a?(Response)
+            return [probe_response] * requests.size unless probe_response.is_a?(Response)
 
             if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
               request.transition(:idle)
-              request.headers["authorization"] = digest.authenticate(request, probe_response.headers["www-authenticate"])
+              request.authorize(digest.authenticate(request, probe_response.headers["www-authenticate"]))
               super(request)
             else
               probe_response

data/lib/httpx/plugins/fiber_concurrency.rb

@@ -6,12 +6,13 @@ module HTTPX
     #
     # This enables integration with fiber scheduler implementations such as [async](https://github.com/async).
     #
-    # # https://gitlab.com/os85/httpx/wikis/FiberConcurrency
+    # # https://gitlab.com/os85/httpx/wikis/Fiber-Concurrency
     #
     module FiberConcurrency
       def self.subplugins
         {
           h2c: FiberConcurrencyH2C,
+          stream: FiberConcurrencyStream,
         }
       end
 
@@ -188,6 +189,20 @@ module HTTPX
           end
         end
       end
+
+      module FiberConcurrencyStream
+        module StreamResponseMethods
+          def close
+            unless @request.current_context?
+              @request.close
+
+              return
+            end
+
+            super
+          end
+        end
+      end
     end
 
     register_plugin :fiber_concurrency, FiberConcurrency

data/lib/httpx/plugins/grpc/grpc_encoding.rb

@@ -48,7 +48,7 @@ module HTTPX
           until message.empty?
             compressed, size = message.unpack("CL>")
 
-            encoded_data = message.byteslice(5..size + 5 - 1)
+            encoded_data = message.byteslice(5..(size + 5 - 1))
 
             if compressed == 1
               grpc_encodings.reverse_each do |encoding|

data/lib/httpx/plugins/grpc.rb

@@ -249,7 +249,7 @@ module HTTPX
           call
         end
 
-        def build_grpc_request(rpc_method, input, deadline:, metadata: nil, **)
+        def build_grpc_request(rpc_method, input, deadline:, metadata: nil, **opts)
           uri = @options.origin.dup
           rpc_method = "/#{rpc_method}" unless rpc_method.start_with?("/")
           rpc_method = "/#{@options.grpc_service}#{rpc_method}" if @options.grpc_service
@@ -273,7 +273,7 @@ module HTTPX
 
           headers.merge!(@options.call_credentials.call.transform_keys(&:to_s)) if @options.call_credentials
 
-          build_request("POST", uri, headers: headers, body: input)
+          build_request("POST", uri, headers: headers, body: input, **opts)
         end
       end
     end

data/lib/httpx/plugins/internal_telemetry.rb

@@ -45,7 +45,7 @@ module HTTPX
             debug_level: @options ? @options.debug_level : DEBUG_LEVEL,
             debug: nil
           ) do
-            "[ELAPSED TIME]: #{label}: #{elapsed} (ms)" << "\e[0m"
+            "[ELAPSED TIME]: #{label}: #{elapsed} (ms)\e[0m"
           end
         end
       end

data/lib/httpx/plugins/ntlm_auth.rb

@@ -7,8 +7,9 @@ module HTTPX
     #
     module NTLMAuth
       class << self
-        def load_dependencies(_klass)
+        def load_dependencies(klass)
           require_relative "auth/ntlm"
+          klass.plugin(:auth)
         end
 
         def extra_options(options)
@@ -38,14 +39,15 @@ module HTTPX
             ntlm = request.options.ntlm
 
             if ntlm
-              request.headers["authorization"] = ntlm.negotiate
+              request.authorize(ntlm.negotiate)
               probe_response = wrap { super(request).first }
 
               return probe_response unless probe_response.is_a?(Response)
 
               if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
                 request.transition(:idle)
-                request.headers["authorization"] = ntlm.authenticate(request, probe_response.headers["www-authenticate"])
+                request.headers.get("authorization").pop
+                request.authorize(ntlm.authenticate(request, probe_response.headers["www-authenticate"]).encode("utf-8"))
                 super(request)
               else
                 probe_response