httpx 1.2.6 → 1.4.4

data/lib/httpx/options.rb

@@ -25,14 +25,14 @@ module HTTPX
       end
     rescue NotImplementedError
       [Socket::AF_INET]
-    end
+    end.freeze
 
     DEFAULT_OPTIONS = {
       :max_requests => Float::INFINITY,
-      :debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
+      :debug => nil,
       :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
-      :ssl => {},
-      :http2_settings => { settings_enable_push: 0 },
+      :ssl => EMPTY_HASH,
+      :http2_settings => { settings_enable_push: 0 }.freeze,
       :fallback_protocol => "http/1.1",
       :supported_compression_formats => %w[gzip deflate],
       :decompress_response_body => true,
@@ -56,13 +56,15 @@ module HTTPX
       :response_class => Class.new(Response),
       :request_body_class => Class.new(Request::Body),
       :response_body_class => Class.new(Response::Body),
+      :pool_class => Class.new(Pool),
       :connection_class => Class.new(Connection),
       :options_class => Class.new(self),
       :transport => nil,
       :addresses => nil,
       :persistent => false,
       :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
-      :resolver_options => { cache: true },
+      :resolver_options => { cache: true }.freeze,
+      :pool_options => EMPTY_HASH,
       :ip_families => ip_address_families,
     }.freeze
 
@@ -91,7 +93,7 @@ module HTTPX
     # :debug :: an object which log messages are written to (must respond to <tt><<</tt>)
     # :debug_level :: the log level of messages (can be 1, 2, or 3).
     # :ssl :: a hash of options which can be set as params of OpenSSL::SSL::SSLContext (see HTTPX::IO::SSL)
-    # :http2_settings :: a hash of options to be passed to a HTTP2Next::Connection (ex: <tt>{ max_concurrent_streams: 2 }</tt>)
+    # :http2_settings :: a hash of options to be passed to a HTTP2::Connection (ex: <tt>{ max_concurrent_streams: 2 }</tt>)
     # :fallback_protocol :: version of HTTP protocol to use by default in the absence of protocol negotiation
     #                       like ALPN (defaults to <tt>"http/1.1"</tt>)
     # :supported_compression_formats :: list of compressions supported by the transcoder layer (defaults to <tt>%w[gzip deflate]</tt>).
@@ -110,6 +112,7 @@ module HTTPX
     # :request_body_class :: class used to instantiate a request body
     # :response_body_class :: class used to instantiate a response body
     # :connection_class :: class used to instantiate connections
+    # :pool_class :: class used to instantiate the session connection pool
     # :options_class :: class used to instantiate options
     # :transport :: type of transport to use (set to "unix" for UNIX sockets)
     # :addresses :: bucket of peer addresses (can be a list of IP addresses, a hash of domain to list of adddresses;
@@ -118,16 +121,13 @@ module HTTPX
     # :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
     # :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
     #                    using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class)
-    # :resolver_options :: hash of options passed to the resolver
+    # :resolver_options :: hash of options passed to the resolver. Accepted keys depend on the resolver type.
+    # :pool_options :: hash of options passed to the connection pool (See Pool#initialize).
     # :ip_families :: which socket families are supported (system-dependent)
     # :origin :: HTTP origin to set on requests with relative path (ex: "https://api.serv.com")
     # :base_path :: path to prefix given relative paths with (ex: "/v2")
     # :max_concurrent_requests :: max number of requests which can be set concurrently
     # :max_requests :: max number of requests which can be made on socket before it reconnects.
-    # :params :: hash or array of key-values which will be encoded and set in the query string of request uris.
-    # :form :: hash of array of key-values which will be form-or-multipart-encoded in requests body payload.
-    # :json :: hash of array of key-values which will be JSON-encoded in requests body payload.
-    # :xml :: Nokogiri XML nodes which will be encoded in requests body payload.
     #
     # This list of options are enhanced with each loaded plugin, see the plugin docs for details.
     def initialize(options = {})
@@ -216,19 +216,21 @@ module HTTPX
     end
 
     %i[
-      params form json xml body ssl http2_settings
+      ssl http2_settings
       request_class response_class headers_class request_body_class
       response_body_class connection_class options_class
+      pool_class pool_options
       io fallback_protocol debug debug_level resolver_class resolver_options
       compress_request_body decompress_response_body
       persistent
     ].each do |method_name|
       class_eval(<<-OUT, __FILE__, __LINE__ + 1)
+        # sets +v+ as the value of #{method_name}
         def option_#{method_name}(v); v; end # def option_smth(v); v; end
       OUT
     end
 
-    REQUEST_BODY_IVARS = %i[@headers @params @form @xml @json @body].freeze
+    REQUEST_BODY_IVARS = %i[@headers].freeze
 
     def ==(other)
       super || options_equals?(other)
@@ -249,14 +251,6 @@ module HTTPX
       end
     end
 
-    OTHER_LOOKUP = ->(obj, k, ivar_map) {
-      case obj
-      when Hash
-        obj[ivar_map[k]]
-      else
-        obj.instance_variable_get(k)
-      end
-    }
     def merge(other)
       ivar_map = nil
       other_ivars = case other
@@ -269,12 +263,12 @@ module HTTPX
 
       return self if other_ivars.empty?
 
-      return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == OTHER_LOOKUP[other, ivar, ivar_map] }
+      return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == access_option(other, ivar, ivar_map) }
 
       opts = dup
 
       other_ivars.each do |ivar|
-        v = OTHER_LOOKUP[other, ivar, ivar_map]
+        v = access_option(other, ivar, ivar_map)
 
         unless v
           opts.instance_variable_set(ivar, v)
@@ -325,6 +319,10 @@ module HTTPX
         @response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
         @response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
       end
+      if defined?(pl::PoolMethods)
+        @pool_class = @pool_class.dup
+        @pool_class.__send__(:include, pl::PoolMethods)
+      end
       if defined?(pl::ConnectionMethods)
         @connection_class = @connection_class.dup
         @connection_class.__send__(:include, pl::ConnectionMethods)
@@ -349,5 +347,14 @@ module HTTPX
         instance_variable_set(:"@#{k}", value)
       end
     end
+
+    def access_option(obj, k, ivar_map)
+      case obj
+      when Hash
+        obj[ivar_map[k]]
+      else
+        obj.instance_variable_get(k)
+      end
+    end
   end
 end

data/lib/httpx/plugins/aws_sdk_authentication.rb

@@ -72,6 +72,9 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :aws_profile :: AWS account profile to retrieve credentials from.
       module OptionsMethods
         def option_aws_profile(value)
           String(value)

data/lib/httpx/plugins/aws_sigv4.rb

@@ -12,6 +12,7 @@ module HTTPX
     module AWSSigV4
       Credentials = Struct.new(:username, :password, :security_token)
 
+      # Signs requests using the AWS sigv4 signing.
       class Signer
         def initialize(
           service:,
@@ -88,7 +89,7 @@ module HTTPX
           sts = "#{algo_line}" \
                 "\n#{datetime}" \
                 "\n#{credential_scope}" \
-                "\n#{hexdigest(creq)}"
+                "\n#{OpenSSL::Digest.new(@algorithm).hexdigest(creq)}"
 
           # signature
           k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
@@ -109,22 +110,38 @@ module HTTPX
         private
 
         def hexdigest(value)
-          if value.respond_to?(:to_path)
-            # files, pathnames
-            OpenSSL::Digest.new(@algorithm).file(value.to_path).hexdigest
-          elsif value.respond_to?(:each)
-            digest = OpenSSL::Digest.new(@algorithm)
-
-            mb_buffer = value.each.with_object("".b) do |chunk, buffer|
-              buffer << chunk
-              break if buffer.bytesize >= 1024 * 1024
-            end
+          digest = OpenSSL::Digest.new(@algorithm)
 
-            digest.update(mb_buffer)
-            value.rewind
-            digest.hexdigest
+          if value.respond_to?(:read)
+            if value.respond_to?(:to_path)
+              # files, pathnames
+              digest.file(value.to_path).hexdigest
+            else
+              # gzipped request bodies
+              raise Error, "request body must be rewindable" unless value.respond_to?(:rewind)
+
+              buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+              begin
+                IO.copy_stream(value, buffer)
+                buffer.flush
+
+                digest.file(buffer.to_path).hexdigest
+              ensure
+                value.rewind
+                buffer.close
+                buffer.unlink
+              end
+            end
           else
-            OpenSSL::Digest.new(@algorithm).hexdigest(value)
+            # error on endless generators
+            raise Error, "hexdigest for endless enumerators is not supported" if value.unbounded_body?
+
+            mb_buffer = value.each.with_object("".b) do |chunk, b|
+              b << chunk
+              break if b.bytesize >= 1024 * 1024
+            end
+
+            digest.hexdigest(mb_buffer)
           end
         end
 
@@ -141,7 +158,6 @@ module HTTPX
         def load_dependencies(*)
           require "set"
           require "digest/sha2"
-          require "openssl"
         end
 
         def configure(klass)
@@ -149,6 +165,9 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :sigv4_signer :: instance of HTTPX::Plugins::AWSSigV4 used to sign requests.
       module OptionsMethods
         def option_sigv4_signer(value)
           value.is_a?(Signer) ? value : Signer.new(value)
@@ -160,7 +179,7 @@ module HTTPX
           with(sigv4_signer: Signer.new(**options))
         end
 
-        def build_request(*, _)
+        def build_request(*)
           request = super
 
           return request if request.headers.key?("authorization")

data/lib/httpx/plugins/callbacks.rb

@@ -25,18 +25,23 @@ module HTTPX
           class_eval(<<-MOD, __FILE__, __LINE__ + 1)
             def on_#{meth}(&blk)   # def on_connection_opened(&blk)
               on(:#{meth}, &blk)   #   on(:connection_opened, &blk)
+              self                 #   self
             end                    # end
           MOD
         end
 
         private
 
-        def init_connection(uri, options)
-          connection = super
+        def do_init_connection(connection, selector)
+          super
           connection.on(:open) do
+            next unless connection.current_session == self
+
             emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
           end
           connection.on(:close) do
+            next unless connection.current_session == self
+
             emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
           end
 
@@ -84,6 +89,12 @@ module HTTPX
         rescue CallbackError => e
           raise e.cause
         end
+
+        def close(*)
+          super
+        rescue CallbackError => e
+          raise e.cause
+        end
       end
     end
     register_plugin :callbacks, Callbacks

data/lib/httpx/plugins/circuit_breaker.rb

@@ -32,15 +32,11 @@ module HTTPX
           @circuit_store = CircuitStore.new(@options)
         end
 
-        def initialize_dup(orig)
-          super
-          @circuit_store = orig.instance_variable_get(:@circuit_store).dup
-        end
-
         %i[circuit_open].each do |meth|
           class_eval(<<-MOD, __FILE__, __LINE__ + 1)
         def on_#{meth}(&blk)   # def on_circuit_open(&blk)
           on(:#{meth}, &blk)   #   on(:circuit_open, &blk)
+          self                 #   self
         end                    # end
           MOD
         end
@@ -97,6 +93,16 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :circuit_breaker_max_attempts :: the number of attempts the circuit allows, before it is opened (defaults to <tt>3</tt>).
+      # :circuit_breaker_reset_attempts_in :: the time a circuit stays open at most, before it resets (defaults to <tt>60</tt>).
+      # :circuit_breaker_break_on :: callable defining an alternative rule for a response to break
+      #                              (i.e. <tt>->(res) { res.status == 429 } </tt>)
+      # :circuit_breaker_break_in :: the time that must elapse before an open circuit can transit to the half-open state
+      #                              (defaults to <tt><60</tt>).
+      # :circuit_breaker_half_open_drip_rate :: the rate of requests a circuit allows to be performed when in an half-open state
+      #                                         (defaults to <tt>1</tt>).
       module OptionsMethods
         def option_circuit_breaker_max_attempts(value)
           attempts = Integer(value)

data/lib/httpx/plugins/content_digest.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds `Content-Digest` headers to requests
+    # and can validate these headers on responses
+    #
+    # https://datatracker.ietf.org/doc/html/rfc9530
+    #
+    module ContentDigest
+      class Error < HTTPX::Error; end
+
+      # Error raised on response "content-digest" header validation.
+      class ValidationError < Error
+        attr_reader :response
+
+        def initialize(message, response)
+          super(message)
+          @response = response
+        end
+      end
+
+      class MissingContentDigestError < ValidationError; end
+      class InvalidContentDigestError < ValidationError; end
+
+      SUPPORTED_ALGORITHMS = {
+        "sha-256" => OpenSSL::Digest::SHA256,
+        "sha-512" => OpenSSL::Digest::SHA512,
+      }.freeze
+
+      class << self
+        def extra_options(options)
+          options.merge(encode_content_digest: true, validate_content_digest: false, content_digest_algorithm: "sha-256")
+        end
+      end
+
+      # add support for the following options:
+      #
+      # :content_digest_algorithm :: the digest algorithm to use. Currently supports `sha-256` and `sha-512`. (defaults to `sha-256`)
+      # :encode_content_digest :: whether a <tt>Content-Digest</tt> header should be computed for the request;
+      #                           can also be a callable object (i.e. <tt>->(req) { ... }</tt>, defaults to <tt>true</tt>)
+      # :validate_content_digest :: whether a <tt>Content-Digest</tt> header in the response should be validated;
+      #                             can also be a callable object (i.e. <tt>->(res) { ... }</tt>, defaults to <tt>false</tt>)
+      module OptionsMethods
+        def option_content_digest_algorithm(value)
+          raise TypeError, ":content_digest_algorithm must be one of 'sha-256', 'sha-512'" unless SUPPORTED_ALGORITHMS.key?(value)
+
+          value
+        end
+
+        def option_encode_content_digest(value)
+          value
+        end
+
+        def option_validate_content_digest(value)
+          value
+        end
+      end
+
+      module ResponseBodyMethods
+        attr_reader :content_digest_buffer
+
+        def initialize(response, options)
+          super
+
+          return unless response.headers.key?("content-digest")
+
+          should_validate = options.validate_content_digest
+          should_validate = should_validate.call(response) if should_validate.respond_to?(:call)
+
+          return unless should_validate
+
+          @content_digest_buffer = Response::Buffer.new(
+            threshold_size: @options.body_threshold_size,
+            bytesize: @length,
+            encoding: @encoding
+          )
+        end
+
+        def write(chunk)
+          @content_digest_buffer.write(chunk) if @content_digest_buffer
+          super
+        end
+
+        def close
+          if @content_digest_buffer
+            @content_digest_buffer.close
+            @content_digest_buffer = nil
+          end
+          super
+        end
+      end
+
+      module InstanceMethods
+        def build_request(*)
+          request = super
+
+          return request if request.empty?
+
+          return request if request.headers.key?("content-digest")
+
+          perform_encoding = @options.encode_content_digest
+          perform_encoding = perform_encoding.call(request) if perform_encoding.respond_to?(:call)
+
+          return request unless perform_encoding
+
+          digest = base64digest(request.body)
+          request.headers.add("content-digest", "#{@options.content_digest_algorithm}=:#{digest}:")
+
+          request
+        end
+
+        private
+
+        def fetch_response(request, _, _)
+          response = super
+          return response unless response.is_a?(Response)
+
+          perform_validation = @options.validate_content_digest
+          perform_validation = perform_validation.call(response) if perform_validation.respond_to?(:call)
+
+          validate_content_digest(response) if perform_validation
+
+          response
+        rescue ValidationError => e
+          ErrorResponse.new(request, e)
+        end
+
+        def validate_content_digest(response)
+          content_digest_header = response.headers["content-digest"]
+
+          raise MissingContentDigestError.new("response is missing a `content-digest` header", response) unless content_digest_header
+
+          digests = extract_content_digests(content_digest_header)
+
+          included_algorithms = SUPPORTED_ALGORITHMS.keys & digests.keys
+
+          raise MissingContentDigestError.new("unsupported algorithms: #{digests.keys.join(", ")}", response) if included_algorithms.empty?
+
+          content_buffer = response.body.content_digest_buffer
+
+          included_algorithms.each do |algorithm|
+            digest = SUPPORTED_ALGORITHMS.fetch(algorithm).new
+            digest_received = digests[algorithm]
+            digest_computed =
+              if content_buffer.respond_to?(:to_path)
+                content_buffer.flush
+                digest.file(content_buffer.to_path).base64digest
+              else
+                digest.base64digest(content_buffer.to_s)
+              end
+
+            raise InvalidContentDigestError.new("#{algorithm} digest does not match content",
+                                                response) unless digest_received == digest_computed
+          end
+        end
+
+        def extract_content_digests(header)
+          header.split(",").to_h do |entry|
+            algorithm, digest = entry.split("=", 2)
+            raise Error, "#{entry} is an invalid digest format" unless algorithm && digest
+
+            [algorithm, digest.byteslice(1..-2)]
+          end
+        end
+
+        def base64digest(body)
+          digest = SUPPORTED_ALGORITHMS.fetch(@options.content_digest_algorithm).new
+
+          if body.respond_to?(:read)
+            if body.respond_to?(:to_path)
+              digest.file(body.to_path).base64digest
+            else
+              raise ContentDigestError, "request body must be rewindable" unless body.respond_to?(:rewind)
+
+              buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+              begin
+                IO.copy_stream(body, buffer)
+                buffer.flush
+
+                digest.file(buffer.to_path).base64digest
+              ensure
+                body.rewind
+                buffer.close
+                buffer.unlink
+              end
+            end
+          else
+            raise ContentDigestError, "base64digest for endless enumerators is not supported" if body.unbounded_body?
+
+            buffer = "".b
+            body.each { |chunk| buffer << chunk }
+
+            digest.base64digest(buffer)
+          end
+        end
+      end
+    end
+    register_plugin :content_digest, ContentDigest
+  end
+end

data/lib/httpx/plugins/cookies.rb

@@ -40,6 +40,12 @@ module HTTPX
           end
         end
 
+        def build_request(*)
+          request = super
+          request.headers.set_cookie(request.options.cookies[request.uri])
+          request
+        end
+
         private
 
         def on_response(_request, response)
@@ -52,12 +58,6 @@ module HTTPX
 
           super
         end
-
-        def build_request(*, _)
-          request = super
-          request.headers.set_cookie(request.options.cookies[request.uri])
-          request
-        end
       end
 
       module HeadersMethods
@@ -70,6 +70,9 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :cookies :: cookie jar for the session (can be a Hash, an Array, an instance of HTTPX::Plugins::Cookies::CookieJar)
       module OptionsMethods
         def option_headers(*)
           value = super

data/lib/httpx/plugins/digest_auth.rb

@@ -20,6 +20,9 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :digest :: instance of HTTPX::Plugins::Authentication::Digest, used to authenticate requests in the session.
       module OptionsMethods
         def option_digest(value)
           raise TypeError, ":digest must be a #{Authentication::Digest}" unless value.is_a?(Authentication::Digest)

data/lib/httpx/plugins/expect.rb

@@ -20,6 +20,11 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :expect_timeout :: time (in seconds) to wait for a 100-expect response,
+      #                    before retrying without the Expect header (defaults to <tt>2</tt>).
+      # :expect_threshold_size :: min threshold (in bytes) of the request payload to enable the 100-continue negotiation on.
       module OptionsMethods
         def option_expect_timeout(value)
           seconds = Float(value)
@@ -79,7 +84,7 @@ module HTTPX
 
           return if expect_timeout.nil? || expect_timeout.infinite?
 
-          set_request_timeout(request, expect_timeout, :expect, %i[body response]) do
+          set_request_timeout(:expect_timeout, request, expect_timeout, :expect, %i[body response]) do
             # expect timeout expired
             if request.state == :expect && !request.expects?
               Expect.no_expect_store << request.origin
@@ -91,15 +96,16 @@ module HTTPX
       end
 
       module InstanceMethods
-        def fetch_response(request, connections, options)
-          response = @responses.delete(request)
+        def fetch_response(request, selector, options)
+          response = super
+
           return unless response
 
           if response.is_a?(Response) && response.status == 417 && request.headers.key?("expect")
             response.close
             request.headers.delete("expect")
             request.transition(:idle)
-            send_request(request, connections, options)
+            send_request(request, selector, options)
             return
           end