httpx 1.3.4 → 1.4.0

data/lib/httpx/plugins/aws_sigv4.rb

@@ -89,7 +89,7 @@ module HTTPX
           sts = "#{algo_line}" \
                 "\n#{datetime}" \
                 "\n#{credential_scope}" \
-                "\n#{hexdigest(creq)}"
+                "\n#{OpenSSL::Digest.new(@algorithm).hexdigest(creq)}"
 
           # signature
           k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
@@ -110,22 +110,38 @@ module HTTPX
         private
 
         def hexdigest(value)
-          if value.respond_to?(:to_path)
-            # files, pathnames
-            OpenSSL::Digest.new(@algorithm).file(value.to_path).hexdigest
-          elsif value.respond_to?(:each)
-            digest = OpenSSL::Digest.new(@algorithm)
-
-            mb_buffer = value.each.with_object("".b) do |chunk, buffer|
-              buffer << chunk
-              break if buffer.bytesize >= 1024 * 1024
-            end
+          digest = OpenSSL::Digest.new(@algorithm)
 
-            digest.update(mb_buffer)
-            value.rewind
-            digest.hexdigest
+          if value.respond_to?(:read)
+            if value.respond_to?(:to_path)
+              # files, pathnames
+              digest.file(value.to_path).hexdigest
+            else
+              # gzipped request bodies
+              raise Error, "request body must be rewindable" unless value.respond_to?(:rewind)
+
+              buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+              begin
+                IO.copy_stream(value, buffer)
+                buffer.flush
+
+                digest.file(buffer.to_path).hexdigest
+              ensure
+                value.rewind
+                buffer.close
+                buffer.unlink
+              end
+            end
           else
-            OpenSSL::Digest.new(@algorithm).hexdigest(value)
+            # error on endless generators
+            raise Error, "hexdigest for endless enumerators is not supported" if value.unbounded_body?
+
+            mb_buffer = value.each.with_object("".b) do |chunk, b|
+              b << chunk
+              break if b.bytesize >= 1024 * 1024
+            end
+
+            digest.hexdigest(mb_buffer)
           end
         end
 
@@ -142,7 +158,6 @@ module HTTPX
         def load_dependencies(*)
           require "set"
           require "digest/sha2"
-          require "openssl"
         end
 
         def configure(klass)

data/lib/httpx/plugins/callbacks.rb

@@ -31,12 +31,16 @@ module HTTPX
 
         private
 
-        def init_connection(uri, options)
-          connection = super
+        def do_init_connection(connection, selector)
+          super
           connection.on(:open) do
+            next unless connection.current_session == self
+
             emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
           end
           connection.on(:close) do
+            next unless connection.current_session == self
+
             emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
           end
 
@@ -84,6 +88,12 @@ module HTTPX
         rescue CallbackError => e
           raise e.cause
         end
+
+        def close(*)
+          super
+        rescue CallbackError => e
+          raise e.cause
+        end
       end
     end
     register_plugin :callbacks, Callbacks

data/lib/httpx/plugins/circuit_breaker.rb

@@ -32,11 +32,6 @@ module HTTPX
           @circuit_store = CircuitStore.new(@options)
         end
 
-        def initialize_dup(orig)
-          super
-          @circuit_store = orig.instance_variable_get(:@circuit_store).dup
-        end
-
         %i[circuit_open].each do |meth|
           class_eval(<<-MOD, __FILE__, __LINE__ + 1)
         def on_#{meth}(&blk)   # def on_circuit_open(&blk)

data/lib/httpx/plugins/content_digest.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds `Content-Digest` headers to requests
+    # and can validate these headers on responses
+    #
+    # https://datatracker.ietf.org/doc/html/rfc9530
+    #
+    module ContentDigest
+      class Error < HTTPX::Error; end
+
+      # Error raised on response "content-digest" header validation.
+      class ValidationError < Error
+        attr_reader :response
+
+        def initialize(message, response)
+          super(message)
+          @response = response
+        end
+      end
+
+      class MissingContentDigestError < ValidationError; end
+      class InvalidContentDigestError < ValidationError; end
+
+      SUPPORTED_ALGORITHMS = {
+        "sha-256" => OpenSSL::Digest::SHA256,
+        "sha-512" => OpenSSL::Digest::SHA512,
+      }.freeze
+
+      class << self
+        def extra_options(options)
+          options.merge(encode_content_digest: true, validate_content_digest: false, content_digest_algorithm: "sha-256")
+        end
+      end
+
+      # add support for the following options:
+      #
+      # :content_digest_algorithm :: the digest algorithm to use. Currently supports `sha-256` and `sha-512`. (defaults to `sha-256`)
+      # :encode_content_digest :: whether a <tt>Content-Digest</tt> header should be computed for the request;
+      #                           can also be a callable object (i.e. <tt>->(req) { ... }</tt>, defaults to <tt>true</tt>)
+      # :validate_content_digest :: whether a <tt>Content-Digest</tt> header in the response should be validated;
+      #                             can also be a callable object (i.e. <tt>->(res) { ... }</tt>, defaults to <tt>false</tt>)
+      module OptionsMethods
+        def option_content_digest_algorithm(value)
+          raise TypeError, ":content_digest_algorithm must be one of 'sha-256', 'sha-512'" unless SUPPORTED_ALGORITHMS.key?(value)
+
+          value
+        end
+
+        def option_encode_content_digest(value)
+          value
+        end
+
+        def option_validate_content_digest(value)
+          value
+        end
+      end
+
+      module ResponseBodyMethods
+        attr_reader :content_digest_buffer
+
+        def initialize(response, options)
+          super
+
+          return unless response.headers.key?("content-digest")
+
+          should_validate = options.validate_content_digest
+          should_validate = should_validate.call(response) if should_validate.respond_to?(:call)
+
+          return unless should_validate
+
+          @content_digest_buffer = Response::Buffer.new(
+            threshold_size: @options.body_threshold_size,
+            bytesize: @length,
+            encoding: @encoding
+          )
+        end
+
+        def write(chunk)
+          @content_digest_buffer.write(chunk) if @content_digest_buffer
+          super
+        end
+
+        def close
+          if @content_digest_buffer
+            @content_digest_buffer.close
+            @content_digest_buffer = nil
+          end
+          super
+        end
+      end
+
+      module InstanceMethods
+        def build_request(*)
+          request = super
+
+          return request if request.empty?
+
+          return request if request.headers.key?("content-digest")
+
+          perform_encoding = @options.encode_content_digest
+          perform_encoding = perform_encoding.call(request) if perform_encoding.respond_to?(:call)
+
+          return request unless perform_encoding
+
+          digest = base64digest(request.body)
+          request.headers.add("content-digest", "#{@options.content_digest_algorithm}=:#{digest}:")
+
+          request
+        end
+
+        private
+
+        def fetch_response(request, _, _)
+          response = super
+          return response unless response.is_a?(Response)
+
+          perform_validation = @options.validate_content_digest
+          perform_validation = perform_validation.call(response) if perform_validation.respond_to?(:call)
+
+          validate_content_digest(response) if perform_validation
+
+          response
+        rescue ValidationError => e
+          ErrorResponse.new(request, e)
+        end
+
+        def validate_content_digest(response)
+          content_digest_header = response.headers["content-digest"]
+
+          raise MissingContentDigestError.new("response is missing a `content-digest` header", response) unless content_digest_header
+
+          digests = extract_content_digests(content_digest_header)
+
+          included_algorithms = SUPPORTED_ALGORITHMS.keys & digests.keys
+
+          raise MissingContentDigestError.new("unsupported algorithms: #{digests.keys.join(", ")}", response) if included_algorithms.empty?
+
+          content_buffer = response.body.content_digest_buffer
+
+          included_algorithms.each do |algorithm|
+            digest = SUPPORTED_ALGORITHMS.fetch(algorithm).new
+            digest_received = digests[algorithm]
+            digest_computed =
+              if content_buffer.respond_to?(:to_path)
+                content_buffer.flush
+                digest.file(content_buffer.to_path).base64digest
+              else
+                digest.base64digest(content_buffer.to_s)
+              end
+
+            raise InvalidContentDigestError.new("#{algorithm} digest does not match content",
+                                                response) unless digest_received == digest_computed
+          end
+        end
+
+        def extract_content_digests(header)
+          header.split(",").to_h do |entry|
+            algorithm, digest = entry.split("=", 2)
+            raise Error, "#{entry} is an invalid digest format" unless algorithm && digest
+
+            [algorithm, digest.byteslice(1..-2)]
+          end
+        end
+
+        def base64digest(body)
+          digest = SUPPORTED_ALGORITHMS.fetch(@options.content_digest_algorithm).new
+
+          if body.respond_to?(:read)
+            if body.respond_to?(:to_path)
+              digest.file(body.to_path).base64digest
+            else
+              raise ContentDigestError, "request body must be rewindable" unless body.respond_to?(:rewind)
+
+              buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+              begin
+                IO.copy_stream(body, buffer)
+                buffer.flush
+
+                digest.file(buffer.to_path).base64digest
+              ensure
+                body.rewind
+                buffer.close
+                buffer.unlink
+              end
+            end
+          else
+            raise ContentDigestError, "base64digest for endless enumerators is not supported" if body.unbounded_body?
+
+            buffer = "".b
+            body.each { |chunk| buffer << chunk }
+
+            digest.base64digest(buffer)
+          end
+        end
+      end
+    end
+    register_plugin :content_digest, ContentDigest
+  end
+end

data/lib/httpx/plugins/expect.rb

@@ -96,15 +96,16 @@ module HTTPX
       end
 
       module InstanceMethods
-        def fetch_response(request, connections, options)
-          response = @responses.delete(request)
+        def fetch_response(request, selector, options)
+          response = super
+
           return unless response
 
           if response.is_a?(Response) && response.status == 417 && request.headers.key?("expect")
             response.close
             request.headers.delete("expect")
             request.transition(:idle)
-            send_request(request, connections, options)
+            send_request(request, selector, options)
             return
           end
 

data/lib/httpx/plugins/follow_redirects.rb

@@ -64,9 +64,9 @@ module HTTPX
 
         private
 
-        def fetch_response(request, connections, options)
+        def fetch_response(request, selector, options)
           redirect_request = request.redirect_request
-          response = super(redirect_request, connections, options)
+          response = super(redirect_request, selector, options)
           return unless response
 
           max_redirects = redirect_request.max_redirects
@@ -146,20 +146,19 @@ module HTTPX
             #
             redirect_after = Utils.parse_retry_after(redirect_after)
 
+            retry_start = Utils.now
             log { "redirecting after #{redirect_after} secs..." }
-
-            deactivate_connection(request, connections, options)
-
-            pool.after(redirect_after) do
+            selector.after(redirect_after) do
               if request.response
                 # request has terminated abruptly meanwhile
                 retry_request.emit(:response, request.response)
               else
-                send_request(retry_request, connections, options)
+                log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
+                send_request(retry_request, selector, options)
               end
             end
           else
-            send_request(retry_request, connections, options)
+            send_request(retry_request, selector, options)
           end
           nil
         end

data/lib/httpx/plugins/h2c.rb

@@ -25,26 +25,6 @@ module HTTPX
         end
       end
 
-      module InstanceMethods
-        def send_requests(*requests)
-          upgrade_request, *remainder = requests
-
-          return super unless VALID_H2C_VERBS.include?(upgrade_request.verb) && upgrade_request.scheme == "http"
-
-          connection = pool.find_connection(upgrade_request.uri, upgrade_request.options)
-
-          return super if connection && connection.upgrade_protocol == "h2c"
-
-          # build upgrade request
-          upgrade_request.headers.add("connection", "upgrade")
-          upgrade_request.headers.add("connection", "http2-settings")
-          upgrade_request.headers["upgrade"] = "h2c"
-          upgrade_request.headers["http2-settings"] = ::HTTP2::Client.settings_header(upgrade_request.options.http2_settings)
-
-          super(upgrade_request, *remainder)
-        end
-      end
-
       class H2CParser < Connection::HTTP2
         def upgrade(request, response)
           # skip checks, it is assumed that this is the first
@@ -65,6 +45,29 @@ module HTTPX
       module ConnectionMethods
         using URIExtensions
 
+        def initialize(*)
+          super
+          @h2c_handshake = false
+        end
+
+        def send(request)
+          return super if @h2c_handshake
+
+          return super unless VALID_H2C_VERBS.include?(request.verb) && request.scheme == "http"
+
+          return super if @upgrade_protocol == "h2c"
+
+          @h2c_handshake = true
+
+          # build upgrade request
+          request.headers.add("connection", "upgrade")
+          request.headers.add("connection", "http2-settings")
+          request.headers["upgrade"] = "h2c"
+          request.headers["http2-settings"] = ::HTTP2::Client.settings_header(request.options.http2_settings)
+
+          super
+        end
+
         def upgrade_to_h2c(request, response)
           prev_parser = @parser
 

data/lib/httpx/plugins/internal_telemetry.rb

@@ -76,6 +76,14 @@ module HTTPX
           meter_elapsed_time("Session -> response") if response
           response
         end
+
+        def coalesce_connections(conn1, conn2, selector, *)
+          result = super
+
+          meter_elapsed_time("Connection##{conn2.object_id} coalescing to Connection##{conn1.object_id}") if result
+
+          result
+        end
       end
 
       module RequestMethods
@@ -103,6 +111,25 @@ module HTTPX
           meter_elapsed_time("Connection##{object_id}[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state
         end
       end
+
+      module PoolMethods
+        def self.included(klass)
+          klass.prepend TrackTimeMethods
+          super
+        end
+
+        def checkout_connection(request_uri, options)
+          super.tap do |connection|
+            meter_elapsed_time("Pool##{object_id}: checked out connection for Connection##{connection.object_id}[#{connection.origin}]}")
+          end
+        end
+
+        def checkin_connection(connection)
+          super.tap do
+            meter_elapsed_time("Pool##{object_id}: checked in connection for Connection##{connection.object_id}[#{connection.origin}]}")
+          end
+        end
+      end
     end
     register_plugin :internal_telemetry, InternalTelemetry
   end

data/lib/httpx/plugins/persistent.rb

@@ -30,6 +30,22 @@ module HTTPX
       def self.extra_options(options)
         options.merge(persistent: true)
       end
+
+      module InstanceMethods
+        private
+
+        def get_current_selector
+          super(&nil) || begin
+            return unless block_given?
+
+            default = yield
+
+            set_current_selector(default)
+
+            default
+          end
+        end
+      end
     end
     register_plugin :persistent, Persistent
   end

data/lib/httpx/plugins/proxy/http.rb

@@ -23,29 +23,19 @@ module HTTPX
             with(proxy: opts.merge(scheme: "ntlm"))
           end
 
-          def fetch_response(request, connections, options)
+          def fetch_response(request, selector, options)
             response = super
 
             if response &&
                response.is_a?(Response) &&
                response.status == 407 &&
                !request.headers.key?("proxy-authorization") &&
-               response.headers.key?("proxy-authenticate")
-
-              uri = request.uri
-
-              proxy_options = proxy_options(uri, options)
-              connection = connections.find do |conn|
-                conn.match?(uri, proxy_options)
-              end
-
-              if connection && connection.options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
-                request.transition(:idle)
-                request.headers["proxy-authorization"] =
-                  connection.options.proxy.authenticate(request, response.headers["proxy-authenticate"])
-                send_request(request, connections)
-                return
-              end
+               response.headers.key?("proxy-authenticate") && options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
+              request.transition(:idle)
+              request.headers["proxy-authorization"] =
+                options.proxy.authenticate(request, response.headers["proxy-authenticate"])
+              send_request(request, selector, options)
+              return
             end
 
             response
@@ -74,7 +64,14 @@ module HTTPX
                 parser = @parser
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
-                parser.on(:close) { transition(:closing) }
+                parser.on(:close) do |force|
+                  next unless @parser
+
+                  if force
+                    reset
+                    emit(:terminate)
+                  end
+                end
                 parser.on(:reset) do
                   if parser.empty?
                     reset
@@ -95,8 +92,9 @@ module HTTPX
 
               case @state
               when :connecting
-                @parser.close
+                parser = @parser
                 @parser = nil
+                parser.close
               when :idle
                 @parser.callbacks.clear
                 set_parser_callbacks(@parser)