httpx-patched 1.6.2.1

data/lib/httpx/plugins/callbacks.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds suppoort for callbacks around the request/response lifecycle.
+    #
+    # https://gitlab.com/os85/httpx/-/wikis/Events
+    #
+    module Callbacks
+      CALLBACKS = %i[
+        connection_opened connection_closed
+        request_error
+        request_started request_body_chunk request_completed
+        response_started response_body_chunk response_completed
+      ].freeze
+
+      # connection closed user-space errors happen after errors can be surfaced to requests,
+      # so they need to pierce through the scheduler, which is only possible by simulating an
+      # interrupt.
+      class CallbackError < Exception; end # rubocop:disable Lint/InheritException
+
+      module InstanceMethods
+        include HTTPX::Callbacks
+
+        CALLBACKS.each do |meth|
+          class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+            def on_#{meth}(&blk)   # def on_connection_opened(&blk)
+              on(:#{meth}, &blk)   #   on(:connection_opened, &blk)
+              self                 #   self
+            end                    # end
+          MOD
+        end
+
+        def plugin(*args, &blk)
+          super(*args).tap do |sess|
+            CALLBACKS.each do |cb|
+              next unless callbacks_for?(cb)
+
+              sess.callbacks(cb).concat(callbacks(cb))
+            end
+
+            sess.wrap(&blk) if blk
+          end
+        end
+
+        private
+
+        def branch(options, &blk)
+          super(options).tap do |sess|
+            CALLBACKS.each do |cb|
+              next unless callbacks_for?(cb)
+
+              sess.callbacks(cb).concat(callbacks(cb))
+            end
+            sess.wrap(&blk) if blk
+          end
+        end
+
+        def do_init_connection(connection, selector)
+          super
+          connection.on(:open) do
+            next unless connection.current_session == self
+
+            emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
+          end
+          connection.on(:close) do
+            next unless connection.current_session == self
+
+            emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
+          end
+
+          connection
+        end
+
+        def set_request_callbacks(request)
+          super
+
+          request.on(:headers) do
+            emit_or_callback_error(:request_started, request)
+          end
+          request.on(:body_chunk) do |chunk|
+            emit_or_callback_error(:request_body_chunk, request, chunk)
+          end
+          request.on(:done) do
+            emit_or_callback_error(:request_completed, request)
+          end
+
+          request.on(:response_started) do |res|
+            if res.is_a?(Response)
+              emit_or_callback_error(:response_started, request, res)
+              res.on(:chunk_received) do |chunk|
+                emit_or_callback_error(:response_body_chunk, request, res, chunk)
+              end
+            else
+              emit_or_callback_error(:request_error, request, res.error)
+            end
+          end
+          request.on(:response) do |res|
+            emit_or_callback_error(:response_completed, request, res) if res.is_a?(Response)
+          end
+        end
+
+        def emit_or_callback_error(*args)
+          emit(*args)
+        rescue StandardError => e
+          ex = CallbackError.new(e.message)
+          ex.set_backtrace(e.backtrace)
+          raise ex
+        end
+
+        def receive_requests(*)
+          super
+        rescue CallbackError => e
+          raise e.cause
+        end
+
+        def close(*)
+          super
+        rescue CallbackError => e
+          raise e.cause
+        end
+      end
+    end
+    register_plugin :callbacks, Callbacks
+  end
+end

data/lib/httpx/plugins/circuit_breaker/circuit.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::CircuitBreaker
+    #
+    # A circuit is assigned to a given absoolute url or origin.
+    #
+    # It sets +max_attempts+, the number of attempts the circuit allows, before it is opened.
+    # It sets +reset_attempts_in+, the time a circuit stays open at most, before it resets.
+    # It sets +break_in+, the time that must elapse before an open circuit can transit to the half-open state.
+    # It sets +circuit_breaker_half_open_drip_rate+, the rate of requests a circuit allows to be performed when in an half-open state.
+    #
+    class Circuit
+      def initialize(max_attempts, reset_attempts_in, break_in, circuit_breaker_half_open_drip_rate)
+        @max_attempts = max_attempts
+        @reset_attempts_in = reset_attempts_in
+        @break_in = break_in
+        @circuit_breaker_half_open_drip_rate = circuit_breaker_half_open_drip_rate
+        @attempts = 0
+
+        total_real_attempts = @max_attempts * @circuit_breaker_half_open_drip_rate
+        @drip_factor = (@max_attempts / total_real_attempts).round
+        @state = :closed
+      end
+
+      def respond
+        try_close
+
+        case @state
+        when :closed
+          nil
+        when :half_open
+          @attempts += 1
+
+          # do real requests while drip rate valid
+          if (@real_attempts % @drip_factor).zero?
+            @real_attempts += 1
+            return
+          end
+
+          @response
+        when :open
+
+          @response
+        end
+      end
+
+      def try_open(response)
+        case @state
+        when :closed
+          now = Utils.now
+
+          if @attempts.positive?
+            # reset if error happened long ago
+            @attempts = 0 if now - @attempted_at > @reset_attempts_in
+          else
+            @attempted_at = now
+          end
+
+          @attempts += 1
+
+          return unless @attempts >= @max_attempts
+
+          @state = :open
+          @opened_at = now
+          @response = response
+        when :half_open
+          # open immediately
+
+          @state = :open
+          @attempted_at = @opened_at = Utils.now
+          @response = response
+        end
+      end
+
+      def try_close
+        case @state
+        when :closed
+          nil
+        when :half_open
+
+          # do not close circuit unless attempts exhausted
+          return unless @attempts >= @max_attempts
+
+          # reset!
+          @attempts = 0
+          @opened_at = @attempted_at = @response = nil
+          @state = :closed
+
+        when :open
+          if Utils.elapsed_time(@opened_at) > @break_in
+            @state = :half_open
+            @attempts = 0
+            @real_attempts = 0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/circuit_breaker/circuit_store.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module HTTPX::Plugins::CircuitBreaker
+  using HTTPX::URIExtensions
+
+  class CircuitStore
+    def initialize(options)
+      @circuits = Hash.new do |h, k|
+        h[k] = Circuit.new(
+          options.circuit_breaker_max_attempts,
+          options.circuit_breaker_reset_attempts_in,
+          options.circuit_breaker_break_in,
+          options.circuit_breaker_half_open_drip_rate
+        )
+      end
+      @circuits_mutex = Thread::Mutex.new
+    end
+
+    def try_open(uri, response)
+      circuit = @circuits_mutex.synchronize { get_circuit_for_uri(uri) }
+
+      circuit.try_open(response)
+    end
+
+    def try_close(uri)
+      circuit = @circuits_mutex.synchronize do
+        return unless @circuits.key?(uri.origin) || @circuits.key?(uri.to_s)
+
+        get_circuit_for_uri(uri)
+      end
+
+      circuit.try_close
+    end
+
+    # if circuit is open, it'll respond with the stored response.
+    # if not, nil.
+    def try_respond(request)
+      circuit = @circuits_mutex.synchronize { get_circuit_for_uri(request.uri) }
+
+      circuit.respond
+    end
+
+    private
+
+    def get_circuit_for_uri(uri)
+      if uri.respond_to?(:origin) && @circuits.key?(uri.origin)
+        @circuits[uri.origin]
+      else
+        @circuits[uri.to_s]
+      end
+    end
+  end
+end

data/lib/httpx/plugins/circuit_breaker.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin implements a circuit breaker around connection errors.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Circuit-Breaker
+    #
+    module CircuitBreaker
+      using URIExtensions
+
+      def self.load_dependencies(*)
+        require_relative "circuit_breaker/circuit"
+        require_relative "circuit_breaker/circuit_store"
+      end
+
+      def self.extra_options(options)
+        options.merge(
+          circuit_breaker_max_attempts: 3,
+          circuit_breaker_reset_attempts_in: 60,
+          circuit_breaker_break_in: 60,
+          circuit_breaker_half_open_drip_rate: 1
+        )
+      end
+
+      module InstanceMethods
+        include HTTPX::Callbacks
+
+        def initialize(*)
+          super
+          @circuit_store = CircuitStore.new(@options)
+        end
+
+        %i[circuit_open].each do |meth|
+          class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+        def on_#{meth}(&blk)   # def on_circuit_open(&blk)
+          on(:#{meth}, &blk)   #   on(:circuit_open, &blk)
+          self                 #   self
+        end                    # end
+          MOD
+        end
+
+        private
+
+        def send_requests(*requests)
+          # @type var short_circuit_responses: Array[response]
+          short_circuit_responses = []
+
+          # run all requests through the circuit breaker, see if the circuit is
+          # open for any of them.
+          real_requests = requests.each_with_index.with_object([]) do |(req, idx), real_reqs|
+            short_circuit_response = @circuit_store.try_respond(req)
+            if short_circuit_response.nil?
+              real_reqs << req
+              next
+            end
+            short_circuit_responses[idx] = short_circuit_response
+          end
+
+          # run requests for the remainder
+          unless real_requests.empty?
+            responses = super(*real_requests)
+
+            real_requests.each_with_index do |request, idx|
+              short_circuit_responses[requests.index(request)] = responses[idx]
+            end
+          end
+
+          short_circuit_responses
+        end
+
+        def set_request_callbacks(request)
+          super
+          request.on(:response) do |response|
+            emit(:circuit_open, request) if try_circuit_open(request, response)
+          end
+        end
+
+        def try_circuit_open(request, response)
+          if response.is_a?(ErrorResponse)
+            case response.error
+            when RequestTimeoutError
+              @circuit_store.try_open(request.uri, response)
+            else
+              @circuit_store.try_open(request.origin, response)
+            end
+          elsif (break_on = request.options.circuit_breaker_break_on) && break_on.call(response)
+            @circuit_store.try_open(request.uri, response)
+          else
+            @circuit_store.try_close(request.uri)
+            nil
+          end
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :circuit_breaker_max_attempts :: the number of attempts the circuit allows, before it is opened (defaults to <tt>3</tt>).
+      # :circuit_breaker_reset_attempts_in :: the time a circuit stays open at most, before it resets (defaults to <tt>60</tt>).
+      # :circuit_breaker_break_on :: callable defining an alternative rule for a response to break
+      #                              (i.e. <tt>->(res) { res.status == 429 } </tt>)
+      # :circuit_breaker_break_in :: the time that must elapse before an open circuit can transit to the half-open state
+      #                              (defaults to <tt><60</tt>).
+      # :circuit_breaker_half_open_drip_rate :: the rate of requests a circuit allows to be performed when in an half-open state
+      #                                         (defaults to <tt>1</tt>).
+      module OptionsMethods
+        private
+
+        def option_circuit_breaker_max_attempts(value)
+          attempts = Integer(value)
+          raise TypeError, ":circuit_breaker_max_attempts must be positive" unless attempts.positive?
+
+          attempts
+        end
+
+        def option_circuit_breaker_reset_attempts_in(value)
+          timeout = Float(value)
+          raise TypeError, ":circuit_breaker_reset_attempts_in must be positive" unless timeout.positive?
+
+          timeout
+        end
+
+        def option_circuit_breaker_break_in(value)
+          timeout = Float(value)
+          raise TypeError, ":circuit_breaker_break_in must be positive" unless timeout.positive?
+
+          timeout
+        end
+
+        def option_circuit_breaker_half_open_drip_rate(value)
+          ratio = Float(value)
+          raise TypeError, ":circuit_breaker_half_open_drip_rate must be a number between 0 and 1" unless (0..1).cover?(ratio)
+
+          ratio
+        end
+
+        def option_circuit_breaker_break_on(value)
+          raise TypeError, ":circuit_breaker_break_on must be called with the response" unless value.respond_to?(:call)
+
+          value
+        end
+      end
+    end
+    register_plugin :circuit_breaker, CircuitBreaker
+  end
+end

data/lib/httpx/plugins/content_digest.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds `Content-Digest` headers to requests
+    # and can validate these headers on responses
+    #
+    # https://datatracker.ietf.org/doc/html/rfc9530
+    #
+    module ContentDigest
+      class Error < HTTPX::Error; end
+
+      # Error raised on response "content-digest" header validation.
+      class ValidationError < Error
+        attr_reader :response
+
+        def initialize(message, response)
+          super(message)
+          @response = response
+        end
+      end
+
+      class MissingContentDigestError < ValidationError; end
+      class InvalidContentDigestError < ValidationError; end
+
+      SUPPORTED_ALGORITHMS = {
+        "sha-256" => OpenSSL::Digest::SHA256,
+        "sha-512" => OpenSSL::Digest::SHA512,
+      }.freeze
+
+      class << self
+        def extra_options(options)
+          options.merge(encode_content_digest: true, validate_content_digest: false, content_digest_algorithm: "sha-256")
+        end
+      end
+
+      # add support for the following options:
+      #
+      # :content_digest_algorithm :: the digest algorithm to use. Currently supports `sha-256` and `sha-512`. (defaults to `sha-256`)
+      # :encode_content_digest :: whether a <tt>Content-Digest</tt> header should be computed for the request;
+      #                           can also be a callable object (i.e. <tt>->(req) { ... }</tt>, defaults to <tt>true</tt>)
+      # :validate_content_digest :: whether a <tt>Content-Digest</tt> header in the response should be validated;
+      #                             can also be a callable object (i.e. <tt>->(res) { ... }</tt>, defaults to <tt>false</tt>)
+      module OptionsMethods
+        private
+
+        def option_content_digest_algorithm(value)
+          raise TypeError, ":content_digest_algorithm must be one of 'sha-256', 'sha-512'" unless SUPPORTED_ALGORITHMS.key?(value)
+
+          value
+        end
+
+        def option_encode_content_digest(value)
+          value
+        end
+
+        def option_validate_content_digest(value)
+          value
+        end
+      end
+
+      module ResponseBodyMethods
+        attr_reader :content_digest_buffer
+
+        def initialize(response, options)
+          super
+
+          return unless response.headers.key?("content-digest")
+
+          should_validate = options.validate_content_digest
+          should_validate = should_validate.call(response) if should_validate.respond_to?(:call)
+
+          return unless should_validate
+
+          @content_digest_buffer = Response::Buffer.new(
+            threshold_size: @options.body_threshold_size,
+            bytesize: @length,
+            encoding: @encoding
+          )
+        end
+
+        def write(chunk)
+          @content_digest_buffer.write(chunk) if @content_digest_buffer
+          super
+        end
+
+        def close
+          if @content_digest_buffer
+            @content_digest_buffer.close
+            @content_digest_buffer = nil
+          end
+          super
+        end
+      end
+
+      module InstanceMethods
+        def build_request(*)
+          request = super
+
+          return request if request.empty?
+
+          return request if request.headers.key?("content-digest")
+
+          perform_encoding = @options.encode_content_digest
+          perform_encoding = perform_encoding.call(request) if perform_encoding.respond_to?(:call)
+
+          return request unless perform_encoding
+
+          digest = base64digest(request.body)
+          request.headers.add("content-digest", "#{@options.content_digest_algorithm}=:#{digest}:")
+
+          request
+        end
+
+        private
+
+        def fetch_response(request, _, _)
+          response = super
+          return response unless response.is_a?(Response)
+
+          perform_validation = @options.validate_content_digest
+          perform_validation = perform_validation.call(response) if perform_validation.respond_to?(:call)
+
+          validate_content_digest(response) if perform_validation
+
+          response
+        rescue ValidationError => e
+          ErrorResponse.new(request, e)
+        end
+
+        def validate_content_digest(response)
+          content_digest_header = response.headers["content-digest"]
+
+          raise MissingContentDigestError.new("response is missing a `content-digest` header", response) unless content_digest_header
+
+          digests = extract_content_digests(content_digest_header)
+
+          included_algorithms = SUPPORTED_ALGORITHMS.keys & digests.keys
+
+          raise MissingContentDigestError.new("unsupported algorithms: #{digests.keys.join(", ")}", response) if included_algorithms.empty?
+
+          content_buffer = response.body.content_digest_buffer
+
+          included_algorithms.each do |algorithm|
+            digest = SUPPORTED_ALGORITHMS.fetch(algorithm).new
+            digest_received = digests[algorithm]
+            digest_computed =
+              if content_buffer.respond_to?(:to_path)
+                content_buffer.flush
+                digest.file(content_buffer.to_path).base64digest
+              else
+                digest.base64digest(content_buffer.to_s)
+              end
+
+            raise InvalidContentDigestError.new("#{algorithm} digest does not match content",
+                                                response) unless digest_received == digest_computed
+          end
+        end
+
+        def extract_content_digests(header)
+          header.split(",").to_h do |entry|
+            algorithm, digest = entry.split("=", 2)
+            raise Error, "#{entry} is an invalid digest format" unless algorithm && digest
+
+            [algorithm, digest.byteslice(1..-2)]
+          end
+        end
+
+        def base64digest(body)
+          digest = SUPPORTED_ALGORITHMS.fetch(@options.content_digest_algorithm).new
+
+          if body.respond_to?(:read)
+            if body.respond_to?(:to_path)
+              digest.file(body.to_path).base64digest
+            else
+              raise ContentDigestError, "request body must be rewindable" unless body.respond_to?(:rewind)
+
+              buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+              begin
+                IO.copy_stream(body, buffer)
+                buffer.flush
+
+                digest.file(buffer.to_path).base64digest
+              ensure
+                body.rewind
+                buffer.close
+                buffer.unlink
+              end
+            end
+          else
+            raise ContentDigestError, "base64digest for endless enumerators is not supported" if body.unbounded_body?
+
+            buffer = "".b
+            body.each { |chunk| buffer << chunk }
+
+            digest.base64digest(buffer)
+          end
+        end
+      end
+    end
+    register_plugin :content_digest, ContentDigest
+  end
+end