httpx 0.20.0 → 1.3.1

data/lib/httpx/plugins/aws_sigv4.rb

@@ -5,13 +5,14 @@ module HTTPX
     #
     # This plugin adds AWS Sigv4 authentication.
     #
-    # https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
+    # https://docs.aws.amazon.com/IAM/latest/UserGuide/signing-elements.html
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/AWS-SigV4
+    # https://gitlab.com/os85/httpx/wikis/AWS-SigV4
     #
     module AWSSigV4
       Credentials = Struct.new(:username, :password, :security_token)
 
+      # Signs requests using the AWS sigv4 signing.
       class Signer
         def initialize(
           service:,
@@ -71,7 +72,7 @@ module HTTPX
           end.join
 
           # canonical request
-          creq = "#{request.verb.to_s.upcase}" \
+          creq = "#{request.verb}" \
                  "\n#{request.canonical_path}" \
                  "\n#{request.canonical_query}" \
                  "\n#{canonical_headers}" \
@@ -115,7 +116,7 @@ module HTTPX
           elsif value.respond_to?(:each)
             digest = OpenSSL::Digest.new(@algorithm)
 
-            mb_buffer = value.each.each_with_object("".b) do |chunk, buffer|
+            mb_buffer = value.each.with_object("".b) do |chunk, buffer|
               buffer << chunk
               break if buffer.bytesize >= 1024 * 1024
             end
@@ -146,10 +147,12 @@ module HTTPX
 
         def configure(klass)
           klass.plugin(:expect)
-          klass.plugin(:compression)
         end
       end
 
+      # adds support for the following options:
+      #
+      # :sigv4_signer :: instance of HTTPX::Plugins::AWSSigV4 used to sign requests.
       module OptionsMethods
         def option_sigv4_signer(value)
           value.is_a?(Signer) ? value : Signer.new(value)
@@ -161,7 +164,7 @@ module HTTPX
           with(sigv4_signer: Signer.new(**options))
         end
 
-        def build_request(*, _)
+        def build_request(*)
           request = super
 
           return request if request.headers.key?("authorization")
@@ -186,7 +189,7 @@ module HTTPX
         def canonical_query
           params = query.split("&")
           # params = params.map { |p| p.match(/=/) ? p : p + '=' }
-          # From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+          # From: https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html#create-canonical-request
           # Sort the parameter names by character code point in ascending order.
           # Parameters with duplicate names should be sorted by value.
           #
@@ -198,8 +201,8 @@ module HTTPX
           params.each.with_index.sort do |a, b|
             a, a_offset = a
             b, b_offset = b
-            a_name, a_value = a.split("=")
-            b_name, b_value = b.split("=")
+            a_name, a_value = a.split("=", 2)
+            b_name, b_value = b.split("=", 2)
             if a_name == b_name
               if a_value == b_value
                 a_offset <=> b_offset

data/lib/httpx/plugins/basic_auth.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds helper methods to implement HTTP Basic Auth (https://datatracker.ietf.org/doc/html/rfc7617)
+    #
+    # https://gitlab.com/os85/httpx/wikis/Auth#basic-auth
+    #
+    module BasicAuth
+      class << self
+        def load_dependencies(_klass)
+          require_relative "auth/basic"
+        end
+
+        def configure(klass)
+          klass.plugin(:auth)
+        end
+      end
+
+      module InstanceMethods
+        def basic_auth(user, password)
+          authorization(Authentication::Basic.new(user, password).authenticate)
+        end
+      end
+    end
+    register_plugin :basic_auth, BasicAuth
+  end
+end

data/lib/httpx/plugins/brotli.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    module Brotli
+      class Deflater < Transcoder::Deflater
+        def deflate(chunk)
+          return unless chunk
+
+          ::Brotli.deflate(chunk)
+        end
+      end
+
+      module RequestBodyClassMethods
+        def initialize_deflater_body(body, encoding)
+          return Brotli.encode(body) if encoding == "br"
+
+          super
+        end
+      end
+
+      module ResponseBodyClassMethods
+        def initialize_inflater_by_encoding(encoding, response, **kwargs)
+          return Brotli.decode(response, **kwargs) if encoding == "br"
+
+          super
+        end
+      end
+
+      module_function
+
+      def load_dependencies(*)
+        require "brotli"
+      end
+
+      def self.extra_options(options)
+        options.merge(supported_compression_formats: %w[br] + options.supported_compression_formats)
+      end
+
+      def encode(body)
+        Deflater.new(body)
+      end
+
+      def decode(_response, **)
+        ::Brotli.method(:inflate)
+      end
+    end
+    register_plugin :brotli, Brotli
+  end
+end

data/lib/httpx/plugins/callbacks.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds suppoort for callbacks around the request/response lifecycle.
+    #
+    # https://gitlab.com/os85/httpx/-/wikis/Events
+    #
+    module Callbacks
+      # connection closed user-space errors happen after errors can be surfaced to requests,
+      # so they need to pierce through the scheduler, which is only possible by simulating an
+      # interrupt.
+      class CallbackError < Exception; end # rubocop:disable Lint/InheritException
+
+      module InstanceMethods
+        include HTTPX::Callbacks
+
+        %i[
+          connection_opened connection_closed
+          request_error
+          request_started request_body_chunk request_completed
+          response_started response_body_chunk response_completed
+        ].each do |meth|
+          class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+            def on_#{meth}(&blk)   # def on_connection_opened(&blk)
+              on(:#{meth}, &blk)   #   on(:connection_opened, &blk)
+            end                    # end
+          MOD
+        end
+
+        private
+
+        def init_connection(uri, options)
+          connection = super
+          connection.on(:open) do
+            emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
+          end
+          connection.on(:close) do
+            emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
+          end
+
+          connection
+        end
+
+        def set_request_callbacks(request)
+          super
+
+          request.on(:headers) do
+            emit_or_callback_error(:request_started, request)
+          end
+          request.on(:body_chunk) do |chunk|
+            emit_or_callback_error(:request_body_chunk, request, chunk)
+          end
+          request.on(:done) do
+            emit_or_callback_error(:request_completed, request)
+          end
+
+          request.on(:response_started) do |res|
+            if res.is_a?(Response)
+              emit_or_callback_error(:response_started, request, res)
+              res.on(:chunk_received) do |chunk|
+                emit_or_callback_error(:response_body_chunk, request, res, chunk)
+              end
+            else
+              emit_or_callback_error(:request_error, request, res.error)
+            end
+          end
+          request.on(:response) do |res|
+            emit_or_callback_error(:response_completed, request, res)
+          end
+        end
+
+        def emit_or_callback_error(*args)
+          emit(*args)
+        rescue StandardError => e
+          ex = CallbackError.new(e.message)
+          ex.set_backtrace(e.backtrace)
+          raise ex
+        end
+
+        def receive_requests(*)
+          super
+        rescue CallbackError => e
+          raise e.cause
+        end
+      end
+    end
+    register_plugin :callbacks, Callbacks
+  end
+end

data/lib/httpx/plugins/circuit_breaker/circuit.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::CircuitBreaker
+    #
+    # A circuit is assigned to a given absoolute url or origin.
+    #
+    # It sets +max_attempts+, the number of attempts the circuit allows, before it is opened.
+    # It sets +reset_attempts_in+, the time a circuit stays open at most, before it resets.
+    # It sets +break_in+, the time that must elapse before an open circuit can transit to the half-open state.
+    # It sets +circuit_breaker_half_open_drip_rate+, the rate of requests a circuit allows to be performed when in an half-open state.
+    #
+    class Circuit
+      def initialize(max_attempts, reset_attempts_in, break_in, circuit_breaker_half_open_drip_rate)
+        @max_attempts = max_attempts
+        @reset_attempts_in = reset_attempts_in
+        @break_in = break_in
+        @circuit_breaker_half_open_drip_rate = circuit_breaker_half_open_drip_rate
+        @attempts = 0
+
+        total_real_attempts = @max_attempts * @circuit_breaker_half_open_drip_rate
+        @drip_factor = (@max_attempts / total_real_attempts).round
+        @state = :closed
+      end
+
+      def respond
+        try_close
+
+        case @state
+        when :closed
+          nil
+        when :half_open
+          @attempts += 1
+
+          # do real requests while drip rate valid
+          if (@real_attempts % @drip_factor).zero?
+            @real_attempts += 1
+            return
+          end
+
+          @response
+        when :open
+
+          @response
+        end
+      end
+
+      def try_open(response)
+        case @state
+        when :closed
+          now = Utils.now
+
+          if @attempts.positive?
+            # reset if error happened long ago
+            @attempts = 0 if now - @attempted_at > @reset_attempts_in
+          else
+            @attempted_at = now
+          end
+
+          @attempts += 1
+
+          return unless @attempts >= @max_attempts
+
+          @state = :open
+          @opened_at = now
+          @response = response
+        when :half_open
+          # open immediately
+
+          @state = :open
+          @attempted_at = @opened_at = Utils.now
+          @response = response
+        end
+      end
+
+      def try_close
+        case @state
+        when :closed
+          nil
+        when :half_open
+
+          # do not close circuit unless attempts exhausted
+          return unless @attempts >= @max_attempts
+
+          # reset!
+          @attempts = 0
+          @opened_at = @attempted_at = @response = nil
+          @state = :closed
+
+        when :open
+          if Utils.elapsed_time(@opened_at) > @break_in
+            @state = :half_open
+            @attempts = 0
+            @real_attempts = 0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/circuit_breaker/circuit_store.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module HTTPX::Plugins::CircuitBreaker
+  using HTTPX::URIExtensions
+
+  class CircuitStore
+    def initialize(options)
+      @circuits = Hash.new do |h, k|
+        h[k] = Circuit.new(
+          options.circuit_breaker_max_attempts,
+          options.circuit_breaker_reset_attempts_in,
+          options.circuit_breaker_break_in,
+          options.circuit_breaker_half_open_drip_rate
+        )
+      end
+      @circuits_mutex = Thread::Mutex.new
+    end
+
+    def try_open(uri, response)
+      circuit = @circuits_mutex.synchronize { get_circuit_for_uri(uri) }
+
+      circuit.try_open(response)
+    end
+
+    def try_close(uri)
+      circuit = @circuits_mutex.synchronize do
+        return unless @circuits.key?(uri.origin) || @circuits.key?(uri.to_s)
+
+        get_circuit_for_uri(uri)
+      end
+
+      circuit.try_close
+    end
+
+    # if circuit is open, it'll respond with the stored response.
+    # if not, nil.
+    def try_respond(request)
+      circuit = @circuits_mutex.synchronize { get_circuit_for_uri(request.uri) }
+
+      circuit.respond
+    end
+
+    private
+
+    def get_circuit_for_uri(uri)
+      if uri.respond_to?(:origin) && @circuits.key?(uri.origin)
+        @circuits[uri.origin]
+      else
+        @circuits[uri.to_s]
+      end
+    end
+  end
+end

data/lib/httpx/plugins/circuit_breaker.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin implements a circuit breaker around connection errors.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Circuit-Breaker
+    #
+    module CircuitBreaker
+      using URIExtensions
+
+      def self.load_dependencies(*)
+        require_relative "circuit_breaker/circuit"
+        require_relative "circuit_breaker/circuit_store"
+      end
+
+      def self.extra_options(options)
+        options.merge(
+          circuit_breaker_max_attempts: 3,
+          circuit_breaker_reset_attempts_in: 60,
+          circuit_breaker_break_in: 60,
+          circuit_breaker_half_open_drip_rate: 1
+        )
+      end
+
+      module InstanceMethods
+        include HTTPX::Callbacks
+
+        def initialize(*)
+          super
+          @circuit_store = CircuitStore.new(@options)
+        end
+
+        def initialize_dup(orig)
+          super
+          @circuit_store = orig.instance_variable_get(:@circuit_store).dup
+        end
+
+        %i[circuit_open].each do |meth|
+          class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+        def on_#{meth}(&blk)   # def on_circuit_open(&blk)
+          on(:#{meth}, &blk)   #   on(:circuit_open, &blk)
+        end                    # end
+          MOD
+        end
+
+        private
+
+        def send_requests(*requests)
+          # @type var short_circuit_responses: Array[response]
+          short_circuit_responses = []
+
+          # run all requests through the circuit breaker, see if the circuit is
+          # open for any of them.
+          real_requests = requests.each_with_index.with_object([]) do |(req, idx), real_reqs|
+            short_circuit_response = @circuit_store.try_respond(req)
+            if short_circuit_response.nil?
+              real_reqs << req
+              next
+            end
+            short_circuit_responses[idx] = short_circuit_response
+          end
+
+          # run requests for the remainder
+          unless real_requests.empty?
+            responses = super(*real_requests)
+
+            real_requests.each_with_index do |request, idx|
+              short_circuit_responses[requests.index(request)] = responses[idx]
+            end
+          end
+
+          short_circuit_responses
+        end
+
+        def on_response(request, response)
+          emit(:circuit_open, request) if try_circuit_open(request, response)
+
+          super
+        end
+
+        def try_circuit_open(request, response)
+          if response.is_a?(ErrorResponse)
+            case response.error
+            when RequestTimeoutError
+              @circuit_store.try_open(request.uri, response)
+            else
+              @circuit_store.try_open(request.origin, response)
+            end
+          elsif (break_on = request.options.circuit_breaker_break_on) && break_on.call(response)
+            @circuit_store.try_open(request.uri, response)
+          else
+            @circuit_store.try_close(request.uri)
+            nil
+          end
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :circuit_breaker_max_attempts :: the number of attempts the circuit allows, before it is opened (defaults to <tt>3</tt>).
+      # :circuit_breaker_reset_attempts_in :: the time a circuit stays open at most, before it resets (defaults to <tt>60</tt>).
+      # :circuit_breaker_break_on :: callable defining an alternative rule for a response to break
+      #                              (i.e. <tt>->(res) { res.status == 429 } </tt>)
+      # :circuit_breaker_break_in :: the time that must elapse before an open circuit can transit to the half-open state
+      #                              (defaults to <tt><60</tt>).
+      # :circuit_breaker_half_open_drip_rate :: the rate of requests a circuit allows to be performed when in an half-open state
+      #                                         (defaults to <tt>1</tt>).
+      module OptionsMethods
+        def option_circuit_breaker_max_attempts(value)
+          attempts = Integer(value)
+          raise TypeError, ":circuit_breaker_max_attempts must be positive" unless attempts.positive?
+
+          attempts
+        end
+
+        def option_circuit_breaker_reset_attempts_in(value)
+          timeout = Float(value)
+          raise TypeError, ":circuit_breaker_reset_attempts_in must be positive" unless timeout.positive?
+
+          timeout
+        end
+
+        def option_circuit_breaker_break_in(value)
+          timeout = Float(value)
+          raise TypeError, ":circuit_breaker_break_in must be positive" unless timeout.positive?
+
+          timeout
+        end
+
+        def option_circuit_breaker_half_open_drip_rate(value)
+          ratio = Float(value)
+          raise TypeError, ":circuit_breaker_half_open_drip_rate must be a number between 0 and 1" unless (0..1).cover?(ratio)
+
+          ratio
+        end
+
+        def option_circuit_breaker_break_on(value)
+          raise TypeError, ":circuit_breaker_break_on must be called with the response" unless value.respond_to?(:call)
+
+          value
+        end
+      end
+    end
+    register_plugin :circuit_breaker, CircuitBreaker
+  end
+end

data/lib/httpx/plugins/cookies/set_cookie_parser.rb

@@ -6,8 +6,6 @@ require "time"
 module HTTPX
   module Plugins::Cookies
     module SetCookieParser
-      using(RegexpExtensions) unless Regexp.method_defined?(:match?)
-
       # Whitespace.
       RE_WSP = /[ \t]+/.freeze
 

data/lib/httpx/plugins/cookies.rb

@@ -9,7 +9,7 @@ module HTTPX
     #
     # It also adds a *#cookies* helper, so that you can pre-fill the cookies of a session.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Cookies
+    # https://gitlab.com/os85/httpx/wikis/Cookies
     #
     module Cookies
       def self.load_dependencies(*)
@@ -40,9 +40,15 @@ module HTTPX
           end
         end
 
+        def build_request(*)
+          request = super
+          request.headers.set_cookie(request.options.cookies[request.uri])
+          request
+        end
+
         private
 
-        def on_response(reuest, response)
+        def on_response(_request, response)
           if response && response.respond_to?(:headers) && (set_cookie = response.headers["set-cookie"])
 
             log { "cookies: set-cookie is over #{Cookie::MAX_LENGTH}" } if set_cookie.bytesize > Cookie::MAX_LENGTH
@@ -52,12 +58,6 @@ module HTTPX
 
           super
         end
-
-        def build_request(*, _)
-          request = super
-          request.headers.set_cookie(request.options.cookies[request.uri])
-          request
-        end
       end
 
       module HeadersMethods
@@ -70,23 +70,36 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :cookies :: cookie jar for the session (can be a Hash, an Array, an instance of HTTPX::Plugins::Cookies::CookieJar)
       module OptionsMethods
-        def __initialize__(*)
-          super
+        def option_headers(*)
+          value = super
 
-          return unless @headers.key?("cookie")
+          merge_cookie_in_jar(value.delete("cookie"), @cookies) if defined?(@cookies) && value.key?("cookie")
 
-          @headers.delete("cookie").each do |ck|
+          value
+        end
+
+        def option_cookies(value)
+          jar = value.is_a?(Jar) ? value : Jar.new(value)
+
+          merge_cookie_in_jar(@headers.delete("cookie"), jar) if defined?(@headers) && @headers.key?("cookie")
+
+          jar
+        end
+
+        private
+
+        def merge_cookie_in_jar(cookies, jar)
+          cookies.each do |ck|
             ck.split(/ *; */).each do |cookie|
               name, value = cookie.split("=", 2)
-              @cookies.add(Cookie.new(name, value))
+              jar.add(Cookie.new(name, value))
             end
           end
         end
-
-        def option_cookies(value)
-          value.is_a?(Jar) ? value : Jar.new(value)
-        end
       end
     end
     register_plugin :cookies, Cookies