httpx 1.6.3 → 1.7.1

data/lib/httpx/plugins/oauth.rb

@@ -2,21 +2,38 @@
 
 module HTTPX
   module Plugins
+    #
+    # This plugin adds support for managing an OAuth Session associated with the given session.
+    #
+    # The scope of OAuth support is limited to the `client_crendentials` and `refresh_token` grants.
     #
     # https://gitlab.com/os85/httpx/wikis/OAuth
     #
     module OAuth
       class << self
-        def load_dependencies(_klass)
+        def load_dependencies(klass)
           require_relative "auth/basic"
+          klass.plugin(:auth)
+        end
+
+        def subplugins
+          {
+            retries: OAuthRetries,
+          }
+        end
+
+        def extra_options(options)
+          options.merge(auth_header_type: "Bearer")
         end
       end
 
       SUPPORTED_GRANT_TYPES = %w[client_credentials refresh_token].freeze
       SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
 
+      # Implements the bulk of functionality and maintains the state associated with the
+      # management of the the lifecycle of an OAuth session.
       class OAuthSession
-        attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope, :audience
+        attr_reader :access_token, :refresh_token
 
         def initialize(
           issuer:,
@@ -27,7 +44,6 @@ module HTTPX
           scope: nil,
           audience: nil,
           token_endpoint: nil,
-          response_type: nil,
           grant_type: nil,
           token_endpoint_auth_method: nil
         )
@@ -35,7 +51,6 @@ module HTTPX
           @client_id = client_id
           @client_secret = client_secret
           @token_endpoint = URI(token_endpoint) if token_endpoint
-          @response_type = response_type
           @scope = case scope
                    when String
                      scope.split
@@ -47,6 +62,8 @@ module HTTPX
           @refresh_token = refresh_token
           @token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
           @grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
+          @access_token = access_token
+          @refresh_token = refresh_token
 
           unless @token_endpoint_auth_method.nil? || SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
             raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
@@ -57,28 +74,75 @@ module HTTPX
           raise Error, "#{@grant_type} is not a supported grant type"
         end
 
+        # returns the URL where to request access and refresh tokens from.
         def token_endpoint
           @token_endpoint || "#{@issuer}/token"
         end
 
+        # returns the oauth-documented authorization method to use when requesting a token.
         def token_endpoint_auth_method
           @token_endpoint_auth_method || "client_secret_basic"
         end
 
-        def load(http)
-          return if @grant_type && @scope
+        def reset!
+          @access_token = nil
+        end
 
-          metadata = http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json
+        # when not available, it uses the +http+ object to request new access and refresh tokens.
+        def fetch_access_token(http)
+          return access_token if access_token
 
-          @token_endpoint = metadata["token_endpoint"]
-          @scope = metadata["scopes_supported"]
-          @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
-          @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
-            SUPPORTED_AUTH_METHODS.include?(am)
+          load(http)
+
+          # always prefer refresh token grant if a refresh token is available
+          grant_type = @refresh_token ? "refresh_token" : @grant_type
+
+          headers = {} # : Hash[String ,String]
+          form_post = {
+            "grant_type" => @grant_type,
+            "scope" => Array(@scope).join(" "),
+            "audience" => @audience,
+          }.compact
+
+          # auth
+          case token_endpoint_auth_method
+          when "client_secret_post"
+            form_post["client_id"] = @client_id
+            form_post["client_secret"] = @client_secret
+          when "client_secret_basic"
+            headers["authorization"] = Authentication::Basic.new(@client_id, @client_secret).authenticate
           end
-          nil
+
+          case grant_type
+          when "client_credentials"
+            # do nothing
+          when "refresh_token"
+            raise Error, "cannot use the `\"refresh_token\"` grant type without a refresh token" unless refresh_token
+
+            form_post["refresh_token"] = refresh_token
+          end
+
+          # POST /token
+          token_request = http.build_request("POST", token_endpoint, headers: headers, form: form_post)
+
+          token_request.headers.delete("authorization") unless token_endpoint_auth_method == "client_secret_basic"
+
+          token_response = http.skip_auth_header { http.request(token_request) }
+
+          begin
+            token_response.raise_for_status
+          rescue HTTPError => e
+            @refresh_token = nil if e.response.status == 401 && (grant_type == "refresh_token")
+            raise e
+          end
+
+          payload = token_response.json
+
+          @refresh_token = payload["refresh_token"] || @refresh_token
+          @access_token = payload["access_token"]
         end
 
+        # TODO: remove this after deprecating the `:oauth_session` option
         def merge(other)
           obj = dup
 
@@ -97,12 +161,36 @@ module HTTPX
           end
           obj
         end
+
+        private
+
+        # uses +http+ to fetch for the oauth server metadata.
+        def load(http)
+          return if @grant_type && @scope
+
+          metadata = http.skip_auth_header { http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json }
+
+          @token_endpoint = metadata["token_endpoint"]
+          @scope = metadata["scopes_supported"]
+          @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
+          @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
+            SUPPORTED_AUTH_METHODS.include?(am)
+          end
+          nil
+        end
       end
 
+      # adds support for the following options:
+      #
+      # :oauth_options :: an hash of options to be used during session management.
+      #                   check the parameters to initialize the OAuthSession class.
       module OptionsMethods
         private
 
         def option_oauth_session(value)
+          warn "DEPRECATION WARNING: option `:oauth_session` is deprecated. " \
+               "Use `:oauth_options` instead."
+
           case value
           when Hash
             OAuthSession.new(**value)
@@ -112,69 +200,80 @@ module HTTPX
             raise TypeError, ":oauth_session must be a #{OAuthSession}"
           end
         end
-      end
 
-      module InstanceMethods
-        def oauth_auth(**args)
-          with(oauth_session: OAuthSession.new(**args))
+        def option_oauth_options(value)
+          value = Hash[value] unless value.is_a?(Hash)
+          value
         end
+      end
 
-        def with_access_token
-          oauth_session = @options.oauth_session
-
-          oauth_session.load(self)
-
-          grant_type = oauth_session.grant_type
+      module InstanceMethods
+        attr_reader :oauth_session
+        protected :oauth_session
 
-          headers = {}
-          form_post = {
-            "grant_type" => grant_type,
-            "scope" => Array(oauth_session.scope).join(" "),
-            "audience" => oauth_session.audience,
-          }.compact
+        def initialize(*)
+          super
 
-          # auth
-          case oauth_session.token_endpoint_auth_method
-          when "client_secret_post"
-            form_post["client_id"] = oauth_session.client_id
-            form_post["client_secret"] = oauth_session.client_secret
-          when "client_secret_basic"
-            headers["authorization"] = Authentication::Basic.new(oauth_session.client_id, oauth_session.client_secret).authenticate
+          @oauth_session = if @options.oauth_options
+            OAuthSession.new(**@options.oauth_options)
+          elsif @options.oauth_session
+            @oauth_session = @options.oauth_session.dup
           end
+        end
 
-          case grant_type
-          when "client_credentials"
-            # do nothing
-          when "refresh_token"
-            form_post["refresh_token"] = oauth_session.refresh_token
-          end
+        def initialize_dup(other)
+          super
+          @oauth_session = other.instance_variable_get(:@oauth_session).dup
+        end
+
+        def oauth_auth(**args)
+          warn "DEPRECATION WARNING: `#{__method__}` is deprecated. " \
+               "Use `with(oauth_options: options)` instead."
 
-          token_request = build_request("POST", oauth_session.token_endpoint, headers: headers, form: form_post)
-          token_request.headers.delete("authorization") unless oauth_session.token_endpoint_auth_method == "client_secret_basic"
+          with(oauth_options: args)
+        end
 
-          token_response = request(token_request)
-          token_response.raise_for_status
+        # will eagerly negotiate new oauth tokens with the issuer
+        def refresh_oauth_tokens!
+          return unless @oauth_session
 
-          payload = token_response.json
+          @oauth_session.reset!
+          @oauth_session.fetch_access_token(self)
+        end
 
-          access_token = payload["access_token"]
-          refresh_token = payload["refresh_token"]
+        # TODO: deprecate
+        def with_access_token
+          warn "DEPRECATION WARNING: `#{__method__}` is deprecated. " \
+               "The session will automatically handle token lifecycles for you."
 
-          with(oauth_session: oauth_session.merge(access_token: access_token, refresh_token: refresh_token))
+          other_session = dup # : instance
+          oauth_session = other_session.oauth_session
+          oauth_session.fetch_access_token(other_session)
+          other_session
         end
 
-        def build_request(*)
-          request = super
+        private
 
-          return request if request.headers.key?("authorization")
+        def generate_auth_token
+          return unless @oauth_session
 
-          oauth_session = @options.oauth_session
+          @oauth_session.fetch_access_token(self)
+        end
 
-          return request unless oauth_session && oauth_session.access_token
+        def dynamic_auth_token?(_)
+          @oauth_session
+        end
+      end
 
-          request.headers["authorization"] = "Bearer #{oauth_session.access_token}"
+      module OAuthRetries
+        module InstanceMethods
+          private
 
-          request
+          def prepare_to_retry(_request, response)
+            @oauth_session.reset! if @oauth_session
+
+            super
+          end
         end
       end
     end

data/lib/httpx/plugins/persistent.rb

@@ -55,10 +55,8 @@ module HTTPX
 
         private
 
-        def repeatable_request?(request, _)
+        def retryable_request?(request, response, *)
           super || begin
-            response = request.response
-
             return false unless response && response.is_a?(ErrorResponse)
 
             error = response.error
@@ -67,13 +65,13 @@ module HTTPX
           end
         end
 
-        def retryable_error?(ex)
+        def retryable_error?(ex, options)
           super &&
             # under the persistent plugin rules, requests are only retried for connection related errors,
             # which do not include request timeout related errors. This only gets overriden if the end user
             # manually changed +:max_retries+ to something else, which means it is aware of the
             # consequences.
-            (!ex.is_a?(RequestTimeoutError) || @options.max_retries != 1)
+            (!ex.is_a?(RequestTimeoutError) || options.max_retries != 1)
         end
       end
     end

data/lib/httpx/plugins/proxy/http.rb

@@ -43,10 +43,6 @@ module HTTPX
         end
 
         module ConnectionMethods
-          def connecting?
-            super || @state == :connecting || @state == :connected
-          end
-
           def force_close(*)
             if @state == :connecting
               # proxy connect related requests should not be reenqueed

data/lib/httpx/plugins/proxy.rb

@@ -326,7 +326,9 @@ module HTTPX
 
       module ProxyRetries
         module InstanceMethods
-          def retryable_error?(ex)
+          private
+
+          def retryable_error?(ex, *)
             super || ex.is_a?(ProxyConnectionError)
           end
         end

data/lib/httpx/plugins/query.rb

@@ -23,7 +23,7 @@ module HTTPX
         module InstanceMethods
           private
 
-          def repeatable_request?(request, options)
+          def retryable_request?(request, *)
             super || request.verb == "QUERY"
           end
         end

data/lib/httpx/plugins/rate_limiter.rb

@@ -12,22 +12,11 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Rate-Limiter
     #
     module RateLimiter
-      class << self
-        RATE_LIMIT_CODES = [429, 503].freeze
-
-        def configure(klass)
-          klass.plugin(:retries,
-                       retry_change_requests: true,
-                       retry_on: method(:retry_on_rate_limited_response),
-                       retry_after: method(:retry_after_rate_limit))
-        end
-
-        def retry_on_rate_limited_response(response)
-          return false unless response.is_a?(Response)
+      RATE_LIMIT_CODES = [429, 503].freeze
 
-          status = response.status
-
-          RATE_LIMIT_CODES.include?(status)
+      class << self
+        def load_dependencies(klass)
+          klass.plugin(:retries, retry_after: method(:retry_after_rate_limit))
         end
 
         # Servers send the "Retry-After" header field to indicate how long the
@@ -48,6 +37,22 @@ module HTTPX
           Utils.parse_retry_after(retry_after)
         end
       end
+
+      module InstanceMethods
+        private
+
+        def retryable_request?(request, response, options)
+          super || rate_limit_error?(response)
+        end
+
+        def retryable_response?(response, options)
+          rate_limit_error?(response) || super
+        end
+
+        def rate_limit_error?(response)
+          response.is_a?(Response) && RATE_LIMIT_CODES.include?(response.status)
+        end
+      end
     end
 
     register_plugin :rate_limiter, RateLimiter

data/lib/httpx/plugins/response_cache.rb

@@ -150,7 +150,7 @@ module HTTPX
             request.headers.add("if-modified-since", last_modified)
           end
 
-          if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
+          if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"])
             request.headers.add("if-none-match", etag)
           end
         end
@@ -296,9 +296,7 @@ module HTTPX
           return @cache_control if defined?(@cache_control)
 
           @cache_control = begin
-            return unless @headers.key?("cache-control")
-
-            @headers["cache-control"].split(/ *, */)
+            @headers["cache-control"].split(/ *, */) if @headers.key?("cache-control")
           end
         end
 
@@ -307,9 +305,7 @@ module HTTPX
           return @vary if defined?(@vary)
 
           @vary = begin
-            return unless @headers.key?("vary")
-
-            @headers["vary"].split(/ *, */).map(&:downcase)
+            @headers["vary"].split(/ *, */).map(&:downcase) if @headers.key?("vary")
           end
         end
 

data/lib/httpx/plugins/retries.rb

@@ -36,15 +36,33 @@ module HTTPX
         Parser::Error,
         TimeoutError,
       ]).freeze
-      DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
 
-      if ENV.key?("HTTPX_NO_JITTER")
-        def self.extra_options(options)
-          options.merge(max_retries: MAX_RETRIES)
+      DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }.freeze
+
+      # list of supported backoff algorithms
+      BACKOFF_ALGORITHMS = %i[exponential_backoff polynomial_backoff].freeze
+
+      class << self
+        if ENV.key?("HTTPX_NO_JITTER")
+          def extra_options(options)
+            options.merge(max_retries: MAX_RETRIES)
+          end
+        else
+          def extra_options(options)
+            options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
+          end
         end
-      else
-        def self.extra_options(options)
-          options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
+
+        # returns the time to wait before resending +request+ as per the polynomial backoff retry strategy.
+        def retry_after_polynomial_backoff(request, _)
+          offset = request.options.max_retries - request.retries
+          2 * (offset - 1)
+        end
+
+        # returns the time to wait before resending +request+ as per the exponential backoff retry strategy.
+        def retry_after_exponential_backoff(request, _)
+          offset = request.options.max_retries - request.retries
+          (offset - 1) * 2
         end
       end
 
@@ -53,6 +71,7 @@ module HTTPX
       # :max_retries :: max number of times a request will be retried (defaults to <tt>3</tt>).
       # :retry_change_requests :: whether idempotent requests are retried (defaults to <tt>false</tt>).
       # :retry_after:: seconds after which a request is retried; can also be a callable object (i.e. <tt>->(req, res) { ... } </tt>)
+      #                or the name of a supported backoff algorithm (i.e. <tt>:exponential_backoff</tt>).
       # :retry_jitter :: number of seconds applied to *:retry_after* (must be a callable, i.e. <tt>->(retry_after) { ... } </tt>).
       # :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
       #              (i.e. <tt>->(res) { ... }</tt>).
@@ -60,10 +79,26 @@ module HTTPX
         private
 
         def option_retry_after(value)
-          # return early if callable
-          unless value.respond_to?(:call)
-            value = Float(value)
-            raise TypeError, ":retry_after must be positive" unless value.positive?
+          if value.respond_to?(:call)
+            value1 = value
+            value1 = value1.method(:call) unless value1.respond_to?(:arity)
+
+            # allow ->(*) arity as well, which is < 0
+            raise TypeError, "`:retry_after` proc has invalid number of parameters" unless value1.arity.negative? || value1.arity.between?(
+              1, 2
+            )
+
+          else
+            case value
+            when Symbol
+              raise TypeError, "`retry_after`: `#{value}` is not a supported backoff algorithm" unless BACKOFF_ALGORITHMS.include?(value)
+
+              value = Retries.method(:"retry_after_#{value}")
+
+            else
+              value = Float(value)
+              raise TypeError, "`:retry_after` must be positive" unless value.positive?
+            end
           end
 
           value
@@ -107,19 +142,11 @@ module HTTPX
 
           if response &&
              request.retries.positive? &&
-             repeatable_request?(request, options) &&
-             (
-               (
-                 response.is_a?(ErrorResponse) && retryable_error?(response.error)
-               ) ||
-               (
-                 options.retry_on && options.retry_on.call(response)
-               )
-             )
+             retryable_request?(request, response, options) &&
+             retryable_response?(response, options)
             try_partial_retry(request, response)
             log { "failed to get response, #{request.retries} tries to go..." }
-            request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
-            request.transition(:idle)
+            prepare_to_retry(request, response)
 
             retry_after = options.retry_after
             retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
@@ -152,12 +179,16 @@ module HTTPX
         end
 
         # returns whether +request+ can be retried.
-        def repeatable_request?(request, options)
+        def retryable_request?(request, _, options)
           IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
         end
 
+        def retryable_response?(response, options)
+          (response.is_a?(ErrorResponse) && retryable_error?(response.error, options)) || options.retry_on&.call(response)
+        end
+
         # returns whether the +ex+ exception happend for a retriable request.
-        def retryable_error?(ex)
+        def retryable_error?(ex, _)
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
         end
 
@@ -165,6 +196,11 @@ module HTTPX
           super && !request.retries.positive?
         end
 
+        def prepare_to_retry(request, _response)
+          request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
+          request.transition(:idle)
+        end
+
         #
         # Attempt to set the request to perform a partial range request.
         # This happens if the peer server accepts byte-range requests, and

data/lib/httpx/plugins/ssrf_filter.rb

@@ -16,7 +16,7 @@ module HTTPX
             mask_addr = @mask_addr
             raise "Invalid mask" if mask_addr.zero?
 
-            mask_addr >>= 1 while (mask_addr & 0x1).zero?
+            mask_addr >>= 1 while mask_addr.nobits?(0x1)
 
             length = 0
             while mask_addr & 0x1 == 0x1