httpx 1.6.3 → 1.7.0

data/lib/httpx/plugins/fiber_concurrency.rb

@@ -6,12 +6,13 @@ module HTTPX
     #
     # This enables integration with fiber scheduler implementations such as [async](https://github.com/async).
     #
-    # # https://gitlab.com/os85/httpx/wikis/FiberConcurrency
+    # # https://gitlab.com/os85/httpx/wikis/Fiber-Concurrency
     #
     module FiberConcurrency
       def self.subplugins
         {
           h2c: FiberConcurrencyH2C,
+          stream: FiberConcurrencyStream,
         }
       end
 
@@ -188,6 +189,20 @@ module HTTPX
           end
         end
       end
+
+      module FiberConcurrencyStream
+        module StreamResponseMethods
+          def close
+            unless @request.current_context?
+              @request.close
+
+              return
+            end
+
+            super
+          end
+        end
+      end
     end
 
     register_plugin :fiber_concurrency, FiberConcurrency

data/lib/httpx/plugins/grpc/grpc_encoding.rb

@@ -48,7 +48,7 @@ module HTTPX
           until message.empty?
             compressed, size = message.unpack("CL>")
 
-            encoded_data = message.byteslice(5..size + 5 - 1)
+            encoded_data = message.byteslice(5..(size + 5 - 1))
 
             if compressed == 1
               grpc_encodings.reverse_each do |encoding|

data/lib/httpx/plugins/grpc.rb

@@ -249,7 +249,7 @@ module HTTPX
           call
         end
 
-        def build_grpc_request(rpc_method, input, deadline:, metadata: nil, **)
+        def build_grpc_request(rpc_method, input, deadline:, metadata: nil, **opts)
           uri = @options.origin.dup
           rpc_method = "/#{rpc_method}" unless rpc_method.start_with?("/")
           rpc_method = "/#{@options.grpc_service}#{rpc_method}" if @options.grpc_service
@@ -273,7 +273,7 @@ module HTTPX
 
           headers.merge!(@options.call_credentials.call.transform_keys(&:to_s)) if @options.call_credentials
 
-          build_request("POST", uri, headers: headers, body: input)
+          build_request("POST", uri, headers: headers, body: input, **opts)
         end
       end
     end

data/lib/httpx/plugins/internal_telemetry.rb

@@ -45,7 +45,7 @@ module HTTPX
             debug_level: @options ? @options.debug_level : DEBUG_LEVEL,
             debug: nil
           ) do
-            "[ELAPSED TIME]: #{label}: #{elapsed} (ms)" << "\e[0m"
+            "[ELAPSED TIME]: #{label}: #{elapsed} (ms)\e[0m"
           end
         end
       end

data/lib/httpx/plugins/ntlm_auth.rb

@@ -7,8 +7,9 @@ module HTTPX
     #
     module NTLMAuth
       class << self
-        def load_dependencies(_klass)
+        def load_dependencies(klass)
           require_relative "auth/ntlm"
+          klass.plugin(:auth)
         end
 
         def extra_options(options)
@@ -38,14 +39,15 @@ module HTTPX
             ntlm = request.options.ntlm
 
             if ntlm
-              request.headers["authorization"] = ntlm.negotiate
+              request.authorize(ntlm.negotiate)
               probe_response = wrap { super(request).first }
 
               return probe_response unless probe_response.is_a?(Response)
 
               if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
                 request.transition(:idle)
-                request.headers["authorization"] = ntlm.authenticate(request, probe_response.headers["www-authenticate"])
+                request.headers.get("authorization").pop
+                request.authorize(ntlm.authenticate(request, probe_response.headers["www-authenticate"]).encode("utf-8"))
                 super(request)
               else
                 probe_response

data/lib/httpx/plugins/oauth.rb

@@ -2,21 +2,38 @@
 
 module HTTPX
   module Plugins
+    #
+    # This plugin adds support for managing an OAuth Session associated with the given session.
+    #
+    # The scope of OAuth support is limited to the `client_crendentials` and `refresh_token` grants.
     #
     # https://gitlab.com/os85/httpx/wikis/OAuth
     #
     module OAuth
       class << self
-        def load_dependencies(_klass)
+        def load_dependencies(klass)
           require_relative "auth/basic"
+          klass.plugin(:auth)
+        end
+
+        def subplugins
+          {
+            retries: OAuthRetries,
+          }
+        end
+
+        def extra_options(options)
+          options.merge(auth_header_type: "Bearer")
         end
       end
 
       SUPPORTED_GRANT_TYPES = %w[client_credentials refresh_token].freeze
       SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
 
+      # Implements the bulk of functionality and maintains the state associated with the
+      # management of the the lifecycle of an OAuth session.
       class OAuthSession
-        attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope, :audience
+        attr_reader :access_token, :refresh_token
 
         def initialize(
           issuer:,
@@ -27,7 +44,6 @@ module HTTPX
           scope: nil,
           audience: nil,
           token_endpoint: nil,
-          response_type: nil,
           grant_type: nil,
           token_endpoint_auth_method: nil
         )
@@ -35,7 +51,6 @@ module HTTPX
           @client_id = client_id
           @client_secret = client_secret
           @token_endpoint = URI(token_endpoint) if token_endpoint
-          @response_type = response_type
           @scope = case scope
                    when String
                      scope.split
@@ -47,6 +62,8 @@ module HTTPX
           @refresh_token = refresh_token
           @token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
           @grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
+          @access_token = access_token
+          @refresh_token = refresh_token
 
           unless @token_endpoint_auth_method.nil? || SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
             raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
@@ -57,28 +74,75 @@ module HTTPX
           raise Error, "#{@grant_type} is not a supported grant type"
         end
 
+        # returns the URL where to request access and refresh tokens from.
         def token_endpoint
           @token_endpoint || "#{@issuer}/token"
         end
 
+        # returns the oauth-documented authorization method to use when requesting a token.
         def token_endpoint_auth_method
           @token_endpoint_auth_method || "client_secret_basic"
         end
 
-        def load(http)
-          return if @grant_type && @scope
+        def reset!
+          @access_token = nil
+        end
 
-          metadata = http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json
+        # when not available, it uses the +http+ object to request new access and refresh tokens.
+        def fetch_access_token(http)
+          return access_token if access_token
 
-          @token_endpoint = metadata["token_endpoint"]
-          @scope = metadata["scopes_supported"]
-          @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
-          @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
-            SUPPORTED_AUTH_METHODS.include?(am)
+          load(http)
+
+          # always prefer refresh token grant if a refresh token is available
+          grant_type = @refresh_token ? "refresh_token" : @grant_type
+
+          headers = {} # : Hash[String ,String]
+          form_post = {
+            "grant_type" => @grant_type,
+            "scope" => Array(@scope).join(" "),
+            "audience" => @audience,
+          }.compact
+
+          # auth
+          case token_endpoint_auth_method
+          when "client_secret_post"
+            form_post["client_id"] = @client_id
+            form_post["client_secret"] = @client_secret
+          when "client_secret_basic"
+            headers["authorization"] = Authentication::Basic.new(@client_id, @client_secret).authenticate
           end
-          nil
+
+          case grant_type
+          when "client_credentials"
+            # do nothing
+          when "refresh_token"
+            raise Error, "cannot use the `\"refresh_token\"` grant type without a refresh token" unless refresh_token
+
+            form_post["refresh_token"] = refresh_token
+          end
+
+          # POST /token
+          token_request = http.build_request("POST", token_endpoint, headers: headers, form: form_post)
+
+          token_request.headers.delete("authorization") unless token_endpoint_auth_method == "client_secret_basic"
+
+          token_response = http.skip_auth_header { http.request(token_request) }
+
+          begin
+            token_response.raise_for_status
+          rescue HTTPError => e
+            @refresh_token = nil if e.response.status == 401 && (grant_type == "refresh_token")
+            raise e
+          end
+
+          payload = token_response.json
+
+          @refresh_token = payload["refresh_token"] || @refresh_token
+          @access_token = payload["access_token"]
         end
 
+        # TODO: remove this after deprecating the `:oauth_session` option
         def merge(other)
           obj = dup
 
@@ -97,12 +161,32 @@ module HTTPX
           end
           obj
         end
+
+        private
+
+        # uses +http+ to fetch for the oauth server metadata.
+        def load(http)
+          return if @grant_type && @scope
+
+          metadata = http.skip_auth_header { http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json }
+
+          @token_endpoint = metadata["token_endpoint"]
+          @scope = metadata["scopes_supported"]
+          @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
+          @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
+            SUPPORTED_AUTH_METHODS.include?(am)
+          end
+          nil
+        end
       end
 
       module OptionsMethods
         private
 
         def option_oauth_session(value)
+          warn "DEPRECATION WARNING: option `:oauth_session` is deprecated. " \
+               "Use `:oauth_options` instead."
+
           case value
           when Hash
             OAuthSession.new(**value)
@@ -112,69 +196,91 @@ module HTTPX
             raise TypeError, ":oauth_session must be a #{OAuthSession}"
           end
         end
+
+        def option_oauth_options(value)
+          value = Hash[value] unless value.is_a?(Hash)
+          value
+        end
       end
 
       module InstanceMethods
-        def oauth_auth(**args)
-          with(oauth_session: OAuthSession.new(**args))
-        end
+        attr_reader :oauth_session
+        protected :oauth_session
 
-        def with_access_token
-          oauth_session = @options.oauth_session
+        def initialize(*)
+          super
 
-          oauth_session.load(self)
+          @oauth_session = if @options.oauth_options
+            OAuthSession.new(**@options.oauth_options)
+          elsif @options.oauth_session
+            @oauth_session = @options.oauth_session.dup
+          end
+        end
 
-          grant_type = oauth_session.grant_type
+        def initialize_dup(other)
+          super
+          @oauth_session = other.instance_variable_get(:@oauth_session).dup
+        end
 
-          headers = {}
-          form_post = {
-            "grant_type" => grant_type,
-            "scope" => Array(oauth_session.scope).join(" "),
-            "audience" => oauth_session.audience,
-          }.compact
+        def oauth_auth(**args)
+          warn "DEPRECATION WARNING: `#{__method__}` is deprecated. " \
+               "Use `with(oauth_options: options)` instead."
 
-          # auth
-          case oauth_session.token_endpoint_auth_method
-          when "client_secret_post"
-            form_post["client_id"] = oauth_session.client_id
-            form_post["client_secret"] = oauth_session.client_secret
-          when "client_secret_basic"
-            headers["authorization"] = Authentication::Basic.new(oauth_session.client_id, oauth_session.client_secret).authenticate
-          end
+          with(oauth_options: args)
+        end
 
-          case grant_type
-          when "client_credentials"
-            # do nothing
-          when "refresh_token"
-            form_post["refresh_token"] = oauth_session.refresh_token
-          end
+        # will eagerly negotiate new oauth tokens with the issuer
+        def refresh_oauth_tokens!
+          return unless @oauth_session
+
+          @oauth_session.reset!
+          @oauth_session.fetch_access_token(self)
+        end
 
-          token_request = build_request("POST", oauth_session.token_endpoint, headers: headers, form: form_post)
-          token_request.headers.delete("authorization") unless oauth_session.token_endpoint_auth_method == "client_secret_basic"
+        # TODO: deprecate
+        def with_access_token
+          warn "DEPRECATION WARNING: `#{__method__}` is deprecated. " \
+               "The session will automatically handle token lifecycles for you."
 
-          token_response = request(token_request)
-          token_response.raise_for_status
+          other_session = dup # : instance
+          oauth_session = other_session.oauth_session
+          oauth_session.fetch_access_token(other_session)
+          other_session
+        end
 
-          payload = token_response.json
+        private
 
-          access_token = payload["access_token"]
-          refresh_token = payload["refresh_token"]
+        def generate_auth_token
+          return unless @oauth_session
 
-          with(oauth_session: oauth_session.merge(access_token: access_token, refresh_token: refresh_token))
+          @oauth_session.fetch_access_token(self)
         end
+      end
 
-        def build_request(*)
-          request = super
-
-          return request if request.headers.key?("authorization")
+      module OAuthRetries
+        class << self
+          def extra_options(options)
+            options.merge(
+              retry_on: method(:response_oauth_error?),
+              generate_auth_value_on_retry: method(:response_oauth_error?)
+            )
+          end
 
-          oauth_session = @options.oauth_session
+          def response_oauth_error?(res)
+            res.is_a?(Response) && res.status == 401
+          end
+        end
 
-          return request unless oauth_session && oauth_session.access_token
+        module InstanceMethods
+          def prepare_to_retry(_request, response)
+            unless @oauth_session && @options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response)
+              return super
+            end
 
-          request.headers["authorization"] = "Bearer #{oauth_session.access_token}"
+            @oauth_session.reset!
 
-          request
+            super
+          end
         end
       end
     end

data/lib/httpx/plugins/rate_limiter.rb

@@ -18,11 +18,11 @@ module HTTPX
         def configure(klass)
           klass.plugin(:retries,
                        retry_change_requests: true,
-                       retry_on: method(:retry_on_rate_limited_response),
+                       retry_on: method(:retry_on_rate_limited_response?),
                        retry_after: method(:retry_after_rate_limit))
         end
 
-        def retry_on_rate_limited_response(response)
+        def retry_on_rate_limited_response?(response)
           return false unless response.is_a?(Response)
 
           status = response.status

data/lib/httpx/plugins/response_cache.rb

@@ -150,7 +150,7 @@ module HTTPX
             request.headers.add("if-modified-since", last_modified)
           end
 
-          if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
+          if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"])
             request.headers.add("if-none-match", etag)
           end
         end
@@ -296,9 +296,7 @@ module HTTPX
           return @cache_control if defined?(@cache_control)
 
           @cache_control = begin
-            return unless @headers.key?("cache-control")
-
-            @headers["cache-control"].split(/ *, */)
+            @headers["cache-control"].split(/ *, */) if @headers.key?("cache-control")
           end
         end
 
@@ -307,9 +305,7 @@ module HTTPX
           return @vary if defined?(@vary)
 
           @vary = begin
-            return unless @headers.key?("vary")
-
-            @headers["vary"].split(/ *, */).map(&:downcase)
+            @headers["vary"].split(/ *, */).map(&:downcase) if @headers.key?("vary")
           end
         end
 

data/lib/httpx/plugins/retries.rb

@@ -36,15 +36,33 @@ module HTTPX
         Parser::Error,
         TimeoutError,
       ]).freeze
-      DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
 
-      if ENV.key?("HTTPX_NO_JITTER")
-        def self.extra_options(options)
-          options.merge(max_retries: MAX_RETRIES)
+      DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }.freeze
+
+      # list of supported backoff algorithms
+      BACKOFF_ALGORITHMS = %i[exponential_backoff polynomial_backoff].freeze
+
+      class << self
+        if ENV.key?("HTTPX_NO_JITTER")
+          def extra_options(options)
+            options.merge(max_retries: MAX_RETRIES)
+          end
+        else
+          def extra_options(options)
+            options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
+          end
+        end
+
+        # returns the time to wait before resending +request+ as per the polynomial backoff retry strategy.
+        def retry_after_polynomial_backoff(request, _)
+          offset = request.options.max_retries - request.retries
+          2 * (offset - 1)
         end
-      else
-        def self.extra_options(options)
-          options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
+
+        # returns the time to wait before resending +request+ as per the exponential backoff retry strategy.
+        def retry_after_exponential_backoff(request, _)
+          offset = request.options.max_retries - request.retries
+          (offset - 1) * 2
         end
       end
 
@@ -53,6 +71,7 @@ module HTTPX
       # :max_retries :: max number of times a request will be retried (defaults to <tt>3</tt>).
       # :retry_change_requests :: whether idempotent requests are retried (defaults to <tt>false</tt>).
       # :retry_after:: seconds after which a request is retried; can also be a callable object (i.e. <tt>->(req, res) { ... } </tt>)
+      #                or the name of a supported backoff algorithm (i.e. <tt>:exponential_backoff</tt>).
       # :retry_jitter :: number of seconds applied to *:retry_after* (must be a callable, i.e. <tt>->(retry_after) { ... } </tt>).
       # :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
       #              (i.e. <tt>->(res) { ... }</tt>).
@@ -60,10 +79,26 @@ module HTTPX
         private
 
         def option_retry_after(value)
-          # return early if callable
-          unless value.respond_to?(:call)
-            value = Float(value)
-            raise TypeError, ":retry_after must be positive" unless value.positive?
+          if value.respond_to?(:call)
+            value1 = value
+            value1 = value1.method(:call) unless value1.respond_to?(:arity)
+
+            # allow ->(*) arity as well, which is < 0
+            raise TypeError, "`:retry_after` proc has invalid number of parameters" unless value1.arity.negative? || value1.arity.between?(
+              1, 2
+            )
+
+          else
+            case value
+            when Symbol
+              raise TypeError, "`retry_after`: `#{value}` is not a supported backoff algorithm" unless BACKOFF_ALGORITHMS.include?(value)
+
+              value = Retries.method(:"retry_after_#{value}")
+
+            else
+              value = Float(value)
+              raise TypeError, "`:retry_after` must be positive" unless value.positive?
+            end
           end
 
           value
@@ -112,14 +147,13 @@ module HTTPX
                (
                  response.is_a?(ErrorResponse) && retryable_error?(response.error)
                ) ||
-               (
-                 options.retry_on && options.retry_on.call(response)
-               )
+
+                 options.retry_on&.call(response)
+
              )
             try_partial_retry(request, response)
             log { "failed to get response, #{request.retries} tries to go..." }
-            request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
-            request.transition(:idle)
+            prepare_to_retry(request, response)
 
             retry_after = options.retry_after
             retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
@@ -165,6 +199,11 @@ module HTTPX
           super && !request.retries.positive?
         end
 
+        def prepare_to_retry(request, _response)
+          request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
+          request.transition(:idle)
+        end
+
         #
         # Attempt to set the request to perform a partial range request.
         # This happens if the peer server accepts byte-range requests, and

data/lib/httpx/plugins/ssrf_filter.rb

@@ -16,7 +16,7 @@ module HTTPX
             mask_addr = @mask_addr
             raise "Invalid mask" if mask_addr.zero?
 
-            mask_addr >>= 1 while (mask_addr & 0x1).zero?
+            mask_addr >>= 1 while mask_addr.nobits?(0x1)
 
             length = 0
             while mask_addr & 0x1 == 0x1

data/lib/httpx/plugins/stream.rb

@@ -55,9 +55,9 @@ module HTTPX
         line << chunk
 
         while (idx = line.index("\n"))
-          yield line.byteslice(0..idx - 1)
+          yield line.byteslice(0..(idx - 1))
 
-          line = line.byteslice(idx + 1..-1)
+          line = line.byteslice((idx + 1)..-1)
         end
       end
 
@@ -121,20 +121,71 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Stream
     #
     module Stream
+      STREAM_REQUEST_OPTIONS = { timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 }.freeze }.freeze
+
       def self.extra_options(options)
-        options.merge(timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 })
+        options.merge(
+          stream: false,
+          timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 },
+          stream_response_class: Class.new(StreamResponse, &Options::SET_TEMPORARY_NAME).freeze
+        )
+      end
+
+      # adds support for the following options:
+      #
+      # :stream :: whether the request to process should be handled as a stream (defaults to <tt>false</tt>).
+      # :stream_response_class :: Class used to build the stream response object.
+      module OptionsMethods
+        def option_stream(val)
+          val
+        end
+
+        def option_stream_response_class(value)
+          value
+        end
+
+        def extend_with_plugin_classes(pl)
+          return super unless defined?(pl::StreamResponseMethods)
+
+          @stream_response_class = @stream_response_class.dup
+          Options::SET_TEMPORARY_NAME[@stream_response_class, pl]
+          @stream_response_class.__send__(:include, pl::StreamResponseMethods) if defined?(pl::StreamResponseMethods)
+
+          super
+        end
       end
 
       module InstanceMethods
-        def request(*args, stream: false, **options)
-          return super(*args, **options) unless stream
+        def request(*args, **options)
+          if args.first.is_a?(Request)
+            requests = args
+
+            request = requests.first
+
+            unless request.options.stream && !request.stream
+              if options[:stream]
+                warn "passing `stream: true` with a request obkect is not supported anymore. " \
+                     "You can instead build the request object with `stream :true`"
+              end
+              return super
+            end
+          else
+            return super unless options[:stream]
+
+            requests = build_requests(*args, options)
+
+            request = requests.first
+          end
 
-          requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
           raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
 
-          request = requests.first
+          @options.stream_response_class.new(request, self)
+        end
+
+        def build_request(verb, uri, params = EMPTY_HASH, options = @options)
+          return super unless params[:stream]
 
-          StreamResponse.new(request, self)
+          super(verb, uri, params, options.merge(STREAM_REQUEST_OPTIONS.merge(stream: true)))
         end
       end