httpx 1.6.2 → 1.7.0

data/lib/httpx/plugins/auth/digest.rb

@@ -8,6 +8,8 @@ module HTTPX
   module Plugins
     module Authentication
       class Digest
+        Error = Class.new(Error)
+
         def initialize(user, password, hashed: false, **)
           @user = user
           @password = password
@@ -29,19 +31,53 @@ module HTTPX
           # discard first token, it's Digest
           auth_info = authenticate[/^(\w+) (.*)/, 2]
 
-          params = auth_info.split(/ *, */)
-                            .to_h { |val| val.split("=", 2) }
-                            .transform_values { |v| v.delete("\"") }
+          raise_format_error unless auth_info
+
+          s = StringScanner.new(auth_info)
+
+          params = {}
+          until s.eos?
+            k = s.scan_until(/=/)
+            raise_format_error unless k&.end_with?("=")
+
+            if s.peek(1) == "\""
+              s.skip("\"")
+              v = s.scan_until(/"/)
+              raise_format_error unless v&.end_with?("\"")
+
+              v = v[0..-2]
+              s.skip_until(/,/)
+            else
+              v = s.scan_until(/,|$/)
+
+              if v&.end_with?(",")
+                v = v[0..-2]
+              else
+                raise_format_error unless s.eos?
+              end
+
+              v = v[0..-2] if v&.end_with?(",")
+            end
+            params[k[0..-2]] = v
+            s.skip(/\s/)
+          end
+
           nonce = params["nonce"]
           nc = next_nonce
 
           # verify qop
           qop = params["qop"]
 
+          if qop
+            # some servers send multiple values wrapped in parentheses (i.e. "(qauth,)")
+            qop = qop[/\(?([^)]+)\)?/, 1]
+            qop = qop.split(",").map { |s| s.delete_prefix("'").delete_suffix("'") }.delete_if(&:empty?).map.first
+          end
+
           if params["algorithm"] =~ /(.*?)(-sess)?$/
             alg = Regexp.last_match(1)
             algorithm = ::Digest.const_get(alg)
-            raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
+            raise Error, "unknown algorithm \"#{alg}\"" unless algorithm
 
             sess = Regexp.last_match(2)
           else
@@ -96,6 +132,10 @@ module HTTPX
         def next_nonce
           @nonce += 1
         end
+
+        def raise_format_error
+          raise Error, "unsupported digest header format"
+        end
       end
     end
   end

data/lib/httpx/plugins/auth.rb

@@ -10,13 +10,96 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Auth#auth
     #
     module Auth
+      def self.subplugins
+        {
+          retries: AuthRetries,
+        }
+      end
+
+      module OptionsMethods
+        def option_auth_header_value(value)
+          value
+        end
+
+        def option_auth_header_type(value)
+          value
+        end
+
+        def option_generate_auth_value_on_retry(value)
+          raise TypeError, "`:generate_auth_value_on_retry` must be a callable" unless value.respond_to?(:call)
+
+          value
+        end
+      end
+
       module InstanceMethods
-        def authorization(token)
-          with(headers: { "authorization" => token })
+        def initialize(*)
+          super
+
+          @auth_header_value = nil
+          @skip_auth_header_value = false
+        end
+
+        def authorization(token = nil, auth_header_type: nil, &blk)
+          with(auth_header_type: auth_header_type, auth_header_value: token || blk)
+        end
+
+        def bearer_auth(token = nil, &blk)
+          authorization(token, auth_header_type: "Bearer", &blk)
+        end
+
+        def skip_auth_header
+          @skip_auth_header_value = true
+          yield
+        ensure
+          @skip_auth_header_value = false
+        end
+
+        def reset_auth_header_value!
+          @auth_header_value = nil
+        end
+
+        private
+
+        def send_request(request, *)
+          return super if @skip_auth_header_value
+
+          @auth_header_value ||= generate_auth_token
+
+          request.authorize(@auth_header_value) if @auth_header_value
+
+          super
+        end
+
+        def generate_auth_token
+          return unless (auth_value = @options.auth_header_value)
+
+          auth_value = auth_value.call(self) if auth_value.respond_to?(:call)
+
+          auth_value
         end
+      end
+
+      module RequestMethods
+        def authorize(auth_value)
+          if (auth_type = @options.auth_header_type)
+            auth_value = "#{auth_type} #{auth_value}"
+          end
+
+          @headers.add("authorization", auth_value)
+        end
+      end
+
+      module AuthRetries
+        module InstanceMethods
+          def prepare_to_retry(request, response)
+            super
+
+            return unless @options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response)
 
-        def bearer_auth(token)
-          authorization("Bearer #{token}")
+            request.headers.get("authorization").pop
+            @auth_header_value = generate_auth_token
+          end
         end
       end
     end

data/lib/httpx/plugins/aws_sdk_authentication.rb

@@ -94,7 +94,6 @@ module HTTPX
           region: AwsSdkAuthentication.region(@options.aws_profile),
           **options
         )
-
           aws_sigv4_authentication(
             credentials: credentials,
             region: region,

data/lib/httpx/plugins/callbacks.rb

@@ -64,7 +64,7 @@ module HTTPX
 
             emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
           end
-          connection.on(:close) do
+          connection.on(:callback_connection_closed) do
             next unless connection.current_session == self
 
             emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
@@ -121,6 +121,20 @@ module HTTPX
           raise e.cause
         end
       end
+
+      module ConnectionMethods
+        private
+
+        def disconnect
+          return if @exhausted
+
+          return unless @current_session && @current_selector
+
+          emit(:callback_connection_closed)
+
+          super
+        end
+      end
     end
     register_plugin :callbacks, Callbacks
   end

data/lib/httpx/plugins/cookies/cookie.rb

@@ -7,6 +7,7 @@ module HTTPX
     # Contains the single cookie info: name, value and attributes.
     class Cookie
       include Comparable
+
       # Maximum number of bytes per cookie (RFC 6265 6.1 requires 4096 at
       # least)
       MAX_LENGTH = 4096

data/lib/httpx/plugins/digest_auth.rb

@@ -8,15 +8,14 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Auth#digest-auth
     #
     module DigestAuth
-      DigestError = Class.new(Error)
-
       class << self
         def extra_options(options)
           options.merge(max_concurrent_requests: 1)
         end
 
-        def load_dependencies(*)
+        def load_dependencies(klass)
           require_relative "auth/digest"
+          klass.plugin(:auth)
         end
       end
 
@@ -48,11 +47,11 @@ module HTTPX
 
             probe_response = wrap { super(request).first }
 
-            return probe_response unless probe_response.is_a?(Response)
+            return [probe_response] * requests.size unless probe_response.is_a?(Response)
 
             if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
               request.transition(:idle)
-              request.headers["authorization"] = digest.authenticate(request, probe_response.headers["www-authenticate"])
+              request.authorize(digest.authenticate(request, probe_response.headers["www-authenticate"]))
               super(request)
             else
               probe_response

data/lib/httpx/plugins/fiber_concurrency.rb

@@ -6,12 +6,13 @@ module HTTPX
     #
     # This enables integration with fiber scheduler implementations such as [async](https://github.com/async).
     #
-    # # https://gitlab.com/os85/httpx/wikis/FiberConcurrency
+    # # https://gitlab.com/os85/httpx/wikis/Fiber-Concurrency
     #
     module FiberConcurrency
       def self.subplugins
         {
           h2c: FiberConcurrencyH2C,
+          stream: FiberConcurrencyStream,
         }
       end
 
@@ -188,6 +189,20 @@ module HTTPX
           end
         end
       end
+
+      module FiberConcurrencyStream
+        module StreamResponseMethods
+          def close
+            unless @request.current_context?
+              @request.close
+
+              return
+            end
+
+            super
+          end
+        end
+      end
     end
 
     register_plugin :fiber_concurrency, FiberConcurrency

data/lib/httpx/plugins/grpc/grpc_encoding.rb

@@ -48,7 +48,7 @@ module HTTPX
           until message.empty?
             compressed, size = message.unpack("CL>")
 
-            encoded_data = message.byteslice(5..size + 5 - 1)
+            encoded_data = message.byteslice(5..(size + 5 - 1))
 
             if compressed == 1
               grpc_encodings.reverse_each do |encoding|

data/lib/httpx/plugins/grpc.rb

@@ -249,7 +249,7 @@ module HTTPX
           call
         end
 
-        def build_grpc_request(rpc_method, input, deadline:, metadata: nil, **)
+        def build_grpc_request(rpc_method, input, deadline:, metadata: nil, **opts)
           uri = @options.origin.dup
           rpc_method = "/#{rpc_method}" unless rpc_method.start_with?("/")
           rpc_method = "/#{@options.grpc_service}#{rpc_method}" if @options.grpc_service
@@ -273,7 +273,7 @@ module HTTPX
 
           headers.merge!(@options.call_credentials.call.transform_keys(&:to_s)) if @options.call_credentials
 
-          build_request("POST", uri, headers: headers, body: input)
+          build_request("POST", uri, headers: headers, body: input, **opts)
         end
       end
     end

data/lib/httpx/plugins/internal_telemetry.rb

@@ -45,7 +45,7 @@ module HTTPX
             debug_level: @options ? @options.debug_level : DEBUG_LEVEL,
             debug: nil
           ) do
-            "[ELAPSED TIME]: #{label}: #{elapsed} (ms)" << "\e[0m"
+            "[ELAPSED TIME]: #{label}: #{elapsed} (ms)\e[0m"
           end
         end
       end

data/lib/httpx/plugins/ntlm_auth.rb

@@ -7,8 +7,9 @@ module HTTPX
     #
     module NTLMAuth
       class << self
-        def load_dependencies(_klass)
+        def load_dependencies(klass)
           require_relative "auth/ntlm"
+          klass.plugin(:auth)
         end
 
         def extra_options(options)
@@ -38,14 +39,15 @@ module HTTPX
             ntlm = request.options.ntlm
 
             if ntlm
-              request.headers["authorization"] = ntlm.negotiate
+              request.authorize(ntlm.negotiate)
               probe_response = wrap { super(request).first }
 
               return probe_response unless probe_response.is_a?(Response)
 
               if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
                 request.transition(:idle)
-                request.headers["authorization"] = ntlm.authenticate(request, probe_response.headers["www-authenticate"])
+                request.headers.get("authorization").pop
+                request.authorize(ntlm.authenticate(request, probe_response.headers["www-authenticate"]).encode("utf-8"))
                 super(request)
               else
                 probe_response

data/lib/httpx/plugins/oauth.rb

@@ -2,21 +2,38 @@
 
 module HTTPX
   module Plugins
+    #
+    # This plugin adds support for managing an OAuth Session associated with the given session.
+    #
+    # The scope of OAuth support is limited to the `client_crendentials` and `refresh_token` grants.
     #
     # https://gitlab.com/os85/httpx/wikis/OAuth
     #
     module OAuth
       class << self
-        def load_dependencies(_klass)
+        def load_dependencies(klass)
           require_relative "auth/basic"
+          klass.plugin(:auth)
+        end
+
+        def subplugins
+          {
+            retries: OAuthRetries,
+          }
+        end
+
+        def extra_options(options)
+          options.merge(auth_header_type: "Bearer")
         end
       end
 
       SUPPORTED_GRANT_TYPES = %w[client_credentials refresh_token].freeze
       SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
 
+      # Implements the bulk of functionality and maintains the state associated with the
+      # management of the the lifecycle of an OAuth session.
       class OAuthSession
-        attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope, :audience
+        attr_reader :access_token, :refresh_token
 
         def initialize(
           issuer:,
@@ -27,7 +44,6 @@ module HTTPX
           scope: nil,
           audience: nil,
           token_endpoint: nil,
-          response_type: nil,
           grant_type: nil,
           token_endpoint_auth_method: nil
         )
@@ -35,7 +51,6 @@ module HTTPX
           @client_id = client_id
           @client_secret = client_secret
           @token_endpoint = URI(token_endpoint) if token_endpoint
-          @response_type = response_type
           @scope = case scope
                    when String
                      scope.split
@@ -47,6 +62,8 @@ module HTTPX
           @refresh_token = refresh_token
           @token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
           @grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
+          @access_token = access_token
+          @refresh_token = refresh_token
 
           unless @token_endpoint_auth_method.nil? || SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
             raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
@@ -57,28 +74,75 @@ module HTTPX
           raise Error, "#{@grant_type} is not a supported grant type"
         end
 
+        # returns the URL where to request access and refresh tokens from.
         def token_endpoint
           @token_endpoint || "#{@issuer}/token"
         end
 
+        # returns the oauth-documented authorization method to use when requesting a token.
         def token_endpoint_auth_method
           @token_endpoint_auth_method || "client_secret_basic"
         end
 
-        def load(http)
-          return if @grant_type && @scope
+        def reset!
+          @access_token = nil
+        end
 
-          metadata = http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json
+        # when not available, it uses the +http+ object to request new access and refresh tokens.
+        def fetch_access_token(http)
+          return access_token if access_token
 
-          @token_endpoint = metadata["token_endpoint"]
-          @scope = metadata["scopes_supported"]
-          @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
-          @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
-            SUPPORTED_AUTH_METHODS.include?(am)
+          load(http)
+
+          # always prefer refresh token grant if a refresh token is available
+          grant_type = @refresh_token ? "refresh_token" : @grant_type
+
+          headers = {} # : Hash[String ,String]
+          form_post = {
+            "grant_type" => @grant_type,
+            "scope" => Array(@scope).join(" "),
+            "audience" => @audience,
+          }.compact
+
+          # auth
+          case token_endpoint_auth_method
+          when "client_secret_post"
+            form_post["client_id"] = @client_id
+            form_post["client_secret"] = @client_secret
+          when "client_secret_basic"
+            headers["authorization"] = Authentication::Basic.new(@client_id, @client_secret).authenticate
           end
-          nil
+
+          case grant_type
+          when "client_credentials"
+            # do nothing
+          when "refresh_token"
+            raise Error, "cannot use the `\"refresh_token\"` grant type without a refresh token" unless refresh_token
+
+            form_post["refresh_token"] = refresh_token
+          end
+
+          # POST /token
+          token_request = http.build_request("POST", token_endpoint, headers: headers, form: form_post)
+
+          token_request.headers.delete("authorization") unless token_endpoint_auth_method == "client_secret_basic"
+
+          token_response = http.skip_auth_header { http.request(token_request) }
+
+          begin
+            token_response.raise_for_status
+          rescue HTTPError => e
+            @refresh_token = nil if e.response.status == 401 && (grant_type == "refresh_token")
+            raise e
+          end
+
+          payload = token_response.json
+
+          @refresh_token = payload["refresh_token"] || @refresh_token
+          @access_token = payload["access_token"]
         end
 
+        # TODO: remove this after deprecating the `:oauth_session` option
         def merge(other)
           obj = dup
 
@@ -97,12 +161,32 @@ module HTTPX
           end
           obj
         end
+
+        private
+
+        # uses +http+ to fetch for the oauth server metadata.
+        def load(http)
+          return if @grant_type && @scope
+
+          metadata = http.skip_auth_header { http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json }
+
+          @token_endpoint = metadata["token_endpoint"]
+          @scope = metadata["scopes_supported"]
+          @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
+          @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
+            SUPPORTED_AUTH_METHODS.include?(am)
+          end
+          nil
+        end
       end
 
       module OptionsMethods
         private
 
         def option_oauth_session(value)
+          warn "DEPRECATION WARNING: option `:oauth_session` is deprecated. " \
+               "Use `:oauth_options` instead."
+
           case value
           when Hash
             OAuthSession.new(**value)
@@ -112,69 +196,91 @@ module HTTPX
             raise TypeError, ":oauth_session must be a #{OAuthSession}"
           end
         end
+
+        def option_oauth_options(value)
+          value = Hash[value] unless value.is_a?(Hash)
+          value
+        end
       end
 
       module InstanceMethods
-        def oauth_auth(**args)
-          with(oauth_session: OAuthSession.new(**args))
-        end
+        attr_reader :oauth_session
+        protected :oauth_session
 
-        def with_access_token
-          oauth_session = @options.oauth_session
+        def initialize(*)
+          super
 
-          oauth_session.load(self)
+          @oauth_session = if @options.oauth_options
+            OAuthSession.new(**@options.oauth_options)
+          elsif @options.oauth_session
+            @oauth_session = @options.oauth_session.dup
+          end
+        end
 
-          grant_type = oauth_session.grant_type
+        def initialize_dup(other)
+          super
+          @oauth_session = other.instance_variable_get(:@oauth_session).dup
+        end
 
-          headers = {}
-          form_post = {
-            "grant_type" => grant_type,
-            "scope" => Array(oauth_session.scope).join(" "),
-            "audience" => oauth_session.audience,
-          }.compact
+        def oauth_auth(**args)
+          warn "DEPRECATION WARNING: `#{__method__}` is deprecated. " \
+               "Use `with(oauth_options: options)` instead."
 
-          # auth
-          case oauth_session.token_endpoint_auth_method
-          when "client_secret_post"
-            form_post["client_id"] = oauth_session.client_id
-            form_post["client_secret"] = oauth_session.client_secret
-          when "client_secret_basic"
-            headers["authorization"] = Authentication::Basic.new(oauth_session.client_id, oauth_session.client_secret).authenticate
-          end
+          with(oauth_options: args)
+        end
 
-          case grant_type
-          when "client_credentials"
-            # do nothing
-          when "refresh_token"
-            form_post["refresh_token"] = oauth_session.refresh_token
-          end
+        # will eagerly negotiate new oauth tokens with the issuer
+        def refresh_oauth_tokens!
+          return unless @oauth_session
+
+          @oauth_session.reset!
+          @oauth_session.fetch_access_token(self)
+        end
 
-          token_request = build_request("POST", oauth_session.token_endpoint, headers: headers, form: form_post)
-          token_request.headers.delete("authorization") unless oauth_session.token_endpoint_auth_method == "client_secret_basic"
+        # TODO: deprecate
+        def with_access_token
+          warn "DEPRECATION WARNING: `#{__method__}` is deprecated. " \
+               "The session will automatically handle token lifecycles for you."
 
-          token_response = request(token_request)
-          token_response.raise_for_status
+          other_session = dup # : instance
+          oauth_session = other_session.oauth_session
+          oauth_session.fetch_access_token(other_session)
+          other_session
+        end
 
-          payload = token_response.json
+        private
 
-          access_token = payload["access_token"]
-          refresh_token = payload["refresh_token"]
+        def generate_auth_token
+          return unless @oauth_session
 
-          with(oauth_session: oauth_session.merge(access_token: access_token, refresh_token: refresh_token))
+          @oauth_session.fetch_access_token(self)
         end
+      end
 
-        def build_request(*)
-          request = super
-
-          return request if request.headers.key?("authorization")
+      module OAuthRetries
+        class << self
+          def extra_options(options)
+            options.merge(
+              retry_on: method(:response_oauth_error?),
+              generate_auth_value_on_retry: method(:response_oauth_error?)
+            )
+          end
 
-          oauth_session = @options.oauth_session
+          def response_oauth_error?(res)
+            res.is_a?(Response) && res.status == 401
+          end
+        end
 
-          return request unless oauth_session && oauth_session.access_token
+        module InstanceMethods
+          def prepare_to_retry(_request, response)
+            unless @oauth_session && @options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response)
+              return super
+            end
 
-          request.headers["authorization"] = "Bearer #{oauth_session.access_token}"
+            @oauth_session.reset!
 
-          request
+            super
+          end
         end
       end
     end