httpx 1.7.8 → 1.8.0

data/lib/httpx/plugins/{response_cache → cache}/store.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX::Plugins
-  module ResponseCache
+  module Cache
     # Implementation of a thread-safe in-memory cache store.
     class Store
       def initialize

data/lib/httpx/plugins/cache.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for caching and reusing responses
+    #
+    # https://gitlab.com/os85/httpx/wikis/Cache
+    #
+    module Cache
+      class << self
+        def load_dependencies(*)
+          require_relative "cache/store"
+          require_relative "cache/file_store"
+        end
+
+        def extra_options(options)
+          options.merge(
+            response_cache_store: :store,
+          )
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :cache_key :: callable which receives a request and returns the corresponding cache key as a string
+      #               (to be used by the cache store when storing cached responses)
+      # :cacheable_request :: callable which receives a request and returns whether this request can use a previously cached response,
+      #                       or for which a freshly retrieved response can be cached.
+      # :cacheable_response :: callable which receives a request and a (freshly retrieved) response and returns whether the response
+      #                        can be cached.
+      # :valid_cached_response :: callable which receives a request and a (previously cached) response and returns whether the response
+      #                           can still be used / returned to the caller.
+      # :response_cache_store :: object where cached responses are fetch from or stored in; defaults to <tt>:store</tt> (in-memory
+      #                          cache), can be set to <tt>:file_store</tt> (file system cache store) as well, or any object which
+      #                          abides by the Cache Store Interface
+      #
+      # The Cache Store Interface requires implementation of the following methods:
+      #
+      # * +#get(request) -> response or nil+
+      # * +#set(request, response) -> void+
+      # * +#clear() -> void+)
+      #
+      module OptionsMethods
+        private
+
+        def option_cache_key(v)
+          raise TypeError, "`:cache_key` must be a callable" unless v.respond_to?(:call)
+
+          v
+        end
+
+        def option_cacheable_request(v)
+          raise TypeError, "`:cacheable_request` must be a callable" unless v.respond_to?(:call)
+
+          v
+        end
+
+        def option_cacheable_response(v)
+          raise TypeError, "`:cacheable_response` must be a callable" unless v.respond_to?(:call)
+
+          v
+        end
+
+        def option_valid_cached_response(v)
+          raise TypeError, "`:valid_cached_response` must be a callable" unless v.respond_to?(:call)
+
+          v
+        end
+
+        def option_response_cache_store(value)
+          case value
+          when :store
+            Store.new
+          when :file_store
+            FileStore.new
+          else
+            value
+          end
+        end
+      end
+
+      module InstanceMethods
+        # wipes out all cached responses from the cache store.
+        def clear_response_cache
+          @options.response_cache_store.clear
+        end
+
+        def build_request(*)
+          request = super
+          return request unless cacheable_request?(request)
+
+          prepare_cache(request)
+
+          request
+        end
+
+        private
+
+        def send_request(request, *)
+          return request if request.response
+
+          super
+        end
+
+        def fetch_response(request, *)
+          response = super
+
+          return unless response
+
+          if cacheable_request?(request) && cacheable_response?(request, response) && !response.cached?
+            log { "caching response for #{request.uri}..." }
+            request.options.response_cache_store.set(request, response)
+          end
+
+          response
+        end
+
+        # whether +request+ can use cached responses.
+        def cacheable_request?(request)
+          return false unless (call = request.options.cacheable_request)
+
+          call[request]
+        end
+
+        # whether the retrieved +response+ can be cached.
+        def cacheable_response?(request, response)
+          return false unless (call = request.options.cacheable_response)
+
+          call[request, response]
+        end
+
+        # whether the cached +cached_response+ is still valid for the current +request+
+        def valid_cached_response?(request, cached_response)
+          return false unless (call = request.options.valid_cached_response)
+
+          call[request, cached_response]
+        end
+
+        # will either assign a still-fresh cached response to +request+, or set up its HTTP
+        # cache invalidation headers in case it's not fresh anymore.
+        def prepare_cache(request)
+          cached_response = retrieve_cached_response(request)
+
+          return unless cached_response && valid_cached_response?(request, cached_response)
+
+          request.cached_response = nil
+
+          # if the cached response is still usable, we use it
+          cached_response.body.rewind
+          cached_response = cached_response.dup
+          cached_response.mark_as_cached!
+          request.response = cached_response
+          request.emit_response(cached_response)
+        end
+
+        # calls the cache store to retrieve the cached response for +request+. Caches it
+        # for convenience of subplugins in order to minimize overhead of retrieval (which may
+        # involve network).
+        def retrieve_cached_response(request)
+          request.cached_response ||= request.options.response_cache_store.get(request)
+        end
+      end
+
+      module RequestMethods
+        # points to a previously cached Response corresponding to this request.
+        attr_accessor :cached_response
+
+        def initialize(*)
+          super
+          @cached_response = nil
+        end
+
+        def merge_headers(*)
+          super
+          @response_cache_key = nil
+        end
+
+        # returns a unique cache key as a String identifying this request
+        def response_cache_key
+          return unless (call = @options.cache_key)
+
+          call[self]
+        end
+      end
+
+      module ResponseMethods
+        attr_writer :original_request
+
+        def initialize(*)
+          super
+          @cached = false
+        end
+
+        # a copy of the request this response was originally cached from
+        def original_request
+          @original_request || @request
+        end
+
+        # whether this Response was duplicated from a previously {RequestMethods#cached_response}.
+        def cached?
+          @cached
+        end
+
+        # sets this Response as being duplicated from a previously cached response.
+        def mark_as_cached!
+          @cached = true
+        end
+      end
+
+      module ResponseBodyMethods
+        def decode_chunk(chunk)
+          return chunk if @response.cached?
+
+          super
+        end
+      end
+    end
+    register_plugin :cache, Cache
+  end
+end

data/lib/httpx/plugins/fiber_concurrency.rb

@@ -76,12 +76,44 @@ module HTTPX
           super
         end
 
-        def send(request)
-          # DoH requests bypass the session, so context needs to be set here.
-          request.set_context!
+        def on_io_error(e)
+          return super unless e.is_a?(IOError) && e.message.include?("stream closed in another thread")
+
+          # @fiber-switch-guard
+          # sockets closed during fiber scheduler switches are raised in separate fibers than the fiber the
+          # socket may be used in. this check verifies that this is actually about this socket.
+          return unless to_io.closed?
+
+          if @state == :closing
+            # @fiber-switch-guard
+            # if the connection is reused across fibers, the socket may have been closed in the other fiber
+            # and switched here during the process, so continue what it was doing and transition to closed
+            # via #call.
+            call
+          elsif !backlog?
+            super
+          end
+        end
+
+        def on_connect_error(e)
+          return super unless e.is_a?(IOError) && e.message.include?("stream closed in another thread")
+
+          # @fiber-switch-guard
+          # sockets closed during fiber scheduler switches are raised in separate fibers than the fiber the
+          # socket may be used in. this check verifies that this is actually about this socket.
+          return unless to_io.closed? && !backlog?
 
           super
         end
+
+        private
+
+        # checks whether the connection has any pending request (which the connection itself may
+        # have stored, or it may be somewhere in the parser).
+        def backlog?
+          @pending.any? ||
+            (@parser && (@parser.pending.any? || @parser.requests.any?))
+        end
       end
 
       module HTTP1Methods
@@ -168,6 +200,21 @@ module HTTPX
 
           super
         end
+
+        def disconnect
+          return unless @connections.all?(&:current_context?)
+
+          super
+        end
+
+        def on_io_error(e)
+          # TODO: return super if this is not stream clsed in another thread
+
+          log { "IO Erroring: #{e.message}, current:#{@name}, queries:#{@queries.size}" }
+          return unless @name
+
+          super
+        end
       end
 
       module ResolverSystemMethods

data/lib/httpx/plugins/ntlm_v2_auth.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    # https://gitlab.com/os85/httpx/wikis/Auth#ntlm-v2-auth
+    module NtlmV2Auth
+      class << self
+        def load_dependencies(klass)
+          require "rubyntlm"
+          klass.plugin(:auth)
+        end
+
+        def extra_options(options)
+          options.merge(max_concurrent_requests: 1)
+        end
+      end
+
+      class Authenticator
+        def initialize(user, password, domain: nil)
+          @user = user
+          @password = password
+          @domain = domain
+        end
+
+        def can_authenticate?(www_authenticate)
+          www_authenticate && /NTLM/i.match?(www_authenticate)
+        end
+
+        def negotiate
+          t1 = Net::NTLM::Message::Type1.new
+          t1.domain = @domain if @domain
+          "NTLM #{t1.encode64}"
+        end
+
+        def authenticate(_request, www_authenticate)
+          challenge_b64 = www_authenticate[/NTLM (.+)/i, 1]
+          t2 = Net::NTLM::Message.decode64(challenge_b64)
+          t3 = t2.response(
+            { user: @user, password: @password, domain: @domain },
+            ntlmv2: true
+          )
+          "NTLM #{t3.encode64}"
+        end
+      end
+
+      module OptionsMethods
+        private
+
+        def option_ntlm(value)
+          raise TypeError, ":ntlm must be a #{Authenticator}" unless value.is_a?(NtlmV2Auth::Authenticator)
+
+          value
+        end
+      end
+
+      module InstanceMethods
+        def ntlm_auth(user, password, domain = nil)
+          with(ntlm: Authenticator.new(user, password, domain: domain))
+        end
+
+        private
+
+        def send_requests(*requests)
+          requests.flat_map do |request|
+            ntlm = request.options.ntlm
+
+            if ntlm
+              request.authorize(ntlm.negotiate)
+              probe_response = wrap { super(request).first }
+
+              return probe_response unless probe_response.is_a?(Response)
+
+              if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
+                request.transition(:idle)
+                request.unauthorize!
+                request.authorize(ntlm.authenticate(request,
+                                                    probe_response.headers["www-authenticate"]).encode("utf-8"))
+                super(request)
+              else
+                probe_response
+              end
+            else
+              super(request)
+            end
+          end
+        end
+      end
+    end
+
+    register_plugin :ntlm_v2_auth, NtlmV2Auth
+  end
+end

data/lib/httpx/plugins/oauth.rb

@@ -12,6 +12,7 @@ module HTTPX
     module OAuth
       class << self
         def load_dependencies(klass)
+          require "monitor"
           require_relative "auth/basic"
           klass.plugin(:auth)
         end
@@ -33,8 +34,6 @@ module HTTPX
       # Implements the bulk of functionality and maintains the state associated with the
       # management of the the lifecycle of an OAuth session.
       class OAuthSession
-        attr_reader :access_token, :refresh_token
-
         def initialize(
           issuer:,
           client_id:,
@@ -62,8 +61,8 @@ module HTTPX
           @refresh_token = refresh_token
           @token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
           @grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
-          @access_token = access_token
-          @refresh_token = refresh_token
+          @expires_at = nil
+          @token_mon = Monitor.new
 
           unless @token_endpoint_auth_method.nil? || SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
             raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
@@ -84,8 +83,24 @@ module HTTPX
           @token_endpoint_auth_method || "client_secret_basic"
         end
 
+        def expires_at
+          @token_mon.synchronize { @expires_at }
+        end
+
+        def access_token
+          @token_mon.synchronize do
+            if (expires_at = @expires_at) && expires_at < Time.now.to_i
+              reset!
+            end
+
+            @access_token
+          end
+        end
+
         def reset!
-          @access_token = nil
+          @token_mon.synchronize do
+            @access_token = @expires_at = nil
+          end
         end
 
         # when not available, it uses the +http+ object to request new access and refresh tokens.
@@ -117,9 +132,11 @@ module HTTPX
           when "client_credentials"
             # do nothing
           when "refresh_token"
-            raise Error, "cannot use the `\"refresh_token\"` grant type without a refresh token" unless refresh_token
+            ref_token = refresh_token
 
-            form_post["refresh_token"] = refresh_token
+            raise Error, "cannot use the `\"refresh_token\"` grant type without a refresh token" unless ref_token
+
+            form_post["refresh_token"] = ref_token
           end
 
           # POST /token
@@ -138,8 +155,13 @@ module HTTPX
 
           payload = token_response.json
 
-          @refresh_token = payload["refresh_token"] || @refresh_token
-          @access_token = payload["access_token"]
+          @token_mon.synchronize do
+            @refresh_token = payload.fetch("refresh_token", @refresh_token)
+            if (expires_in = payload["expires_in"])
+              @expires_at = Time.now.to_i + Integer(expires_in)
+            end
+            @access_token = payload["access_token"]
+          end
         end
 
         # TODO: remove this after deprecating the `:oauth_session` option
@@ -164,17 +186,23 @@ module HTTPX
 
         private
 
+        def refresh_token
+          @token_mon.synchronize { @refresh_token }
+        end
+
         # uses +http+ to fetch for the oauth server metadata.
         def load(http)
           return if @grant_type && @scope
 
           metadata = http.skip_auth_header { http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json }
 
-          @token_endpoint = metadata["token_endpoint"]
-          @scope = metadata["scopes_supported"]
-          @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
-          @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
-            SUPPORTED_AUTH_METHODS.include?(am)
+          @token_mon.synchronize do
+            @token_endpoint = metadata["token_endpoint"]
+            @scope = metadata["scopes_supported"]
+            @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
+            @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
+              SUPPORTED_AUTH_METHODS.include?(am)
+            end
           end
           nil
         end
@@ -239,6 +267,11 @@ module HTTPX
 
           @oauth_session.reset!
           @oauth_session.fetch_access_token(self)
+          if (expires_at = @oauth_session.expires_at)
+            @auth_header_value_mtx.synchronize do
+              @auth_header_expires_at = expires_at
+            end
+          end
         end
 
         # TODO: deprecate
@@ -249,9 +282,18 @@ module HTTPX
           other_session = dup # : instance
           oauth_session = other_session.oauth_session
           oauth_session.fetch_access_token(other_session)
+          if (expires_at = oauth_session.expires_at)
+            @auth_header_expires_at = expires_at
+          end
           other_session
         end
 
+        def reset_auth_header_value!
+          super.tap do
+            @oauth_session.reset if @oauth_session
+          end
+        end
+
         private
 
         def generate_auth_token
@@ -260,6 +302,16 @@ module HTTPX
           @oauth_session.fetch_access_token(self)
         end
 
+        def set_auth_header_expires_at(_)
+          return super unless @oauth_session
+
+          expires_at = @oauth_session.expires_at
+
+          return super unless expires_at
+
+          @auth_header_expires_at = expires_at
+        end
+
         def dynamic_auth_token?(_)
           @oauth_session
         end

data/lib/httpx/plugins/proxy.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "cgi"
+
 module HTTPX
   class ProxyError < ConnectionError; end
 
@@ -65,6 +67,9 @@ module HTTPX
 
           return unless @scheme
 
+          @username = CGI.unescape(@username) if @username
+          @password = CGI.unescape(@password) if @password
+
           @authenticator = load_authenticator(@scheme, @username, @password, **extra)
         end