httpx 1.7.8 → 1.8.1

data/lib/httpx/plugins/auth.rb

@@ -20,6 +20,9 @@ module HTTPX
       #
       # :auth_header_value :: the token to use as a string, or a callable which returns a string when called.
       # :auth_header_type :: the authentication type to use in the "authorization" header value (i.e. "Bearer", "Digest"...)
+      # :auth_header_expires_at :: timestamp at which the auth header will be discarded. should be a callable (like a proc)
+      #                            receiving the request as an argument, and should return either a Time object, or an integer (UNIX time).
+      # :auth_header_expires_in :: time (in seconds) since the first use of an auth header after which that header will be discarded.
       # :generate_auth_value_on_retry :: callable which returns whether the request should regenerate the auth_header_value
       #                                  when the request is retried (this option will only work if the session also loads the
       #                                  <tt>:retries</tt> plugin).
@@ -32,6 +35,22 @@ module HTTPX
           value
         end
 
+        def option_auth_header_expires_at(value)
+          unless value.respond_to?(:call)
+            value = Float(value)
+            raise TypeError, "`:auth_header_expires_at` must be positive" unless value.positive?
+          end
+
+          value
+        end
+
+        def option_auth_header_expires_in(value)
+          value = Float(value)
+          raise TypeError, "`:auth_header_expires_in` must be positive" unless value.positive?
+
+          value
+        end
+
         def option_generate_auth_value_on_retry(value)
           raise TypeError, "`:generate_auth_value_on_retry` must be a callable" unless value.respond_to?(:call)
 
@@ -43,7 +62,7 @@ module HTTPX
         def initialize(*)
           super
 
-          @auth_header_value = nil
+          @auth_header_value = @auth_header_expires_at = nil
           @auth_header_value_mtx = Thread::Mutex.new
           @skip_auth_header_value = false
         end
@@ -65,7 +84,7 @@ module HTTPX
 
         def reset_auth_header_value!
           @auth_header_value_mtx.synchronize do
-            @auth_header_value = nil
+            @auth_header_value = @auth_header_expires_at = nil
           end
         end
 
@@ -75,7 +94,12 @@ module HTTPX
           return super if @skip_auth_header_value || request.authorized?
 
           auth_header_value = @auth_header_value_mtx.synchronize do
-            @auth_header_value ||= generate_auth_token
+            try_invalidate_auth_header_value
+
+            @auth_header_value ||= begin
+              set_auth_header_expires_at(request)
+              generate_auth_token
+            end
           end
 
           request.authorize(auth_header_value) if auth_header_value
@@ -83,6 +107,14 @@ module HTTPX
           super
         end
 
+        def try_invalidate_auth_header_value
+          return unless (expires_at = @auth_header_expires_at)
+
+          return if expires_at > Time.now.utc.to_i
+
+          @auth_header_value = @auth_header_expires_at = nil
+        end
+
         def generate_auth_token
           return unless (auth_value = @options.auth_header_value)
 
@@ -91,6 +123,19 @@ module HTTPX
           auth_value
         end
 
+        def set_auth_header_expires_at(request)
+          @auth_header_expires_at = if (expires_in = request.options.auth_header_expires_in)
+            Time.now.to_i + expires_in
+          elsif (expires_at = request.options.auth_header_expires_at)
+            if expires_at.respond_to?(:call) && (expires_at = expires_at.call(request))
+              expires_at = expires_at.to_f
+              raise Error, "`:auth_header_expires_at` must be positive" unless expires_at.positive?
+
+              expires_at
+            end
+          end
+        end
+
         def dynamic_auth_token?(auth_header_value)
           auth_header_value&.respond_to?(:call)
         end
@@ -150,12 +195,24 @@ module HTTPX
             # otherwise, it means that the first request already passed here, so this request should
             # use whatever was generated for it.
             @auth_header_value_mtx.synchronize do
-              @auth_header_value = generate_auth_token if request.auth_token_value == @auth_header_value
+              if request.auth_token_value == @auth_header_value
+                @auth_header_value = generate_auth_token
+                set_auth_header_expires_at(request)
+              end
             end
 
             request.unauthorize!
           end
 
+          def when_to_retry(request, response, *)
+            return super unless response.is_a?(Response)
+
+            # retry immediately if authentication no longer valid.
+            return if response.status == 401
+
+            super
+          end
+
           def auth_error?(response, options)
             response.is_a?(Response) && response.status == 401 && dynamic_auth_token?(options.auth_header_value)
           end

data/lib/httpx/plugins/{response_cache → cache}/file_store.rb

@@ -3,7 +3,7 @@
 require "pathname"
 
 module HTTPX::Plugins
-  module ResponseCache
+  module Cache
     # Implementation of a file system based cache store.
     #
     # It stores cached responses in a file under a directory pointed by the +dir+

data/lib/httpx/plugins/{response_cache → cache}/store.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX::Plugins
-  module ResponseCache
+  module Cache
     # Implementation of a thread-safe in-memory cache store.
     class Store
       def initialize

data/lib/httpx/plugins/cache.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for caching and reusing responses
+    #
+    # https://gitlab.com/os85/httpx/wikis/Cache
+    #
+    module Cache
+      class << self
+        def load_dependencies(*)
+          require_relative "cache/store"
+          require_relative "cache/file_store"
+        end
+
+        def extra_options(options)
+          options.merge(
+            response_cache_store: :store,
+          )
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :cache_key :: callable which receives a request and returns the corresponding cache key as a string
+      #               (to be used by the cache store when storing cached responses)
+      # :cacheable_request :: callable which receives a request and returns whether this request can use a previously cached response,
+      #                       or for which a freshly retrieved response can be cached.
+      # :cacheable_response :: callable which receives a request and a (freshly retrieved) response and returns whether the response
+      #                        can be cached.
+      # :valid_cached_response :: callable which receives a request and a (previously cached) response and returns whether the response
+      #                           can still be used / returned to the caller.
+      # :response_cache_store :: object where cached responses are fetch from or stored in; defaults to <tt>:store</tt> (in-memory
+      #                          cache), can be set to <tt>:file_store</tt> (file system cache store) as well, or any object which
+      #                          abides by the Cache Store Interface
+      #
+      # The Cache Store Interface requires implementation of the following methods:
+      #
+      # * +#get(request) -> response or nil+
+      # * +#set(request, response) -> void+
+      # * +#clear() -> void+)
+      #
+      module OptionsMethods
+        private
+
+        def option_cache_key(v)
+          raise TypeError, "`:cache_key` must be a callable" unless v.respond_to?(:call)
+
+          v
+        end
+
+        def option_cacheable_request(v)
+          raise TypeError, "`:cacheable_request` must be a callable" unless v.respond_to?(:call)
+
+          v
+        end
+
+        def option_cacheable_response(v)
+          raise TypeError, "`:cacheable_response` must be a callable" unless v.respond_to?(:call)
+
+          v
+        end
+
+        def option_valid_cached_response(v)
+          raise TypeError, "`:valid_cached_response` must be a callable" unless v.respond_to?(:call)
+
+          v
+        end
+
+        def option_response_cache_store(value)
+          case value
+          when :store
+            Store.new
+          when :file_store
+            FileStore.new
+          else
+            value
+          end
+        end
+      end
+
+      module InstanceMethods
+        # wipes out all cached responses from the cache store.
+        def clear_response_cache
+          @options.response_cache_store.clear
+        end
+
+        def build_request(*)
+          request = super
+          return request unless cacheable_request?(request)
+
+          prepare_cache(request)
+
+          request
+        end
+
+        private
+
+        def send_request(request, *)
+          return request if request.response
+
+          super
+        end
+
+        def fetch_response(request, *)
+          response = super
+
+          return unless response
+
+          if cacheable_request?(request) && cacheable_response?(request, response) && !response.cached?
+            log { "caching response for #{request.uri}..." }
+            request.options.response_cache_store.set(request, response)
+          end
+
+          response
+        end
+
+        # whether +request+ can use cached responses.
+        def cacheable_request?(request)
+          return false unless (call = request.options.cacheable_request)
+
+          call[request]
+        end
+
+        # whether the retrieved +response+ can be cached.
+        def cacheable_response?(request, response)
+          return false unless (call = request.options.cacheable_response)
+
+          call[request, response]
+        end
+
+        # whether the cached +cached_response+ is still valid for the current +request+
+        def valid_cached_response?(request, cached_response)
+          return false unless (call = request.options.valid_cached_response)
+
+          call[request, cached_response]
+        end
+
+        # will either assign a still-fresh cached response to +request+, or set up its HTTP
+        # cache invalidation headers in case it's not fresh anymore.
+        def prepare_cache(request)
+          cached_response = retrieve_cached_response(request)
+
+          return unless cached_response && valid_cached_response?(request, cached_response)
+
+          request.cached_response = nil
+
+          # if the cached response is still usable, we use it
+          cached_response.body.rewind
+          cached_response = cached_response.dup
+          cached_response.mark_as_cached!
+          request.response = cached_response
+          request.emit_response(cached_response)
+        end
+
+        # calls the cache store to retrieve the cached response for +request+. Caches it
+        # for convenience of subplugins in order to minimize overhead of retrieval (which may
+        # involve network).
+        def retrieve_cached_response(request)
+          request.cached_response ||= request.options.response_cache_store.get(request)
+        end
+      end
+
+      module RequestMethods
+        # points to a previously cached Response corresponding to this request.
+        attr_accessor :cached_response
+
+        def initialize(*)
+          super
+          @cached_response = nil
+        end
+
+        def merge_headers(*)
+          super
+          @response_cache_key = nil
+        end
+
+        # returns a unique cache key as a String identifying this request
+        def response_cache_key
+          return unless (call = @options.cache_key)
+
+          call[self]
+        end
+      end
+
+      module ResponseMethods
+        attr_writer :original_request
+
+        def initialize(*)
+          super
+          @cached = false
+        end
+
+        # a copy of the request this response was originally cached from
+        def original_request
+          @original_request || @request
+        end
+
+        # whether this Response was duplicated from a previously {RequestMethods#cached_response}.
+        def cached?
+          @cached
+        end
+
+        # sets this Response as being duplicated from a previously cached response.
+        def mark_as_cached!
+          @cached = true
+        end
+      end
+
+      module ResponseBodyMethods
+        def decode_chunk(chunk)
+          return chunk if @response.cached?
+
+          super
+        end
+      end
+    end
+    register_plugin :cache, Cache
+  end
+end

data/lib/httpx/plugins/callbacks.rb

@@ -122,6 +122,18 @@ module HTTPX
         end
       end
 
+      module RequestMethods
+        def drain_body
+          super.tap do |chunk|
+            emit(:body_chunk, chunk) if chunk
+          rescue StandardError => e
+            # in case an error occurs in callback code
+            @drain_error = e
+            nil
+          end
+        end
+      end
+
       module ConnectionMethods
         private
 

data/lib/httpx/plugins/digest_auth.rb

@@ -51,9 +51,13 @@ module HTTPX
 
             if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
               request.transition(:idle)
+
               if (auth_header = digest.authenticate(request, probe_response.headers["www-authenticate"]))
                 request.authorize(auth_header)
               end
+
+              next(request.response) if request.response
+
               super(request)
             else
               probe_response

data/lib/httpx/plugins/fiber_concurrency.rb

@@ -70,18 +70,56 @@ module HTTPX
           )
         end
 
+        def initial_call
+          return unless current_context?
+
+          super
+        end
+
         def interests
           return if connecting? && @pending.none?(&:current_context?)
 
           super
         end
 
-        def send(request)
-          # DoH requests bypass the session, so context needs to be set here.
-          request.set_context!
+        def on_io_error(e)
+          return super unless e.is_a?(IOError) && e.message.include?("stream closed in another thread")
+
+          # @fiber-switch-guard
+          # sockets closed during fiber scheduler switches are raised in separate fibers than the fiber the
+          # socket may be used in. this check verifies that this is actually about this socket.
+          return unless to_io.closed?
+
+          if @state == :closing
+            # @fiber-switch-guard
+            # if the connection is reused across fibers, the socket may have been closed in the other fiber
+            # and switched here during the process, so continue what it was doing and transition to closed
+            # via #call.
+            call
+          elsif !backlog?
+            super
+          end
+        end
+
+        def on_connect_error(e)
+          return super unless e.is_a?(IOError) && e.message.include?("stream closed in another thread")
+
+          # @fiber-switch-guard
+          # sockets closed during fiber scheduler switches are raised in separate fibers than the fiber the
+          # socket may be used in. this check verifies that this is actually about this socket.
+          return unless to_io.closed? && !backlog?
 
           super
         end
+
+        private
+
+        # checks whether the connection has any pending request (which the connection itself may
+        # have stored, or it may be somewhere in the parser).
+        def backlog?
+          @pending.any? ||
+            (@parser && (@parser.pending.any? || @parser.requests.any?))
+        end
       end
 
       module HTTP1Methods
@@ -161,6 +199,12 @@ module HTTPX
       end
 
       module ResolverNativeMethods
+        def initial_call
+          return unless @queries.values.any?(&:current_context?) || @connections.any?(&:current_context?)
+
+          super
+        end
+
         def calculate_interests
           return if @queries.empty?
 
@@ -168,14 +212,41 @@ module HTTPX
 
           super
         end
+
+        def disconnect
+          return unless @connections.all?(&:current_context?)
+
+          super
+        end
+
+        def on_io_error(e)
+          # TODO: return super if this is not stream clsed in another thread
+
+          log { "IO Erroring: #{e.message}, current:#{@name}, queries:#{@queries.size}" }
+          return unless @name
+
+          super
+        end
       end
 
       module ResolverSystemMethods
+        def initial_call
+          return unless current_context?
+
+          super
+        end
+
         def interests
-          return unless @queries.any? { |_, conn| conn.current_context? }
+          return unless current_context?
 
           super
         end
+
+        private
+
+        def current_context?
+          @queries.any? { |_, conn| conn.current_context? }
+        end
       end
 
       module FiberConcurrencyH2C

data/lib/httpx/plugins/ntlm_v2_auth.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    # https://gitlab.com/os85/httpx/wikis/Auth#ntlm-v2-auth
+    module NtlmV2Auth
+      class << self
+        def load_dependencies(klass)
+          require "rubyntlm"
+          klass.plugin(:auth)
+        end
+
+        def extra_options(options)
+          options.merge(max_concurrent_requests: 1)
+        end
+      end
+
+      class Authenticator
+        def initialize(user, password, domain: nil)
+          @user = user
+          @password = password
+          @domain = domain
+        end
+
+        def can_authenticate?(www_authenticate)
+          www_authenticate && /NTLM/i.match?(www_authenticate)
+        end
+
+        def negotiate
+          t1 = Net::NTLM::Message::Type1.new
+          t1.domain = @domain if @domain
+          "NTLM #{t1.encode64}"
+        end
+
+        def authenticate(_request, www_authenticate)
+          challenge_b64 = www_authenticate[/NTLM (.+)/i, 1]
+          t2 = Net::NTLM::Message.decode64(challenge_b64)
+          t3 = t2.response(
+            { user: @user, password: @password, domain: @domain },
+            ntlmv2: true
+          )
+          "NTLM #{t3.encode64}"
+        end
+      end
+
+      module OptionsMethods
+        private
+
+        def option_ntlm(value)
+          raise TypeError, ":ntlm must be a #{Authenticator}" unless value.is_a?(NtlmV2Auth::Authenticator)
+
+          value
+        end
+      end
+
+      module InstanceMethods
+        def ntlm_auth(user, password, domain = nil)
+          with(ntlm: Authenticator.new(user, password, domain: domain))
+        end
+
+        private
+
+        def send_requests(*requests)
+          requests.flat_map do |request|
+            ntlm = request.options.ntlm
+
+            if ntlm
+              request.authorize(ntlm.negotiate)
+              probe_response = wrap { super(request).first }
+
+              return probe_response unless probe_response.is_a?(Response)
+
+              if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
+                request.transition(:idle)
+                request.unauthorize!
+                request.authorize(ntlm.authenticate(request,
+                                                    probe_response.headers["www-authenticate"]).encode("utf-8"))
+                super(request)
+              else
+                probe_response
+              end
+            else
+              super(request)
+            end
+          end
+        end
+      end
+    end
+
+    register_plugin :ntlm_v2_auth, NtlmV2Auth
+  end
+end