httpx 1.7.8 → 1.8.1

data/lib/httpx/plugins/oauth.rb

@@ -12,6 +12,7 @@ module HTTPX
     module OAuth
       class << self
         def load_dependencies(klass)
+          require "monitor"
           require_relative "auth/basic"
           klass.plugin(:auth)
         end
@@ -33,8 +34,6 @@ module HTTPX
       # Implements the bulk of functionality and maintains the state associated with the
       # management of the the lifecycle of an OAuth session.
       class OAuthSession
-        attr_reader :access_token, :refresh_token
-
         def initialize(
           issuer:,
           client_id:,
@@ -62,8 +61,8 @@ module HTTPX
           @refresh_token = refresh_token
           @token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
           @grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
-          @access_token = access_token
-          @refresh_token = refresh_token
+          @expires_at = nil
+          @token_mon = Monitor.new
 
           unless @token_endpoint_auth_method.nil? || SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
             raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
@@ -84,8 +83,24 @@ module HTTPX
           @token_endpoint_auth_method || "client_secret_basic"
         end
 
+        def expires_at
+          @token_mon.synchronize { @expires_at }
+        end
+
+        def access_token
+          @token_mon.synchronize do
+            if (expires_at = @expires_at) && expires_at < Time.now.to_i
+              reset!
+            end
+
+            @access_token
+          end
+        end
+
         def reset!
-          @access_token = nil
+          @token_mon.synchronize do
+            @access_token = @expires_at = nil
+          end
         end
 
         # when not available, it uses the +http+ object to request new access and refresh tokens.
@@ -117,9 +132,11 @@ module HTTPX
           when "client_credentials"
             # do nothing
           when "refresh_token"
-            raise Error, "cannot use the `\"refresh_token\"` grant type without a refresh token" unless refresh_token
+            ref_token = refresh_token
 
-            form_post["refresh_token"] = refresh_token
+            raise Error, "cannot use the `\"refresh_token\"` grant type without a refresh token" unless ref_token
+
+            form_post["refresh_token"] = ref_token
           end
 
           # POST /token
@@ -138,8 +155,13 @@ module HTTPX
 
           payload = token_response.json
 
-          @refresh_token = payload["refresh_token"] || @refresh_token
-          @access_token = payload["access_token"]
+          @token_mon.synchronize do
+            @refresh_token = payload.fetch("refresh_token", @refresh_token)
+            if (expires_in = payload["expires_in"])
+              @expires_at = Time.now.to_i + Integer(expires_in)
+            end
+            @access_token = payload["access_token"]
+          end
         end
 
         # TODO: remove this after deprecating the `:oauth_session` option
@@ -164,17 +186,23 @@ module HTTPX
 
         private
 
+        def refresh_token
+          @token_mon.synchronize { @refresh_token }
+        end
+
         # uses +http+ to fetch for the oauth server metadata.
         def load(http)
           return if @grant_type && @scope
 
           metadata = http.skip_auth_header { http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json }
 
-          @token_endpoint = metadata["token_endpoint"]
-          @scope = metadata["scopes_supported"]
-          @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
-          @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
-            SUPPORTED_AUTH_METHODS.include?(am)
+          @token_mon.synchronize do
+            @token_endpoint = metadata["token_endpoint"]
+            @scope = metadata["scopes_supported"]
+            @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
+            @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
+              SUPPORTED_AUTH_METHODS.include?(am)
+            end
           end
           nil
         end
@@ -239,6 +267,11 @@ module HTTPX
 
           @oauth_session.reset!
           @oauth_session.fetch_access_token(self)
+          if (expires_at = @oauth_session.expires_at)
+            @auth_header_value_mtx.synchronize do
+              @auth_header_expires_at = expires_at
+            end
+          end
         end
 
         # TODO: deprecate
@@ -249,9 +282,18 @@ module HTTPX
           other_session = dup # : instance
           oauth_session = other_session.oauth_session
           oauth_session.fetch_access_token(other_session)
+          if (expires_at = oauth_session.expires_at)
+            @auth_header_expires_at = expires_at
+          end
           other_session
         end
 
+        def reset_auth_header_value!
+          super.tap do
+            @oauth_session.reset if @oauth_session
+          end
+        end
+
         private
 
         def generate_auth_token
@@ -260,6 +302,16 @@ module HTTPX
           @oauth_session.fetch_access_token(self)
         end
 
+        def set_auth_header_expires_at(_)
+          return super unless @oauth_session
+
+          expires_at = @oauth_session.expires_at
+
+          return super unless expires_at
+
+          @auth_header_expires_at = expires_at
+        end
+
         def dynamic_auth_token?(_)
           @oauth_session
         end

data/lib/httpx/plugins/persistent.rb

@@ -55,13 +55,29 @@ module HTTPX
 
         private
 
+        def reconnectable_error?(error)
+          Retries::RECONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
+        end
+
+        def when_to_retry(request, response, *)
+          return super unless response.is_a?(ErrorResponse)
+
+          error = response.error
+          # allow request to be retried immediately if the request failed right after the keep alive timeout.
+          # the chances are, the request failed because the connect has been dropped by the peer server, so it's
+          # fine to reopen.
+          return if request.ping? && reconnectable_error?(error)
+
+          super
+        end
+
         def retryable_request?(request, response, *)
           super || begin
-            return false unless response && response.is_a?(ErrorResponse)
+            return false unless response.is_a?(ErrorResponse)
 
             error = response.error
 
-            Retries::RECONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
+            reconnectable_error?(error)
           end
         end
 

data/lib/httpx/plugins/proxy.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "cgi"
+
 module HTTPX
   class ProxyError < ConnectionError; end
 
@@ -65,6 +67,9 @@ module HTTPX
 
           return unless @scheme
 
+          @username = CGI.unescape(@username) if @username
+          @password = CGI.unescape(@password) if @password
+
           @authenticator = load_authenticator(@scheme, @username, @password, **extra)
         end
 

data/lib/httpx/plugins/push_promise.rb

@@ -44,9 +44,9 @@ module HTTPX
 
         def __on_promise_request(parser, stream, h)
           log(level: 1, color: :yellow) do
-            # :nocov:
+            # simplecov:disable
             h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
-            # :nocov:
+            # simplecov:enable
           end
           headers = @options.headers_class.new(h)
           path = headers[":path"]

data/lib/httpx/plugins/response_cache.rb

@@ -3,7 +3,8 @@
 module HTTPX
   module Plugins
     #
-    # This plugin adds support for retrying requests when certain errors happen.
+    # This plugin caches and reuses responses based on HTTP caching directives defined by
+    # the [HTTP Caching RFC](https://www.rfc-editor.org/rfc/rfc9111.html)
     #
     # https://gitlab.com/os85/httpx/wikis/Response-Cache
     #
@@ -15,9 +16,8 @@ module HTTPX
       private_constant :CACHEABLE_STATUS_CODES
 
       class << self
-        def load_dependencies(*)
-          require_relative "response_cache/store"
-          require_relative "response_cache/file_store"
+        def load_dependencies(klass)
+          klass.plugin(:cache)
         end
 
         # whether the +response+ can be stored in the response cache.
@@ -47,7 +47,6 @@ module HTTPX
         def extra_options(options)
           options.merge(
             supported_vary_headers: SUPPORTED_VARY_HEADERS,
-            response_cache_store: :store,
           )
         end
       end
@@ -56,58 +55,18 @@ module HTTPX
       #
       # :supported_vary_headers :: array of header values that will be considered for a "vary" header based cache validation
       #                            (defaults to {SUPPORTED_VARY_HEADERS}).
-      # :response_cache_store :: object where cached responses are fetch from or stored in; defaults to <tt>:store</tt> (in-memory
-      #                          cache), can be set to <tt>:file_store</tt> (file system cache store) as well, or any object which
-      #                          abides by the Cache Store Interface
-      #
-      # The Cache Store Interface requires implementation of the following methods:
-      #
-      # * +#get(request) -> response or nil+
-      # * +#set(request, response) -> void+
-      # * +#clear() -> void+)
       #
       module OptionsMethods
         private
 
-        def option_response_cache_store(value)
-          case value
-          when :store
-            Store.new
-          when :file_store
-            FileStore.new
-          else
-            value
-          end
-        end
-
         def option_supported_vary_headers(value)
           Array(value).sort
         end
       end
 
       module InstanceMethods
-        # wipes out all cached responses from the cache store.
-        def clear_response_cache
-          @options.response_cache_store.clear
-        end
-
-        def build_request(*)
-          request = super
-          return request unless cacheable_request?(request)
-
-          prepare_cache(request)
-
-          request
-        end
-
         private
 
-        def send_request(request, *)
-          return request if request.response
-
-          super
-        end
-
         def fetch_response(request, *)
           response = super
 
@@ -117,11 +76,7 @@ module HTTPX
             log { "returning cached response for #{request.uri}" }
 
             response.copy_from_cached!
-          elsif request.cacheable_verb? && ResponseCache.cacheable_response?(response)
-            unless response.cached?
-              log { "caching response for #{request.uri}..." }
-              request.options.response_cache_store.set(request, response)
-            end
+
           end
 
           response
@@ -130,21 +85,13 @@ module HTTPX
         # will either assign a still-fresh cached response to +request+, or set up its HTTP
         # cache invalidation headers in case it's not fresh anymore.
         def prepare_cache(request)
-          cached_response = request.options.response_cache_store.get(request)
+          super
 
-          return unless cached_response && match_by_vary?(request, cached_response)
+          return if request.response # already cached
 
-          cached_response.body.rewind
+          cached_response = retrieve_cached_response(request)
 
-          if cached_response.fresh?
-            cached_response = cached_response.dup
-            cached_response.mark_as_cached!
-            request.response = cached_response
-            request.emit_response(cached_response)
-            return
-          end
-
-          request.cached_response = cached_response
+          return unless cached_response && match_by_vary?(request, cached_response)
 
           if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
             request.headers.add("if-modified-since", last_modified)
@@ -156,20 +103,31 @@ module HTTPX
         end
 
         def cacheable_request?(request)
-          request.cacheable_verb? &&
+          (
+            request.cacheable_verb? &&
             (
               !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
             )
+          ) || super
         end
 
-        # whether the +response+ complies with the directives set by the +request+ "vary" header
+        def cacheable_response?(_, response)
+          ResponseCache.cacheable_response?(response) || super
+        end
+
+        # +cached_response+ is still valid if it's still fresh
+        def valid_cached_response?(_, cached_response)
+          cached_response.fresh?
+        end
+
+        # whether the +cached_response+ complies with the directives set by the +request+ "vary" header
         # (true when none is available).
-        def match_by_vary?(request, response)
-          vary = response.vary
+        def match_by_vary?(request, cached_response)
+          vary = cached_response.vary
 
           return true unless vary
 
-          original_request = response.original_request
+          original_request = cached_response.original_request
 
           if vary == %w[*]
             request.options.supported_vary_headers.each do |field|
@@ -186,19 +144,6 @@ module HTTPX
       end
 
       module RequestMethods
-        # points to a previously cached Response corresponding to this request.
-        attr_accessor :cached_response
-
-        def initialize(*)
-          super
-          @cached_response = nil
-        end
-
-        def merge_headers(*)
-          super
-          @response_cache_key = nil
-        end
-
         # returns whether this request is cacheable as per HTTP caching rules.
         def cacheable_verb?
           CACHEABLE_VERBS.include?(@verb)
@@ -220,29 +165,13 @@ module HTTPX
       end
 
       module ResponseMethods
-        attr_writer :original_request, :revalidated_at
+        attr_writer :revalidated_at
 
         def initialize(*)
           super
-          @cached = false
           @revalidated_at = nil
         end
 
-        # a copy of the request this response was originally cached from
-        def original_request
-          @original_request || @request
-        end
-
-        # whether this Response was duplicated from a previously {RequestMethods#cached_response}.
-        def cached?
-          @cached
-        end
-
-        # sets this Response as being duplicated from a previously cached response.
-        def mark_as_cached!
-          @cached = true
-        end
-
         # eager-copies the response headers and body from {RequestMethods#cached_response}.
         def copy_from_cached!
           cached_response = @request.cached_response
@@ -333,14 +262,6 @@ module HTTPX
           Time.now
         end
       end
-
-      module ResponseBodyMethods
-        def decode_chunk(chunk)
-          return chunk if @response.cached?
-
-          super
-        end
-      end
     end
     register_plugin :response_cache, ResponseCache
   end

data/lib/httpx/plugins/retries.rb

@@ -53,16 +53,18 @@ module HTTPX
           end
         end
 
-        # returns the time to wait before resending +request+ as per the polynomial backoff retry strategy.
+        # returns the time to wait before resending +request+ as per the polynomial backoff retry strategy,
+        # where base is 1 and exponent is 2.
         def retry_after_polynomial_backoff(request, _)
           offset = request.options.max_retries - request.retries
-          2 * (offset - 1)
+          1 * ((offset - 1)**2)
         end
 
-        # returns the time to wait before resending +request+ as per the exponential backoff retry strategy.
+        # returns the time to wait before resending +request+ as per the exponential backoff retry strategy,
+        # where base is 2
         def retry_after_exponential_backoff(request, _)
           offset = request.options.max_retries - request.retries
-          (offset - 1) * 2
+          2**(offset - 1)
         end
       end
 
@@ -186,7 +188,7 @@ module HTTPX
 
         # returns whether the +ex+ exception happend for a retriable request.
         def retryable_error?(ex, _)
-          RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
+          RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) } && !ex.is_a?(TotalRequestTimeoutError)
         end
 
         def proxy_error?(request, response, _)

data/lib/httpx/plugins/server_sent_events.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin implements convenience methods for Server Sent Events streams.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Server-Sent-Events
+    #
+    module ServerSentEvents
+      Message = if RUBY_VERSION >= "3.2.0" # rubocop:disable Naming/ConstantName
+        Data.define(:data, :event, :id, :retry_after) do
+          def initialize(event: nil, id: nil, retry_after: nil, **kwargs)
+            super
+          end
+        end
+      else
+        Struct.new(:data, :event, :id, :retry_after, keyword_init: true)
+      end
+
+      class << self
+        def subplugins
+          {
+            retries: ServerSentEventsRetries,
+          }
+        end
+
+        def load_dependencies(klass)
+          klass.plugin(:stream)
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :event_stream :: whether the request is a server-sent events text event stream (defaults to <tt>false</tt>).
+      module OptionsMethods
+        def option_event_stream(val)
+          val
+        end
+      end
+
+      module InstanceMethods
+        def request(*args, **options)
+          options[:stream] = true if options[:event_stream]
+
+          super
+        end
+
+        def build_request(*)
+          super.tap do |request|
+            if request.options.event_stream
+              request.headers["accept"] = "text/event-stream"
+              request.headers["cache-control"] = "no-cache"
+            end
+          end
+        end
+      end
+
+      module RequestMethods
+        attr_accessor :last_server_sent_message
+
+        def initialize(*)
+          super
+
+          @last_server_sent_message = nil
+        end
+      end
+
+      module StreamResponseMethods
+        # yields each event Message as the server emits them.
+        def each_message(&block)
+          return enum_for(__method__) unless block
+
+          payload = {}
+
+          each_line do |line|
+            if line.empty?
+              if payload[:comment]
+                payload.clear
+                next
+              end
+
+              next if payload.empty?
+
+              message = Message.new(**payload)
+
+              payload.clear
+
+              @request.last_server_sent_message = message
+
+              yield message
+            else
+              type, value = line.split(": ", 2)
+
+              case type
+              when "data"
+                type = type.to_sym
+                if payload.key?(type)
+                  payload[type] << "\n" << value
+                else
+                  payload[type] = value
+                end
+              when "id", "event", "retry"
+                type = type.to_sym
+                raise_format_error(line) if payload.key?(type) || value.empty?
+
+                type = :retry_after if type == :retry # avoid using keyword
+
+                payload[type] = value
+              else
+                # skip if it's a comment
+                if line.start_with?(":")
+                  payload[:comment] = true
+                  next
+                end
+
+                raise_format_error(line)
+              end
+            end
+          end
+        end
+
+        private
+
+        def raise_format_error(line)
+          raise Error, "'#{line}': invalid or unsupported event stream format"
+        end
+      end
+
+      module ServerSentEventsRetries
+        module InstanceMethods
+          private
+
+          def prepare_to_retry(request, *)
+            super
+
+            last_message = request.last_server_sent_message
+
+            return unless last_message && last_message.id
+
+            request.headers["last-event-id"] = last_message.id
+          ensure
+            request.last_server_sent_message = nil
+          end
+
+          def when_to_retry(request, *)
+            retry_after = request.last_server_sent_message&.retry_after
+
+            retry_after / 1_000.0 if retry_after # original in milliseconds
+
+            request.last_server_sent_message&.retry_after && super
+          end
+        end
+      end
+    end
+    register_plugin(:server_sent_events, ServerSentEvents)
+  end
+end

data/lib/httpx/plugins/ssrf_filter.rb

@@ -90,12 +90,22 @@ module HTTPX
       # adds support for the following options:
       #
       # :allowed_schemes :: list of URI schemes allowed (defaults to <tt>["https", "http"]</tt>)
+      # :extra_unsafe_ranges :: A list of IP ranges (or addresses) that will be filtered, in addition to the defaults
+      # :safe_private_ranges :: A list of IP ranges (or addresses) that will not be filtered, even if they'd be filtered by default
       module OptionsMethods
         private
 
         def option_allowed_schemes(value)
           Array(value)
         end
+
+        def option_extra_unsafe_ranges(value)
+          Array(value).map { |v| v.is_a?(IPAddr) ? v : IPAddr.new(v) }
+        end
+
+        def option_safe_private_ranges(value)
+          Array(value).map { |v| v.is_a?(IPAddr) ? v : IPAddr.new(v) }
+        end
       end
 
       module InstanceMethods
@@ -132,9 +142,14 @@ module HTTPX
         end
 
         def addresses=(addrs)
-          addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
+          addrs.reject! do |ipaddr|
+            ipaddr = ipaddr.address
+            next false if @options.safe_private_ranges&.any? { |r| r.include?(ipaddr) }
+
+            SsrfFilter.unsafe_ip_address?(ipaddr) || @options.extra_unsafe_ranges&.any? { |r| r.include?(ipaddr) }
+          end
 
-          raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?
+          raise ServerSideRequestForgeryError, "#{@origin.host} has no allowed IP addresses" if addrs.empty?
 
           super
         end