mongo 2.23.0 → 2.24.0

data/lib/mongo/retryable/backpressure.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Mongo
+  module Retryable
+    # Constants and helpers for client backpressure (exponential backoff
+    # and jitter in retry loops).
+    #
+    # @api private
+    module Backpressure
+      # Base backoff delay in seconds.
+      BASE_BACKOFF = 0.1
+
+      # Maximum backoff delay in seconds.
+      MAX_BACKOFF = 10
+
+      # Default maximum number of retries for overload errors.
+      DEFAULT_MAX_RETRIES = 2
+
+      # Calculate the backoff delay for a given retry attempt.
+      #
+      # @param [ Integer ] attempt The retry attempt number (1-indexed).
+      # @param [ Float ] jitter A random float in [0.0, 1.0). Defaults to
+      #   a random value. Can be injected for deterministic testing.
+      #
+      # @return [ Float ] The backoff delay in seconds.
+      def self.backoff_delay(attempt, jitter: rand)
+        jitter * [ MAX_BACKOFF, BASE_BACKOFF * (2**(attempt - 1)) ].min
+      end
+    end
+  end
+end

data/lib/mongo/retryable/base_worker.rb

@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# rubocop:todo all
 
 # Copyright (C) 2015-2023 MongoDB Inc.
 #
@@ -17,7 +16,6 @@
 
 module Mongo
   module Retryable
-
     # The abstract superclass for workers employed by Mongo::Retryable.
     #
     # @api private
@@ -25,12 +23,12 @@ module Mongo
       extend Forwardable
 
       def_delegators :retryable,
-        :client,
-        :cluster,
-        :select_server
+                     :client,
+                     :cluster,
+                     :select_server
 
       # @return [ Mongo::Retryable ] retryable A reference to the client object
-      #   that instatiated this worker.
+      #   that instantiated this worker.
       attr_reader :retryable
 
       # Constructs a new worker.
@@ -79,7 +77,6 @@ module Mongo
         ].freeze
       end
 
-
       # Tests to see if the given exception instance is of a type that can
       # be retried with modern retry mechanism.
       #
@@ -95,22 +92,51 @@ module Mongo
       def is_legacy_retryable_exception?(e)
         legacy_retryable_exceptions.any? { |klass| klass === e }
       end
+
       # Logs the given deprecation warning the first time it is called for a
       # given key; after that, it does nothing when given the same key.
       def deprecation_warning(key, warning)
         $_deprecation_warnings ||= {}
-        unless $_deprecation_warnings[key]
-          $_deprecation_warnings[key] = true
-          Logger.logger.warn(warning)
-        end
+        return if $_deprecation_warnings[key]
+
+        $_deprecation_warnings[key] = true
+        Logger.logger.warn(warning)
       end
 
       # Log a warning so that any application slow down is immediately obvious.
       def log_retry(e, options = nil)
-        message = (options || {}).fetch(:message, "Retry")
+        message = (options || {}).fetch(:message, 'Retry')
         Logger.logger.warn "#{message} due to: #{e.class.name}: #{e.message}"
       end
-    end
 
+      # Returns the retry policy from the client.
+      #
+      # @return [ Mongo::Retryable::RetryPolicy ] The retry policy.
+      def retry_policy
+        client.retry_policy
+      end
+
+      # Whether the error indicates server overload.
+      #
+      # @param [ Exception ] e The error to check.
+      #
+      # @return [ true | false ] true if the error has the
+      #   SystemOverloadedError label.
+      def overload_error?(e)
+        e.respond_to?(:label?) && e.label?('SystemOverloadedError')
+      end
+
+      # Whether the error is a retryable overload error. An error is
+      # retryable overload when it has both the SystemOverloadedError and
+      # RetryableError labels.
+      #
+      # @param [ Exception ] e The error to check.
+      #
+      # @return [ true | false ] true if the error has both labels.
+      def retryable_overload_error?(e)
+        overload_error?(e) &&
+          e.respond_to?(:label?) && e.label?('RetryableError')
+      end
+    end
   end
 end

data/lib/mongo/retryable/read_worker.rb

@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# rubocop:todo all
 
 # Copyright (C) 2015-2023 MongoDB Inc.
 #
@@ -19,7 +18,6 @@ require 'mongo/retryable/base_worker'
 
 module Mongo
   module Retryable
-
     # Implements the logic around retrying read operations.
     #
     # @api private
@@ -65,7 +63,7 @@ module Mongo
       # @param [ Proc ] block The block to execute.
       #
       # @return [ Cursor ] The cursor for the result set.
-      def read_with_retry_cursor(session, server_selector, view, context: nil, &block)
+      def read_with_retry_cursor(session, server_selector, view, context: nil)
         read_with_retry(session, server_selector, context) do |server|
           result = yield server
 
@@ -171,8 +169,8 @@ module Mongo
       # @return [ Result ] The result of the operation.
       def deprecated_legacy_read_with_retry(&block)
         deprecation_warning :read_with_retry,
-          'Legacy read_with_retry invocation - ' \
-          'please update the application and/or its dependencies'
+                            'Legacy read_with_retry invocation - ' \
+                            'please update the application and/or its dependencies'
 
         # Since we don't have a session, we cannot use the modern read retries.
         # And we need to select a server but we don't have a server selector.
@@ -205,9 +203,15 @@ module Mongo
         yield server
       rescue *retryable_exceptions, Error::OperationFailure::Family, Auth::Unauthorized, Error::PoolError => e
         e.add_notes('modern retry', 'attempt 1')
-        raise e if session.in_transaction?
-        raise e if !is_retryable_exception?(e) && !e.write_retryable?
-        retry_read(e, session, server_selector, context: context, failed_server: server, &block)
+        raise e if session.in_transaction? && !retryable_overload_error?(e)
+
+        if retryable_overload_error?(e)
+          overload_read_retry(e, session, server_selector, context, server, error_count: 1, &block)
+        else
+          raise e if !is_retryable_exception?(e) && !e.write_retryable?
+
+          retry_read(e, session, server_selector, context: context, failed_server: server, &block)
+        end
       end
 
       # Attempts to do a "legacy" read with retry. The operation will be
@@ -223,7 +227,7 @@ module Mongo
       # @param [ Proc ] block The block to execute.
       #
       # @return [ Result ] The result of the operation.
-      def legacy_read_with_retry(session, server_selector, context = nil, &block)
+      def legacy_read_with_retry(session, server_selector, context = nil)
         context&.check_timeout!
         attempt = attempt ? attempt + 1 : 1
         yield select_server(cluster, server_selector, session)
@@ -253,7 +257,7 @@ module Mongo
       # @param [ Proc ] block The block to execute.
       #
       # @return [ Result ] The result of the operation.
-      def read_without_retry(session, server_selector, &block)
+      def read_without_retry(session, server_selector)
         server = select_server(cluster, server_selector, session)
 
         begin
@@ -294,22 +298,25 @@ module Mongo
           raise
         rescue *retryable_exceptions => e
           e.add_notes('modern retry', "attempt #{attempt}")
-          if context&.csot?
-            failed_server = server
-            retry
-          else
-            raise e
+          if retryable_overload_error?(e)
+            return overload_read_retry(e, session, server_selector, context, server, error_count: attempt, &block)
           end
+
+          raise e unless context&.csot?
+
+          retry
         rescue Error::OperationFailure::Family, Error::PoolError => e
           e.add_note('modern retry')
+          if retryable_overload_error?(e)
+            e.add_note("attempt #{attempt}")
+            return overload_read_retry(e, session, server_selector, context, server, error_count: attempt, &block)
+          end
           if e.write_retryable?
             e.add_note("attempt #{attempt}")
-            if context&.csot?
-              failed_server = server
-              retry
-            else
-              raise e
-            end
+            raise e unless context&.csot?
+
+            retry
+
           else
             original_error.add_note("later retry failed: #{e.class}: #{e}")
             raise original_error
@@ -321,12 +328,61 @@ module Mongo
         end
       end
 
+      # Retry loop for overload errors with exponential backoff.
+      # Each retry sleeps with jittered backoff, respects MAX_RETRIES,
+      # and consumes a token from the bucket when adaptive retries
+      # are enabled.
+      #
+      # Per the client-backpressure spec, backoff is applied if and only
+      # if the error triggering the retry is an overload error. Non-overload
+      # retryable errors that occur within this loop are retried immediately
+      # (without backoff) but still count toward MAX_RETRIES.
+      def overload_read_retry(last_error, session, server_selector, context, failed_server, error_count:)
+        last_was_overload = true
+        loop do
+          delay = last_was_overload ? retry_policy.backoff_delay(error_count) : 0
+          raise last_error unless retry_policy.should_retry_overload?(error_count, delay, context: context)
+
+          log_retry(last_error, message: 'Read retry (overload backoff)')
+          sleep(delay) if last_was_overload
+
+          begin
+            server = select_server(
+              cluster, server_selector, session, failed_server,
+              error: last_error,
+              timeout: context&.remaining_timeout_sec
+            )
+          rescue Error, Error::AuthError => e
+            last_error.add_note("later retry failed: #{e.class}: #{e}")
+            raise last_error
+          end
+
+          begin
+            context&.check_timeout!
+            result = yield server, true
+            return result
+          rescue Error::TimeoutError
+            raise
+          rescue *retryable_exceptions, Error::OperationFailure::Family, Auth::Unauthorized, Error::PoolError => e
+            error_count += 1
+            e.add_notes('modern retry', "attempt #{error_count}")
+            is_overload = retryable_overload_error?(e)
+            raise e unless is_overload || is_retryable_exception?(e) || e.write_retryable?
+
+            last_was_overload = is_overload
+            failed_server = server
+            last_error = e
+          end
+        end
+      end
+
       def select_server_for_retry(original_error, session, server_selector, context, failed_server)
         select_server(
           cluster,
           server_selector,
           session,
           failed_server,
+          error: original_error,
           timeout: context&.remaining_timeout_sec
         )
       rescue Error, Error::AuthError => e

data/lib/mongo/retryable/retry_policy.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Mongo
+  module Retryable
+    # Encapsulates the retry policy for client backpressure with
+    # exponential backoff and jitter.
+    #
+    # One instance is created per Client and shared across all operations
+    # on that client.
+    #
+    # @api private
+    class RetryPolicy
+      # @return [ Integer ] The maximum number of overload retries.
+      attr_reader :max_retries
+
+      # Create a new retry policy.
+      #
+      # @param [ Integer ] max_retries The maximum number of overload
+      #   retry attempts. Defaults to Backpressure::DEFAULT_MAX_RETRIES.
+      def initialize(max_retries: Backpressure::DEFAULT_MAX_RETRIES)
+        @max_retries = max_retries
+      end
+
+      # Calculate the backoff delay for a given retry attempt.
+      #
+      # @param [ Integer ] attempt The retry attempt number (1-indexed).
+      # @param [ Float ] jitter A random float in [0.0, 1.0).
+      #
+      # @return [ Float ] The backoff delay in seconds.
+      def backoff_delay(attempt, jitter: rand)
+        Backpressure.backoff_delay(attempt, jitter: jitter)
+      end
+
+      # Determine whether an overload retry should be attempted.
+      #
+      # @param [ Integer ] attempt The retry attempt number (1-indexed).
+      # @param [ Float ] delay The backoff delay in seconds.
+      # @param [ Mongo::Operation::Context | nil ] context The operation
+      #   context (for CSOT deadline checking).
+      #
+      # @return [ true | false ] Whether the retry should proceed.
+      def should_retry_overload?(attempt, delay, context: nil)
+        return false if attempt > @max_retries
+        return false if exceeds_deadline?(delay, context)
+
+        true
+      end
+
+      private
+
+      def exceeds_deadline?(delay, context)
+        return false unless context&.csot?
+
+        deadline = context&.deadline
+        deadline&.nonzero? && Utils.monotonic_time + delay > deadline
+      end
+    end
+  end
+end