aws-sdk-core 3.90.1 → 3.91.0

data/lib/aws-sdk-core/plugins/retry_errors.rb

@@ -1,13 +1,17 @@
 require 'set'
+require_relative 'retries/error_inspector'
+require_relative 'retries/retry_quota'
+require_relative 'retries/client_rate_limiter'
+require_relative 'retries/clock_skew'
 
 module Aws
   module Plugins
     # @api private
     class RetryErrors < Seahorse::Client::Plugin
-
-      EQUAL_JITTER = lambda { |delay| (delay / 2) + Kernel.rand(0..(delay/2))}
-      FULL_JITTER = lambda { |delay| Kernel.rand(0..delay) }
-      NO_JITTER = lambda { |delay| delay }
+      # BEGIN LEGACY OPTIONS
+      EQUAL_JITTER = ->(delay) { (delay / 2) + Kernel.rand(0..(delay / 2)) }
+      FULL_JITTER = ->(delay) { Kernel.rand(0..delay) }
+      NO_JITTER = ->(delay) { delay }
 
       JITTERS = {
         none: NO_JITTER,
@@ -15,168 +19,308 @@ module Aws
         full: FULL_JITTER
       }
 
-      JITTERS.default_proc = lambda { |h,k|
-        raise KeyError, "#{k} is not a named jitter function. Must be one of #{h.keys}"
+      JITTERS.default_proc = lambda { |h, k|
+        raise KeyError,
+              "#{k} is not a named jitter function. Must be one of #{h.keys}"
       }
 
       DEFAULT_BACKOFF = lambda do |c|
-        delay = 2 ** c.retries * c.config.retry_base_delay
-        delay = [delay, c.config.retry_max_delay].min if (c.config.retry_max_delay || 0) > 0
+        delay = 2**c.retries * c.config.retry_base_delay
+        if (c.config.retry_max_delay || 0) > 0
+          delay = [delay, c.config.retry_max_delay].min
+        end
         jitter = c.config.retry_jitter
-        jitter = JITTERS[jitter] if Symbol === jitter
+        jitter = JITTERS[jitter] if jitter.is_a?(Symbol)
         delay = jitter.call(delay) if jitter
         Kernel.sleep(delay)
       end
 
-      option(:retry_limit,
+      option(
+        :retry_limit,
         default: 3,
         doc_type: Integer,
         docstring: <<-DOCS)
 The maximum number of times to retry failed requests.  Only
 ~ 500 level server errors and certain ~ 400 level client errors
 are retried.  Generally, these are throttling errors, data
-checksum errors, networking errors, timeout errors and auth
-errors from expired credentials.
+checksum errors, networking errors, timeout errors, auth errors,
+endpoint discovery, and errors from expired credentials.
+This option is only used in the `legacy` retry mode.
         DOCS
 
-      option(:retry_max_delay,
+      option(
+        :retry_max_delay,
         default: 0,
         doc_type: Integer,
         docstring: <<-DOCS)
-The maximum number of seconds to delay between retries (0 for no limit) used by the default backoff function.
+The maximum number of seconds to delay between retries (0 for no limit)
+used by the default backoff function. This option is only used in the
+`legacy` retry mode.
         DOCS
 
-      option(:retry_base_delay,
+      option(
+        :retry_base_delay,
         default: 0.3,
         doc_type: Float,
         docstring: <<-DOCS)
-The base delay in seconds used by the default backoff function.
+The base delay in seconds used by the default backoff function. This option
+is only used in the `legacy` retry mode.
         DOCS
 
-      option(:retry_jitter,
+      option(
+        :retry_jitter,
         default: :none,
         doc_type: Symbol,
         docstring: <<-DOCS)
-A delay randomiser function used by the default backoff function. Some predefined functions can be referenced by name - :none, :equal, :full, otherwise a Proc that takes and returns a number.
+A delay randomiser function used by the default backoff function.
+Some predefined functions can be referenced by name - :none, :equal, :full,
+otherwise a Proc that takes and returns a number. This option is only used
+in the `legacy` retry mode.
 
 @see https://www.awsarchitectureblog.com/2015/03/backoff.html
         DOCS
 
-      option(:retry_backoff, DEFAULT_BACKOFF)
+      option(
+        :retry_backoff,
+        default: DEFAULT_BACKOFF,
+        doc_type: Proc,
+        docstring: <<-DOCS)
+A proc or lambda used for backoff. Defaults to 2**retries * retry_base_delay.
+This option is only used in the `legacy` retry mode.
+        DOCS
 
-      # @api private
-      class ErrorInspector
+      # END LEGACY OPTIONS
+
+      option(
+        :retry_mode,
+        default: 'legacy',
+        doc_type: String,
+        docstring: <<-DOCS) do |cfg|
+Specifies which retry algorithm to use. Values are:
+  * `legacy` - The pre-existing retry behavior.  This is default value if
+    no retry mode is provided.
+  * `standard` - A standardized set of retry rules across the AWS SDKs.
+    This includes support for retry quotas, which limit the number of
+    unsuccessful retries a client can make.
+  * `adaptive` - An experimental retry mode that includes all the
+    functionality of `standard` mode along with automatic client side
+    throttling.  This is a provisional mode that may change behavior
+    in the future.
+        DOCS
+        resolve_retry_mode(cfg)
+      end
 
-        EXPIRED_CREDS = Set.new([
-          'InvalidClientTokenId',        # query services
-          'UnrecognizedClientException', # json services
-          'InvalidAccessKeyId',          # s3
-          'AuthFailure',                 # ec2
-          'InvalidIdentityToken',        # sts
-          'ExpiredToken',                # route53
-        ])
+      option(
+        :max_attempts,
+        default: 3,
+        doc_type: Integer,
+        docstring: <<-DOCS) do |cfg|
+An integer representing the maximum number attempts that will be made for
+a single request, including the initial attempt.  For example,
+setting this value to 5 will result in a request being retried up to
+4 times. Used in `standard` and `adaptive` retry modes.
+        DOCS
+        resolve_max_attempts(cfg)
+      end
 
-        THROTTLING_ERRORS = Set.new([
-          'Throttling',                             # query services
-          'ThrottlingException',                    # json services
-          'RequestThrottled',                       # sqs
-          'RequestThrottledException',
-          'ProvisionedThroughputExceededException', # dynamodb
-          'TransactionInProgressException',         # dynamodb
-          'RequestLimitExceeded',                   # ec2
-          'BandwidthLimitExceeded',                 # cloud search
-          'LimitExceededException',                 # kinesis
-          'TooManyRequestsException',               # batch
-          'PriorRequestNotComplete',                # route53
-        ])
+      option(
+        :adaptive_retry_wait_to_fill,
+        default: true,
+        doc_type: 'Boolean',
+        docstring: <<-DOCS) do |cfg|
+Used only in `adaptive` retry mode.  When true, the request will sleep
+until there is sufficent client side capacity to retry the request.
+When false, the request will raise a `RetryCapacityNotAvailableError` and will
+not retry instead of sleeping.
+        DOCS
+        resolve_adaptive_retry_wait_to_fill(cfg)
+      end
 
-        CHECKSUM_ERRORS = Set.new([
-          'CRC32CheckFailed', # dynamodb
-        ])
+      option(
+        :correct_clock_skew,
+        default: true,
+        doc_type: 'Boolean',
+        docstring: <<-DOCS) do |cfg|
+Used only in `standard` and adaptive retry modes. Specifies whether to apply
+a clock skew correction and retry requests with skewed client clocks.
+      DOCS
+        resolve_correct_clock_skew(cfg)
+      end
 
-        NETWORKING_ERRORS = Set.new([
-          'RequestTimeout',         # s3
-          'IDPCommunicationError',  # sts
-        ])
+      # @api private undocumented
+      option(:client_rate_limiter) { Retries::ClientRateLimiter.new }
 
-        def initialize(error, http_status_code)
-          @error = error
-          @name = extract_name(error)
-          @http_status_code = http_status_code
-        end
+      # @api private undocumented
+      option(:retry_quota) { Retries::RetryQuota.new }
 
-        def expired_credentials?
-          !!(EXPIRED_CREDS.include?(@name) || @name.match(/expired/i))
-        end
+      # @api private undocumented
+      option(:clock_skew) { Retries::ClockSkew.new }
 
-        def throttling_error?
-          !!(THROTTLING_ERRORS.include?(@name) || @name.match(/throttl/i) || @http_status_code == 429)
+      def self.resolve_retry_mode(cfg)
+        value = ENV['AWS_RETRY_MODE'] ||
+                Aws.shared_config.retry_mode(profile: cfg.profile) ||
+                'legacy'
+        # Raise if provided value is not one of the retry modes
+        if value != 'legacy' && value != 'standard' && value != 'adaptive'
+          raise ArgumentError,
+                'Must provide either `legacy`, `standard`, or `adaptive` for '\
+                'retry_mode profile option or for ENV[\'AWS_RETRY_MODE\']'
         end
+        value
+      end
 
-        def checksum?
-          CHECKSUM_ERRORS.include?(@name) || @error.is_a?(Errors::ChecksumError)
+      def self.resolve_max_attempts(cfg)
+        value = ENV['AWS_MAX_ATTEMPTS'] ||
+                Aws.shared_config.max_attempts(profile: cfg.profile) ||
+                3
+        # Raise if provided value is not a positive integer
+        if !value.is_a?(Integer) || value <= 0
+          raise ArgumentError,
+                'Must provide a positive integer for max_attempts profile '\
+                'option or for ENV[\'AWS_MAX_ATTEMPTS\']'
         end
+        value
+      end
 
-        def networking?
-          @error.is_a?(Seahorse::Client::NetworkingError) ||
-          @error.is_a?(Errors::NoSuchEndpointError) ||
-          NETWORKING_ERRORS.include?(@name)
+      def self.resolve_adaptive_retry_wait_to_fill(cfg)
+        value = ENV['AWS_ADAPTIVE_RETRY_WAIT_TO_FILL'] ||
+          Aws.shared_config.adaptive_retry_wait_to_fill(profile: cfg.profile) ||
+          'true'
+
+        # Raise if provided value is not true or false
+        if value != 'true' && value != 'false'
+          raise ArgumentError,
+                'Must provide either `true` or `false` for '\
+                'adaptive_retry_wait_to_fill profile option or for '\
+                'ENV[\'AWS_ADAPTIVE_RETRY_WAIT_TO_FILL\']'
         end
 
-        def server?
-          (500..599).include?(@http_status_code)
+        value == 'true'
+      end
+
+      def self.resolve_correct_clock_skew(cfg)
+        value = ENV['AWS_CORRECT_CLOCK_SKEW'] ||
+          Aws.shared_config.correct_clock_skew(profile: cfg.profile) ||
+          'true'
+
+        # Raise if provided value is not true or false
+        if value != 'true' && value != 'false'
+          raise ArgumentError,
+                'Must provide either `true` or `false` for '\
+                'correct_clock_skew profile option or for '\
+                'ENV[\'AWS_CORRECT_CLOCK_SKEW\']'
         end
 
-        def endpoint_discovery?(context)
-          return false unless context.operation.endpoint_discovery
+        value == 'true'
+      end
+
+      class Handler < Seahorse::Client::Handler
+        # Max backoff (in seconds)
+        MAX_BACKOFF = 20
+
+        def call(context)
+          context.metadata[:retries] ||= {}
+          config = context.config
 
-          if @http_status_code == 421 ||
-            extract_name(@error) == 'InvalidEndpointException'
-            @error = Errors::EndpointDiscoveryError.new
+          get_send_token(config)
+          response = @handler.call(context)
+          error_inspector = Retries::ErrorInspector.new(
+            response.error, response.context.http_response.status_code
+          )
+
+          request_bookkeeping(context, response, error_inspector)
+
+          if error_inspector.endpoint_discovery?(context)
+            key = config.endpoint_cache.extract_key(context)
+            config.endpoint_cache.delete(key)
           end
 
-          # When endpoint discovery error occurs
-          # evict the endpoint from cache
-          if @error.is_a?(Errors::EndpointDiscoveryError)
-            key = context.config.endpoint_cache.extract_key(context)
-            context.config.endpoint_cache.delete(key)
-            true
-          else
-            false
+          # Clock skew needs to be updated from the response even when
+          # the request is not retryable
+          if error_inspector.clock_skew?(context)
+            config.clock_skew.update_clock_skew(context)
           end
-        end
 
-        def retryable?(context)
-          (expired_credentials? and refreshable_credentials?(context)) or
-            throttling_error? or
-            checksum? or
-            networking? or
-            server? or
-            endpoint_discovery?(context)
+          return response unless retryable?(context, response, error_inspector)
+
+          return response if context.retries >= config.max_attempts - 1
+
+          context.metadata[:retries][:capacity_amount] =
+            config.retry_quota.checkout_capacity(error_inspector)
+          return response unless context.metadata[:retries][:capacity_amount] > 0
+
+          delay = exponential_backoff(context.retries)
+          Kernel.sleep(delay)
+          retry_request(context, error_inspector)
         end
 
         private
 
-        def refreshable_credentials?(context)
-          context.config.credentials.respond_to?(:refresh!)
+        def get_send_token(config)
+          # either fail fast or block until a token becomes available
+          # must be configurable
+          # need a maximum rate at which we can send requests (max_send_rate)
+          # is unset until a throttle is seen
+          if config.retry_mode == 'adaptive'
+            config.client_rate_limiter.token_bucket_acquire(
+              1,
+              config.adaptive_retry_wait_to_fill
+            )
+          end
         end
 
-        def extract_name(error)
-          if error.is_a?(Errors::ServiceError)
-            error.class.code
-          else
-            error.class.name.to_s
+        # maxsendrate is updated if on adaptive mode and based on response
+        # retry quota is updated if the request is successful (both modes)
+        def request_bookkeeping(context, response, error_inspector)
+          config = context.config
+          if response.successful?
+            config.retry_quota.release(
+              context.metadata[:retries][:capacity_amount]
+            )
+          end
+
+          if config.retry_mode == 'adaptive'
+            is_throttling_error = error_inspector.throttling_error?
+            config.client_rate_limiter.update_sending_rate(is_throttling_error)
           end
         end
 
+        def retryable?(context, response, error_inspector)
+          return false if response.successful?
+
+          error_inspector.retryable?(context) &&
+            context.http_response.body.respond_to?(:truncate)
+        end
+
+        def exponential_backoff(retries)
+          # for a transient error, use backoff
+          [Kernel.rand * 2**retries, MAX_BACKOFF].min
+        end
+
+        def retry_request(context, error)
+          context.retries += 1
+          context.config.credentials.refresh! if error.expired_credentials?
+          context.http_request.body.rewind
+          context.http_response.reset
+          call(context)
+        end
       end
 
-      class Handler < Seahorse::Client::Handler
+      class LegacyHandler < Seahorse::Client::Handler
 
         def call(context)
           response = @handler.call(context)
           if response.error
-            retry_if_possible(response)
+            error_inspector = Retries::ErrorInspector.new(
+              response.error, response.context.http_response.status_code
+            )
+
+            if error_inspector.endpoint_discovery?(context)
+              key = context.config.endpoint_cache.extract_key(context)
+              context.config.endpoint_cache.delete(key)
+            end
+
+            retry_if_possible(response, error_inspector)
           else
             response
           end
@@ -184,21 +328,15 @@ A delay randomiser function used by the default backoff function. Some predefine
 
         private
 
-        def retry_if_possible(response)
+        def retry_if_possible(response, error_inspector)
           context = response.context
-          error = error_for(response)
-          if should_retry?(context, error)
-            retry_request(context, error)
+          if should_retry?(context, error_inspector)
+            retry_request(context, error_inspector)
           else
             response
           end
         end
 
-        def error_for(response)
-          status_code = response.context.http_response.status_code
-          ErrorInspector.new(response.error, status_code)
-        end
-
         def retry_request(context, error)
           delay_retry(context)
           context.retries += 1
@@ -213,9 +351,9 @@ A delay randomiser function used by the default backoff function. Some predefine
         end
 
         def should_retry?(context, error)
-          error.retryable?(context) and
-          context.retries < retry_limit(context) and
-          response_truncatable?(context)
+          error.retryable?(context) &&
+            context.retries < retry_limit(context) &&
+            response_truncatable?(context)
         end
 
         def retry_limit(context)
@@ -225,15 +363,17 @@ A delay randomiser function used by the default backoff function. Some predefine
         def response_truncatable?(context)
           context.http_response.body.respond_to?(:truncate)
         end
-
       end
 
       def add_handlers(handlers, config)
-        if config.retry_limit > 0
+        if config.retry_mode == 'legacy'
+          if config.retry_limit > 0
+            handlers.add(LegacyHandler, step: :sign, priority: 99)
+          end
+        else
           handlers.add(Handler, step: :sign, priority: 99)
         end
       end
-
     end
   end
 end