aws-sdk-core 3.90.1 → 3.91.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b55b3e3cd1c9ec04bc1d375d08ca93de5a8c0212
-  data.tar.gz: 7e9f75a40eb67070b4fa0d5696193fb011196211
+  metadata.gz: '018176486b2589c3e5643f0517d600048b788239'
+  data.tar.gz: bf23afa4bc903911cc3a571b318eb38911a4cb62
 SHA512:
-  metadata.gz: c3dde38d099f43ae112d5b7887c38bca099897df24295c871581a9d888f7f9d95adf908caf9976cba6b195211b7d1ee9e1d7defd1b908c9bcc2cec4e6b2a8a11
-  data.tar.gz: 1c7d52fbbb564fd92a209d37c25e7f37a27206ec1461953b1f9aa0dee251379195af93494a9fb0a2d7e4d142ba4567fe69e334096b970fbacd9e36b2ca95e31d
+  metadata.gz: 8c5827b6dfca9a435419df7ec67f006179fe4656e909769e7545dbd4803b6723bdfedc7a9e89f23bff052d3e93428dcd39c9b803a1f35aba3ac26cfc3c33fb87
+  data.tar.gz: a244072123870020dbca0dee7bbe8f7057367b1ce81e18c7078b994e79a5f48c5ca5639e0df4113d4e68fdbc12627097751d21e45a65c4359e7d0abd4a548474

data/VERSION

@@ -1 +1 @@
-3.90.1
+3.91.0

data/lib/aws-sdk-core/credential_provider_chain.rb

@@ -28,7 +28,11 @@ module Aws
         [:assume_role_credentials, {}],
         [:shared_credentials, {}],
         [:process_credentials, {}],
-        [:instance_profile_credentials, {}]
+        [:instance_profile_credentials, {
+          retries: @config ? @config.instance_profile_credentials_retries : 0,
+          http_open_timeout: @config ? @config.instance_profile_credentials_timeout : 1,
+          http_read_timeout: @config ? @config.instance_profile_credentials_timeout : 1
+        }]
       ]
     end
 

data/lib/aws-sdk-core/errors.rb

@@ -36,6 +36,16 @@ module Aws
         attr_accessor :code
 
       end
+
+      # @api private undocumented
+      def retryable?
+        false
+      end
+
+      # @api private undocumented
+      def throttling?
+        false
+      end
     end
 
     # Raised when InstanceProfileCredentialsProvider or
@@ -255,6 +265,15 @@ Known AWS regions include (not specific to this service):
 
     end
 
+    # Raised when attempting to retry a request
+    # and no capacity is available to retry (See adaptive retry_mode)
+    class RetryCapacityNotAvailableError < RuntimeError
+      def initialize(*args)
+        msg = 'Insufficient client side capacity available to retry request.'
+        super(msg)
+      end
+    end
+
     # This module is mixed into another module, providing dynamic
     # error classes.  Error classes all inherit from {ServiceError}.
     #

data/lib/aws-sdk-core/plugins/client_metrics_plugin.rb

@@ -1,4 +1,5 @@
 require 'date'
+require_relative 'retries/error_inspector'
 
 module Aws
   module Plugins
@@ -141,7 +142,7 @@ all generated client side metrics. Defaults to an empty string.
             @handler.call(context)
           rescue StandardError => e
             # Handle SDK Exceptions
-            inspector = Aws::Plugins::RetryErrors::ErrorInspector.new(
+            inspector = Retries::ErrorInspector.new(
               e,
               context.http_response.status_code
             )

data/lib/aws-sdk-core/plugins/retries/client_rate_limiter.rb

@@ -0,0 +1,137 @@
+module Aws
+  module Plugins
+    module Retries
+      # @api private
+      # Used only in 'adaptive' retry mode
+      class ClientRateLimiter
+        MIN_CAPACITY = 1
+        MIN_FILL_RATE = 0.5
+        SMOOTH = 0.8
+        # How much to scale back after a throttling response
+        BETA = 0.7
+        # Controls how aggressively we scale up after being throttled
+        SCALE_CONSTANT = 0.4
+
+        def initialize
+          @mutex                = Mutex.new
+          @fill_rate            = nil
+          @max_capacity         = nil
+          @current_capacity     = 0
+          @last_timestamp       = nil
+          @enabled              = false
+          @measured_tx_rate     = 0
+          @last_tx_rate_bucket  = Aws::Util.monotonic_seconds
+          @request_count        = 0
+          @last_max_rate        = 0
+          @last_throttle_time   = Aws::Util.monotonic_seconds
+          @calculated_rate      = nil
+        end
+
+        def token_bucket_acquire(amount, wait_to_fill = true)
+          # Client side throttling is not enabled until we see a
+          # throttling error
+          return unless @enabled
+
+          @mutex.synchronize do
+            token_bucket_refill
+
+            # Next see if we have enough capacity for the requested amount
+            while @current_capacity < amount
+              raise Aws::Errors::RetryCapacityNotAvailableError unless wait_to_fill
+              @mutex.sleep((amount - @current_capacity) / @fill_rate)
+              token_bucket_refill
+            end
+            @current_capacity -= amount
+          end
+        end
+
+        def update_sending_rate(is_throttling_error)
+          @mutex.synchronize do
+            update_measured_rate
+
+            if is_throttling_error
+              rate_to_use = if @enabled
+                              [@measured_tx_rate, @fill_rate].min
+                            else
+                              @measured_tx_rate
+                            end
+
+              # The fill_rate is from the token bucket
+              @last_max_rate = rate_to_use
+              calculate_time_window
+              @last_throttle_time = Aws::Util.monotonic_seconds
+              @calculated_rate = cubic_throttle(rate_to_use)
+              enable_token_bucket
+            else
+              calculate_time_window
+              @calculated_rate = cubic_success(Aws::Util.monotonic_seconds)
+            end
+
+            new_rate = [@calculated_rate, 2 * @measured_tx_rate].min
+            token_bucket_update_rate(new_rate)
+          end
+        end
+
+        private
+
+        def token_bucket_refill
+          timestamp = Aws::Util.monotonic_seconds
+          unless @last_timestamp
+            @last_timestamp = timestamp
+            return
+          end
+
+          fill_amount = (timestamp - @last_timestamp) * @fill_rate
+          @current_capacity = [
+            @max_capacity, @current_capacity + fill_amount
+          ].min
+
+          @last_timestamp = timestamp
+        end
+
+        def token_bucket_update_rate(new_rps)
+          # Refill based on our current rate before we update to the
+          # new fill rate
+          token_bucket_refill
+          @fill_rate = [new_rps, MIN_FILL_RATE].max
+          @max_capacity = [new_rps, MIN_CAPACITY].max
+          # When we scale down we can't have a current capacity that exceeds our
+          # max_capacity.
+          @current_capacity = [@current_capacity, @max_capacity].min
+        end
+
+        def enable_token_bucket
+          @enabled = true
+        end
+
+        def update_measured_rate
+          t = Aws::Util.monotonic_seconds
+          time_bucket = (t * 2).floor / 2.0
+          @request_count += 1
+          if time_bucket > @last_tx_rate_bucket
+            current_rate = @request_count / (time_bucket - @last_tx_rate_bucket)
+            @measured_tx_rate = (current_rate * SMOOTH) +
+              (@measured_tx_rate * (1 - SMOOTH))
+            @request_count = 0
+            @last_tx_rate_bucket = time_bucket
+          end
+        end
+
+        def calculate_time_window
+          # This is broken out into a separate calculation because it only
+          # gets updated when @last_max_rate changes so it can be cached.
+          @time_window = ((@last_max_rate * (1 - BETA)) / SCALE_CONSTANT)**(1.0 / 3)
+        end
+
+        def cubic_success(timestamp)
+          dt = timestamp - @last_throttle_time
+          (SCALE_CONSTANT * ((dt - @time_window)**3)) + @last_max_rate
+        end
+
+        def cubic_throttle(rate_to_use)
+          rate_to_use * BETA
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/retries/clock_skew.rb

@@ -0,0 +1,63 @@
+module Aws
+  module Plugins
+    module Retries
+
+      # @api private
+      class ClockSkew
+
+        CLOCK_SKEW_THRESHOLD = 5 * 60 # five minutes
+
+        def initialize
+          @mutex = Mutex.new
+          @endpoint_clock_corrections = Hash.new(0)
+        end
+
+        # Gets the clock_correction in seconds to apply to a given endpoint
+        # @param endpoint [URI / String]
+        def clock_correction(endpoint)
+          @mutex.synchronize { @endpoint_clock_corrections[endpoint.to_s] }
+        end
+
+        # Sets the clock correction for an endpoint
+        # @param endpoint [URI / String]
+        # @param correction [Number]
+        def set_clock_correction(endpoint, correction)
+          @mutex.synchronize { @endpoint_clock_corrections[endpoint.to_s] = correction }
+        end
+
+        # Determines whether a request has clock skew by comparing
+        # the current time against the server's time in the response
+        # @param context [Seahorse::Client::RequestContext]
+        def clock_skewed?(context)
+          server_time = server_time(context.http_response)
+          !!server_time && (Time.now.utc - server_time).abs > CLOCK_SKEW_THRESHOLD
+        end
+
+        # Update the stored clock skew value for an endpoint
+        # from the server's time in the response
+        # @param context [Seahorse::Client::RequestContext]
+        def update_clock_skew(context)
+          endpoint = context.http_request.endpoint
+          now_utc = Time.now.utc
+          server_time = server_time(context.http_response)
+          if server_time && (now_utc - server_time).abs > CLOCK_SKEW_THRESHOLD
+            set_clock_correction(endpoint, server_time - now_utc)
+          end
+        end
+
+        private
+
+        # @param response [Seahorse::Client::Http::Response:]
+        def server_time(response)
+          begin
+            Time.parse(response.headers['date']).utc
+          rescue
+            nil
+          end
+        end
+
+      end
+    end
+  end
+end
+

data/lib/aws-sdk-core/plugins/retries/error_inspector.rb

@@ -0,0 +1,142 @@
+module Aws
+  module Plugins
+    module Retries
+      # @api private
+      # This class will be obsolete when APIs contain modeled exceptions
+      class ErrorInspector
+        EXPIRED_CREDS = Set.new(
+          [
+            'InvalidClientTokenId',        # query services
+            'UnrecognizedClientException', # json services
+            'InvalidAccessKeyId',          # s3
+            'AuthFailure',                 # ec2
+            'InvalidIdentityToken',        # sts
+            'ExpiredToken'                 # route53
+          ]
+        )
+
+        THROTTLING_ERRORS = Set.new(
+          [
+            'Throttling',                             # query services
+            'ThrottlingException',                    # json services
+            'ThrottledException',                     # sns
+            'RequestThrottled',                       # sqs
+            'RequestThrottledException',              # generic service
+            'ProvisionedThroughputExceededException', # dynamodb
+            'TransactionInProgressException',         # dynamodb
+            'RequestLimitExceeded',                   # ec2
+            'BandwidthLimitExceeded',                 # cloud search
+            'LimitExceededException',                 # kinesis
+            'TooManyRequestsException',               # batch
+            'PriorRequestNotComplete',                # route53
+            'SlowDown',                               # s3
+            'EC2ThrottledException'                   # ec2
+          ]
+        )
+
+        CHECKSUM_ERRORS = Set.new(
+          [
+            'CRC32CheckFailed' # dynamodb
+          ]
+        )
+
+        NETWORKING_ERRORS = Set.new(
+          [
+            'RequestTimeout',          # s3
+            'RequestTimeoutException', # glacier
+            'IDPCommunicationError'    # sts
+          ]
+        )
+
+        # See: https://github.com/aws/aws-sdk-net/blob/5810dfe401e0eac2e59d02276d4b479224b4538e/sdk/src/Core/Amazon.Runtime/Pipeline/RetryHandler/RetryPolicy.cs#L78
+        CLOCK_SKEW_ERRORS = Set.new(
+          [
+            'RequestTimeTooSkewed',
+            'RequestExpired',
+            'InvalidSignatureException',
+            'SignatureDoesNotMatch',
+            'AuthFailure',
+            'RequestInTheFuture'
+          ]
+        )
+
+        def initialize(error, http_status_code)
+          @error = error
+          @name = extract_name(@error)
+          @http_status_code = http_status_code
+        end
+
+        def expired_credentials?
+          !!(EXPIRED_CREDS.include?(@name) || @name.match(/expired/i))
+        end
+
+        def throttling_error?
+          !!(THROTTLING_ERRORS.include?(@name) ||
+            @name.match(/throttl/i) ||
+            @http_status_code == 429) ||
+            modeled_throttling?
+        end
+
+        def checksum?
+          CHECKSUM_ERRORS.include?(@name) || @error.is_a?(Errors::ChecksumError)
+        end
+
+        def networking?
+          @error.is_a?(Seahorse::Client::NetworkingError) ||
+            @error.is_a?(Errors::NoSuchEndpointError) ||
+            NETWORKING_ERRORS.include?(@name)
+        end
+
+        def server?
+          (500..599).cover?(@http_status_code)
+        end
+
+        def endpoint_discovery?(context)
+          return false unless context.operation.endpoint_discovery
+
+          @http_status_code == 421 ||
+            @name == 'InvalidEndpointException' ||
+            @error.is_a?(Errors::EndpointDiscoveryError)
+        end
+
+        def modeled_retryable?
+          @error.is_a?(Errors::ServiceError) && @error.retryable?
+        end
+
+        def modeled_throttling?
+          @error.is_a?(Errors::ServiceError) && @error.throttling?
+        end
+
+        def clock_skew?(context)
+          CLOCK_SKEW_ERRORS.include?(@name) &&
+            context.config.clock_skew.clock_skewed?(context)
+        end
+
+        def retryable?(context)
+          server? ||
+            modeled_retryable? ||
+            throttling_error? ||
+            networking? ||
+            checksum? ||
+            endpoint_discovery?(context) ||
+            (expired_credentials? && refreshable_credentials?(context)) ||
+            clock_skew?(context)
+        end
+
+        private
+
+        def refreshable_credentials?(context)
+          context.config.credentials.respond_to?(:refresh!)
+        end
+
+        def extract_name(error)
+          if error.is_a?(Errors::ServiceError)
+            error.class.code || error.class.name.to_s
+          else
+            error.class.name.to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/retries/retry_quota.rb

@@ -0,0 +1,57 @@
+module Aws
+  module Plugins
+    module Retries
+
+      # @api private
+      # Used in 'standard' and 'adaptive' retry modes.
+      class RetryQuota
+        INITIAL_RETRY_TOKENS = 500
+        RETRY_COST = 5
+        NO_RETRY_INCREMENT = 1
+        TIMEOUT_RETRY_COST = 10
+
+        def initialize(opts = {})
+          @mutex              = Mutex.new
+          @max_capacity       = opts.fetch(:max_capacity, INITIAL_RETRY_TOKENS)
+          @available_capacity = @max_capacity
+        end
+
+        # check if there is sufficient capacity to retry
+        # and return it.  If there is insufficient capacity
+        # return 0
+        # @return [Integer] The amount of capacity checked out
+        def checkout_capacity(error_inspector)
+          @mutex.synchronize do
+            capacity_amount = if error_inspector.networking?
+                                TIMEOUT_RETRY_COST
+                              else
+                                RETRY_COST
+                              end
+
+            # unable to acquire capacity
+            return 0 if capacity_amount > @available_capacity
+
+            @available_capacity -= capacity_amount
+            capacity_amount
+          end
+        end
+
+        # capacity_amount refers to the amount of capacity requested from
+        # the last retry.  It can either be RETRY_COST, TIMEOUT_RETRY_COST,
+        # or unset.
+        def release(capacity_amount)
+          # Implementation note:  The release() method is called for
+          # every API call.  In the common case where the request is
+          # successful and we're at full capacity, we can avoid locking.
+          # We can't exceed max capacity so there's no work we have to do.
+          return if @available_capacity == @max_capacity
+
+          @mutex.synchronize do
+            @available_capacity += capacity_amount || NO_RETRY_INCREMENT
+            @available_capacity = [@available_capacity, @max_capacity].min
+          end
+        end
+      end
+    end
+  end
+end