aws-sdk-core 2.11.502 → 3.94.0

data/lib/aws-sdk-core/plugins/regional_endpoint.rb

@@ -1,14 +1,6 @@
 module Aws
   module Plugins
-
-    # @seahorse.client.option [required, String] :region
-    #   The AWS region to connect to.  The region is used to construct
-    #   the client endpoint.  Defaults to `ENV['AWS_REGION']`.
-    #   Also checks `AMAZON_REGION` and `AWS_DEFAULT_REGION`.
-    #
-    # @seahorse.client.option [String] :endpoint A default endpoint is
-    #   constructed from the `:region`.
-    #
+    # @api private
     class RegionalEndpoint < Seahorse::Client::Plugin
 
       # raised when region is not configured
@@ -16,27 +8,47 @@ module Aws
 
       option(:profile)
 
-      option(:region) do |cfg|
+      option(:region,
+        required: true,
+        doc_type: String,
+        docstring: <<-DOCS) do |cfg|
+The AWS region to connect to.  The configured `:region` is
+used to determine the service `:endpoint`. When not passed,
+a default `:region` is search for in the following locations:
+
+* `Aws.config[:region]`
+* `ENV['AWS_REGION']`
+* `ENV['AMAZON_REGION']`
+* `ENV['AWS_DEFAULT_REGION']`
+* `~/.aws/credentials`
+* `~/.aws/config`
+        DOCS
         resolve_region(cfg)
       end
 
       option(:regional_endpoint, false)
 
-      option(:endpoint) do |cfg|
+      option(:endpoint, doc_type: String, docstring: <<-DOCS) do |cfg|
+The client endpoint is normally constructed from the `:region`
+option. You should only configure an `:endpoint` when connecting
+to test endpoints. This should be avalid HTTP(S) URI.
+        DOCS
         endpoint_prefix = cfg.api.metadata['endpointPrefix']
         if cfg.region && endpoint_prefix
           if cfg.respond_to?(:sts_regional_endpoints)
             sts_regional = cfg.sts_regional_endpoints
           end
-          EndpointProvider.resolve(cfg.region, endpoint_prefix, sts_regional)
+          Aws::Partitions::EndpointProvider.resolve(
+            cfg.region,
+            endpoint_prefix,
+            sts_regional
+          )
         end
       end
 
       def after_initialize(client)
         if client.config.region.nil? or client.config.region == ''
-          msg = "missing region; use :region option or "
-          msg << "export region name to ENV['AWS_REGION']"
-          raise Errors::MissingRegionError, msg
+          raise Errors::MissingRegionError
         end
       end
 

data/lib/aws-sdk-core/plugins/response_paging.rb

@@ -1,35 +1,22 @@
 module Aws
   module Plugins
+    # @api private
     class ResponsePaging < Seahorse::Client::Plugin
 
-      def add_handlers(handlers, config)
-        handlers.add(Handler,
-          operations: pageable_operations(config),
-          step: :initialize,
-          priority: 90)
-      end
-
-      private
-
-      def pageable_operations(config)
-        config.api.operations.inject([]) do |pageable, (name, operation)|
-          pageable << name if operation[:pager]
-          pageable
-        end
-      end
-
-      # @api private
       class Handler < Seahorse::Client::Handler
 
         def call(context)
           context[:original_params] = context.params
           resp = @handler.call(context)
           resp.extend(PageableResponse)
-          resp.pager = context.operation[:pager]
+          resp.pager = context.operation[:pager] || Aws::Pager::NullPager.new
           resp
         end
 
       end
+
+      handle(Handler, step: :initialize, priority: 90)
+
     end
   end
 end

data/lib/aws-sdk-core/plugins/retries/client_rate_limiter.rb

@@ -0,0 +1,137 @@
+module Aws
+  module Plugins
+    module Retries
+      # @api private
+      # Used only in 'adaptive' retry mode
+      class ClientRateLimiter
+        MIN_CAPACITY = 1
+        MIN_FILL_RATE = 0.5
+        SMOOTH = 0.8
+        # How much to scale back after a throttling response
+        BETA = 0.7
+        # Controls how aggressively we scale up after being throttled
+        SCALE_CONSTANT = 0.4
+
+        def initialize
+          @mutex                = Mutex.new
+          @fill_rate            = nil
+          @max_capacity         = nil
+          @current_capacity     = 0
+          @last_timestamp       = nil
+          @enabled              = false
+          @measured_tx_rate     = 0
+          @last_tx_rate_bucket  = Aws::Util.monotonic_seconds
+          @request_count        = 0
+          @last_max_rate        = 0
+          @last_throttle_time   = Aws::Util.monotonic_seconds
+          @calculated_rate      = nil
+        end
+
+        def token_bucket_acquire(amount, wait_to_fill = true)
+          # Client side throttling is not enabled until we see a
+          # throttling error
+          return unless @enabled
+
+          @mutex.synchronize do
+            token_bucket_refill
+
+            # Next see if we have enough capacity for the requested amount
+            while @current_capacity < amount
+              raise Aws::Errors::RetryCapacityNotAvailableError unless wait_to_fill
+              @mutex.sleep((amount - @current_capacity) / @fill_rate)
+              token_bucket_refill
+            end
+            @current_capacity -= amount
+          end
+        end
+
+        def update_sending_rate(is_throttling_error)
+          @mutex.synchronize do
+            update_measured_rate
+
+            if is_throttling_error
+              rate_to_use = if @enabled
+                              [@measured_tx_rate, @fill_rate].min
+                            else
+                              @measured_tx_rate
+                            end
+
+              # The fill_rate is from the token bucket
+              @last_max_rate = rate_to_use
+              calculate_time_window
+              @last_throttle_time = Aws::Util.monotonic_seconds
+              @calculated_rate = cubic_throttle(rate_to_use)
+              enable_token_bucket
+            else
+              calculate_time_window
+              @calculated_rate = cubic_success(Aws::Util.monotonic_seconds)
+            end
+
+            new_rate = [@calculated_rate, 2 * @measured_tx_rate].min
+            token_bucket_update_rate(new_rate)
+          end
+        end
+
+        private
+
+        def token_bucket_refill
+          timestamp = Aws::Util.monotonic_seconds
+          unless @last_timestamp
+            @last_timestamp = timestamp
+            return
+          end
+
+          fill_amount = (timestamp - @last_timestamp) * @fill_rate
+          @current_capacity = [
+            @max_capacity, @current_capacity + fill_amount
+          ].min
+
+          @last_timestamp = timestamp
+        end
+
+        def token_bucket_update_rate(new_rps)
+          # Refill based on our current rate before we update to the
+          # new fill rate
+          token_bucket_refill
+          @fill_rate = [new_rps, MIN_FILL_RATE].max
+          @max_capacity = [new_rps, MIN_CAPACITY].max
+          # When we scale down we can't have a current capacity that exceeds our
+          # max_capacity.
+          @current_capacity = [@current_capacity, @max_capacity].min
+        end
+
+        def enable_token_bucket
+          @enabled = true
+        end
+
+        def update_measured_rate
+          t = Aws::Util.monotonic_seconds
+          time_bucket = (t * 2).floor / 2.0
+          @request_count += 1
+          if time_bucket > @last_tx_rate_bucket
+            current_rate = @request_count / (time_bucket - @last_tx_rate_bucket)
+            @measured_tx_rate = (current_rate * SMOOTH) +
+              (@measured_tx_rate * (1 - SMOOTH))
+            @request_count = 0
+            @last_tx_rate_bucket = time_bucket
+          end
+        end
+
+        def calculate_time_window
+          # This is broken out into a separate calculation because it only
+          # gets updated when @last_max_rate changes so it can be cached.
+          @time_window = ((@last_max_rate * (1 - BETA)) / SCALE_CONSTANT)**(1.0 / 3)
+        end
+
+        def cubic_success(timestamp)
+          dt = timestamp - @last_throttle_time
+          (SCALE_CONSTANT * ((dt - @time_window)**3)) + @last_max_rate
+        end
+
+        def cubic_throttle(rate_to_use)
+          rate_to_use * BETA
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/retries/clock_skew.rb

@@ -0,0 +1,98 @@
+module Aws
+  module Plugins
+    module Retries
+
+      # @api private
+      class ClockSkew
+
+        CLOCK_SKEW_THRESHOLD = 5 * 60 # five minutes
+
+        def initialize
+          @mutex = Mutex.new
+          # clock_corrections are recorded only on errors
+          # and only when time difference is greater than the
+          # CLOCK_SKEW_THRESHOLD
+          @endpoint_clock_corrections = Hash.new(0)
+
+          # estimated_skew is calculated on every request
+          # and is used to estimate a TTL for requests
+          @endpoint_estimated_skews = Hash.new(nil)
+        end
+
+        # Gets the clock_correction in seconds to apply to a given endpoint
+        # @param endpoint [URI / String]
+        def clock_correction(endpoint)
+          @mutex.synchronize { @endpoint_clock_corrections[endpoint.to_s] }
+        end
+
+        # The estimated skew factors in any clock skew from
+        # the service along with any network latency.
+        # This provides a more accurate value for the ttl,
+        # which should represent when the client will stop
+        # waiting for a request.
+        # Estimated Skew should not be used to correct clock skew errors
+        # it should only be used to estimate TTL for a request
+        def estimated_skew(endpoint)
+          @mutex.synchronize { @endpoint_estimated_skews[endpoint.to_s] }
+        end
+
+        # Determines whether a request has clock skew by comparing
+        # the current time against the server's time in the response
+        # @param context [Seahorse::Client::RequestContext]
+        def clock_skewed?(context)
+          server_time = server_time(context.http_response)
+          !!server_time &&
+            (Time.now.utc - server_time).abs > CLOCK_SKEW_THRESHOLD
+        end
+
+        # Called only on clock skew related errors
+        # Update the stored clock skew correction value for an endpoint
+        # from the server's time in the response
+        # @param context [Seahorse::Client::RequestContext]
+        def update_clock_correction(context)
+          endpoint = context.http_request.endpoint
+          now_utc = Time.now.utc
+          server_time = server_time(context.http_response)
+          if server_time && (now_utc - server_time).abs > CLOCK_SKEW_THRESHOLD
+            set_clock_correction(endpoint, server_time - now_utc)
+          end
+        end
+
+        # Called for every request
+        # Update our estimated clock skew for the endpoint
+        # from the servers time in the response
+        # @param context [Seahorse::Client::RequestContext]
+        def update_estimated_skew(context)
+          endpoint = context.http_request.endpoint
+          now_utc = Time.now.utc
+          server_time = server_time(context.http_response)
+          return unless server_time
+          @mutex.synchronize do
+            @endpoint_estimated_skews[endpoint.to_s] = server_time - now_utc
+          end
+        end
+
+        private
+
+        # @param response [Seahorse::Client::Http::Response:]
+        def server_time(response)
+          begin
+            Time.parse(response.headers['date']).utc
+          rescue
+            nil
+          end
+        end
+
+        # Sets the clock correction for an endpoint
+        # @param endpoint [URI / String]
+        # @param correction [Number]
+        def set_clock_correction(endpoint, correction)
+          @mutex.synchronize do
+            @endpoint_clock_corrections[endpoint.to_s] = correction
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/aws-sdk-core/plugins/retries/error_inspector.rb

@@ -0,0 +1,142 @@
+module Aws
+  module Plugins
+    module Retries
+      # @api private
+      # This class will be obsolete when APIs contain modeled exceptions
+      class ErrorInspector
+        EXPIRED_CREDS = Set.new(
+          [
+            'InvalidClientTokenId',        # query services
+            'UnrecognizedClientException', # json services
+            'InvalidAccessKeyId',          # s3
+            'AuthFailure',                 # ec2
+            'InvalidIdentityToken',        # sts
+            'ExpiredToken'                 # route53
+          ]
+        )
+
+        THROTTLING_ERRORS = Set.new(
+          [
+            'Throttling',                             # query services
+            'ThrottlingException',                    # json services
+            'ThrottledException',                     # sns
+            'RequestThrottled',                       # sqs
+            'RequestThrottledException',              # generic service
+            'ProvisionedThroughputExceededException', # dynamodb
+            'TransactionInProgressException',         # dynamodb
+            'RequestLimitExceeded',                   # ec2
+            'BandwidthLimitExceeded',                 # cloud search
+            'LimitExceededException',                 # kinesis
+            'TooManyRequestsException',               # batch
+            'PriorRequestNotComplete',                # route53
+            'SlowDown',                               # s3
+            'EC2ThrottledException'                   # ec2
+          ]
+        )
+
+        CHECKSUM_ERRORS = Set.new(
+          [
+            'CRC32CheckFailed' # dynamodb
+          ]
+        )
+
+        NETWORKING_ERRORS = Set.new(
+          [
+            'RequestTimeout',          # s3
+            'RequestTimeoutException', # glacier
+            'IDPCommunicationError'    # sts
+          ]
+        )
+
+        # See: https://github.com/aws/aws-sdk-net/blob/5810dfe401e0eac2e59d02276d4b479224b4538e/sdk/src/Core/Amazon.Runtime/Pipeline/RetryHandler/RetryPolicy.cs#L78
+        CLOCK_SKEW_ERRORS = Set.new(
+          [
+            'RequestTimeTooSkewed',
+            'RequestExpired',
+            'InvalidSignatureException',
+            'SignatureDoesNotMatch',
+            'AuthFailure',
+            'RequestInTheFuture'
+          ]
+        )
+
+        def initialize(error, http_status_code)
+          @error = error
+          @name = extract_name(@error)
+          @http_status_code = http_status_code
+        end
+
+        def expired_credentials?
+          !!(EXPIRED_CREDS.include?(@name) || @name.match(/expired/i))
+        end
+
+        def throttling_error?
+          !!(THROTTLING_ERRORS.include?(@name) ||
+            @name.match(/throttl/i) ||
+            @http_status_code == 429) ||
+            modeled_throttling?
+        end
+
+        def checksum?
+          CHECKSUM_ERRORS.include?(@name) || @error.is_a?(Errors::ChecksumError)
+        end
+
+        def networking?
+          @error.is_a?(Seahorse::Client::NetworkingError) ||
+            @error.is_a?(Errors::NoSuchEndpointError) ||
+            NETWORKING_ERRORS.include?(@name)
+        end
+
+        def server?
+          (500..599).cover?(@http_status_code)
+        end
+
+        def endpoint_discovery?(context)
+          return false unless context.operation.endpoint_discovery
+
+          @http_status_code == 421 ||
+            @name == 'InvalidEndpointException' ||
+            @error.is_a?(Errors::EndpointDiscoveryError)
+        end
+
+        def modeled_retryable?
+          @error.is_a?(Errors::ServiceError) && @error.retryable?
+        end
+
+        def modeled_throttling?
+          @error.is_a?(Errors::ServiceError) && @error.throttling?
+        end
+
+        def clock_skew?(context)
+          CLOCK_SKEW_ERRORS.include?(@name) &&
+            context.config.clock_skew.clock_skewed?(context)
+        end
+
+        def retryable?(context)
+          server? ||
+            modeled_retryable? ||
+            throttling_error? ||
+            networking? ||
+            checksum? ||
+            endpoint_discovery?(context) ||
+            (expired_credentials? && refreshable_credentials?(context)) ||
+            clock_skew?(context)
+        end
+
+        private
+
+        def refreshable_credentials?(context)
+          context.config.credentials.respond_to?(:refresh!)
+        end
+
+        def extract_name(error)
+          if error.is_a?(Errors::ServiceError)
+            error.class.code || error.class.name.to_s
+          else
+            error.class.name.to_s
+          end
+        end
+      end
+    end
+  end
+end