aws-sdk-core 3.54.2 → 3.126.0

data/lib/aws-sdk-core/plugins/protocols/rest_json.rb

@@ -1,12 +1,29 @@
+# frozen_string_literal: true
+
 module Aws
   module Plugins
     module Protocols
       class RestJson < Seahorse::Client::Plugin
 
+        class ContentTypeHandler < Seahorse::Client::Handler
+          def call(context)
+            body = context.http_request.body
+            # Rest::Handler will set a default JSON body, so size can be checked
+            # if this handler is run after serialization.
+            if !body.respond_to?(:size) ||
+               (body.respond_to?(:size) && body.size > 0)
+              context.http_request.headers['Content-Type'] ||=
+                'application/json'
+            end
+            @handler.call(context)
+          end
+        end
+
         handler(Rest::Handler)
+        handler(ContentTypeHandler, priority: 30)
         handler(Json::ErrorHandler, step: :sign)
-
       end
+
     end
   end
 end

data/lib/aws-sdk-core/plugins/protocols/rest_xml.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module Plugins
     module Protocols

data/lib/aws-sdk-core/plugins/recursion_detection.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Aws
+  module Plugins
+    # @api private
+    class RecursionDetection < Seahorse::Client::Plugin
+
+      # @api private
+      class Handler < Seahorse::Client::Handler
+        def call(context)
+
+          unless context.http_request.headers.key?('x-amz-trace-id')
+            if ENV['AWS_LAMBDA_FUNCTION_NAME'] &&
+              (trace_id = ENV['_X_AMZ_TRACE_ID'])
+              context.http_request.headers['x-amz-trace-id'] = trace_id
+            end
+          end
+          @handler.call(context)
+        end
+      end
+
+      # should be at the end of build so that
+      # modeled traits / service customizations apply first
+      handler(Handler, step: :build, order: 99)
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/regional_endpoint.rb

@@ -1,11 +1,9 @@
+# frozen_string_literal: true
+
 module Aws
   module Plugins
     # @api private
     class RegionalEndpoint < Seahorse::Client::Plugin
-
-      # raised when region is not configured
-      MISSING_REGION = 'missing required configuration option :region'
-
       option(:profile)
 
       option(:region,
@@ -14,7 +12,7 @@ module Aws
         docstring: <<-DOCS) do |cfg|
 The AWS region to connect to.  The configured `:region` is
 used to determine the service `:endpoint`. When not passed,
-a default `:region` is search for in the following locations:
+a default `:region` is searched for in the following locations:
 
 * `Aws.config[:region]`
 * `ENV['AWS_REGION']`
@@ -26,35 +24,95 @@ a default `:region` is search for in the following locations:
         resolve_region(cfg)
       end
 
+      option(:use_dualstack_endpoint,
+        doc_type: 'Boolean',
+        docstring: <<-DOCS) do |cfg|
+When set to `true`, dualstack enabled endpoints (with `.aws` TLD)
+will be used if available.
+        DOCS
+        resolve_use_dualstack_endpoint(cfg)
+      end
+
+      option(:use_fips_endpoint,
+        doc_type: 'Boolean',
+        docstring: <<-DOCS) do |cfg|
+When set to `true`, fips compatible endpoints will be used if available.
+When a `fips` region is used, the region is normalized and this config
+is set to `true`.
+        DOCS
+        resolve_use_fips_endpoint(cfg)
+      end
+
       option(:regional_endpoint, false)
 
       option(:endpoint, doc_type: String, docstring: <<-DOCS) do |cfg|
 The client endpoint is normally constructed from the `:region`
 option. You should only configure an `:endpoint` when connecting
-to test endpoints. This should be avalid HTTP(S) URI.
+to test or custom endpoints. This should be a valid HTTP(S) URI.
         DOCS
         endpoint_prefix = cfg.api.metadata['endpointPrefix']
         if cfg.region && endpoint_prefix
-          Aws::Partitions::EndpointProvider.resolve(cfg.region, endpoint_prefix)
+          if cfg.respond_to?(:sts_regional_endpoints)
+            sts_regional = cfg.sts_regional_endpoints
+          end
+
+          # check region is a valid RFC host label
+          unless Seahorse::Util.host_label?(cfg.region)
+            raise Errors::InvalidRegionError
+          end
+
+          region = cfg.region
+          new_region = region.gsub('fips-', '').gsub('-fips', '')
+          if region != new_region
+            warn("Legacy region #{region} was transformed to #{new_region}."\
+                 '`use_fips_endpoint` config was set to true.')
+            cfg.override_config(:use_fips_endpoint, true)
+            cfg.override_config(:region, new_region)
+          end
+
+          Aws::Partitions::EndpointProvider.resolve(
+            cfg.region,
+            endpoint_prefix,
+            sts_regional,
+            {
+              dualstack: cfg.use_dualstack_endpoint,
+              fips: cfg.use_fips_endpoint
+            }
+          )
         end
       end
 
       def after_initialize(client)
-        if client.config.region.nil? or client.config.region == ''
+        if client.config.region.nil? || client.config.region == ''
           raise Errors::MissingRegionError
         end
       end
 
-      private
+      class << self
+        private
 
-      def self.resolve_region(cfg)
-        keys = %w(AWS_REGION AMAZON_REGION AWS_DEFAULT_REGION)
-        env_region = ENV.values_at(*keys).compact.first
-        env_region = nil if env_region == ''
-        cfg_region = Aws.shared_config.region(profile: cfg.profile)
-        env_region || cfg_region
-      end
+        def resolve_region(cfg)
+          keys = %w[AWS_REGION AMAZON_REGION AWS_DEFAULT_REGION]
+          env_region = ENV.values_at(*keys).compact.first
+          env_region = nil if env_region == ''
+          cfg_region = Aws.shared_config.region(profile: cfg.profile)
+          env_region || cfg_region
+        end
+
+        def resolve_use_dualstack_endpoint(cfg)
+          value = ENV['AWS_USE_DUALSTACK_ENDPOINT']
+          value ||= Aws.shared_config.use_dualstack_endpoint(
+            profile: cfg.profile
+          )
+          Aws::Util.str_2_bool(value) || false
+        end
 
+        def resolve_use_fips_endpoint(cfg)
+          value = ENV['AWS_USE_FIPS_ENDPOINT']
+          value ||= Aws.shared_config.use_fips_endpoint(profile: cfg.profile)
+          Aws::Util.str_2_bool(value) || false
+        end
+      end
     end
   end
 end

data/lib/aws-sdk-core/plugins/response_paging.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module Plugins
     # @api private

data/lib/aws-sdk-core/plugins/retries/client_rate_limiter.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module Aws
+  module Plugins
+    module Retries
+      # @api private
+      # Used only in 'adaptive' retry mode
+      class ClientRateLimiter
+        MIN_CAPACITY = 1
+        MIN_FILL_RATE = 0.5
+        SMOOTH = 0.8
+        # How much to scale back after a throttling response
+        BETA = 0.7
+        # Controls how aggressively we scale up after being throttled
+        SCALE_CONSTANT = 0.4
+
+        def initialize
+          @mutex                = Mutex.new
+          @fill_rate            = nil
+          @max_capacity         = nil
+          @current_capacity     = 0
+          @last_timestamp       = nil
+          @enabled              = false
+          @measured_tx_rate     = 0
+          @last_tx_rate_bucket  = Aws::Util.monotonic_seconds
+          @request_count        = 0
+          @last_max_rate        = 0
+          @last_throttle_time   = Aws::Util.monotonic_seconds
+          @calculated_rate      = nil
+        end
+
+        def token_bucket_acquire(amount, wait_to_fill = true)
+          # Client side throttling is not enabled until we see a
+          # throttling error
+          return unless @enabled
+
+          @mutex.synchronize do
+            token_bucket_refill
+
+            # Next see if we have enough capacity for the requested amount
+            while @current_capacity < amount
+              raise Aws::Errors::RetryCapacityNotAvailableError unless wait_to_fill
+              @mutex.sleep((amount - @current_capacity) / @fill_rate)
+              token_bucket_refill
+            end
+            @current_capacity -= amount
+          end
+        end
+
+        def update_sending_rate(is_throttling_error)
+          @mutex.synchronize do
+            update_measured_rate
+
+            if is_throttling_error
+              rate_to_use = if @enabled
+                              [@measured_tx_rate, @fill_rate].min
+                            else
+                              @measured_tx_rate
+                            end
+
+              # The fill_rate is from the token bucket
+              @last_max_rate = rate_to_use
+              calculate_time_window
+              @last_throttle_time = Aws::Util.monotonic_seconds
+              @calculated_rate = cubic_throttle(rate_to_use)
+              enable_token_bucket
+            else
+              calculate_time_window
+              @calculated_rate = cubic_success(Aws::Util.monotonic_seconds)
+            end
+
+            new_rate = [@calculated_rate, 2 * @measured_tx_rate].min
+            token_bucket_update_rate(new_rate)
+          end
+        end
+
+        private
+
+        def token_bucket_refill
+          timestamp = Aws::Util.monotonic_seconds
+          unless @last_timestamp
+            @last_timestamp = timestamp
+            return
+          end
+
+          fill_amount = (timestamp - @last_timestamp) * @fill_rate
+          @current_capacity = [
+            @max_capacity, @current_capacity + fill_amount
+          ].min
+
+          @last_timestamp = timestamp
+        end
+
+        def token_bucket_update_rate(new_rps)
+          # Refill based on our current rate before we update to the
+          # new fill rate
+          token_bucket_refill
+          @fill_rate = [new_rps, MIN_FILL_RATE].max
+          @max_capacity = [new_rps, MIN_CAPACITY].max
+          # When we scale down we can't have a current capacity that exceeds our
+          # max_capacity.
+          @current_capacity = [@current_capacity, @max_capacity].min
+        end
+
+        def enable_token_bucket
+          @enabled = true
+        end
+
+        def update_measured_rate
+          t = Aws::Util.monotonic_seconds
+          time_bucket = (t * 2).floor / 2.0
+          @request_count += 1
+          if time_bucket > @last_tx_rate_bucket
+            current_rate = @request_count / (time_bucket - @last_tx_rate_bucket)
+            @measured_tx_rate = (current_rate * SMOOTH) +
+              (@measured_tx_rate * (1 - SMOOTH))
+            @request_count = 0
+            @last_tx_rate_bucket = time_bucket
+          end
+        end
+
+        def calculate_time_window
+          # This is broken out into a separate calculation because it only
+          # gets updated when @last_max_rate changes so it can be cached.
+          @time_window = ((@last_max_rate * (1 - BETA)) / SCALE_CONSTANT)**(1.0 / 3)
+        end
+
+        def cubic_success(timestamp)
+          dt = timestamp - @last_throttle_time
+          (SCALE_CONSTANT * ((dt - @time_window)**3)) + @last_max_rate
+        end
+
+        def cubic_throttle(rate_to_use)
+          rate_to_use * BETA
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/retries/clock_skew.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Aws
+  module Plugins
+    module Retries
+
+      # @api private
+      class ClockSkew
+
+        CLOCK_SKEW_THRESHOLD = 5 * 60 # five minutes
+
+        def initialize
+          @mutex = Mutex.new
+          # clock_corrections are recorded only on errors
+          # and only when time difference is greater than the
+          # CLOCK_SKEW_THRESHOLD
+          @endpoint_clock_corrections = Hash.new(0)
+
+          # estimated_skew is calculated on every request
+          # and is used to estimate a TTL for requests
+          @endpoint_estimated_skews = Hash.new(nil)
+        end
+
+        # Gets the clock_correction in seconds to apply to a given endpoint
+        # @param endpoint [URI / String]
+        def clock_correction(endpoint)
+          @mutex.synchronize { @endpoint_clock_corrections[endpoint.to_s] }
+        end
+
+        # The estimated skew factors in any clock skew from
+        # the service along with any network latency.
+        # This provides a more accurate value for the ttl,
+        # which should represent when the client will stop
+        # waiting for a request.
+        # Estimated Skew should not be used to correct clock skew errors
+        # it should only be used to estimate TTL for a request
+        def estimated_skew(endpoint)
+          @mutex.synchronize { @endpoint_estimated_skews[endpoint.to_s] }
+        end
+
+        # Determines whether a request has clock skew by comparing
+        # the current time against the server's time in the response
+        # @param context [Seahorse::Client::RequestContext]
+        def clock_skewed?(context)
+          server_time = server_time(context.http_response)
+          !!server_time &&
+            (Time.now.utc - server_time).abs > CLOCK_SKEW_THRESHOLD
+        end
+
+        # Called only on clock skew related errors
+        # Update the stored clock skew correction value for an endpoint
+        # from the server's time in the response
+        # @param context [Seahorse::Client::RequestContext]
+        def update_clock_correction(context)
+          endpoint = context.http_request.endpoint
+          now_utc = Time.now.utc
+          server_time = server_time(context.http_response)
+          if server_time && (now_utc - server_time).abs > CLOCK_SKEW_THRESHOLD
+            set_clock_correction(endpoint, server_time - now_utc)
+          end
+        end
+
+        # Called for every request
+        # Update our estimated clock skew for the endpoint
+        # from the servers time in the response
+        # @param context [Seahorse::Client::RequestContext]
+        def update_estimated_skew(context)
+          endpoint = context.http_request.endpoint
+          now_utc = Time.now.utc
+          server_time = server_time(context.http_response)
+          return unless server_time
+          @mutex.synchronize do
+            @endpoint_estimated_skews[endpoint.to_s] = server_time - now_utc
+          end
+        end
+
+        private
+
+        # @param response [Seahorse::Client::Http::Response:]
+        def server_time(response)
+          begin
+            Time.parse(response.headers['date']).utc
+          rescue
+            nil
+          end
+        end
+
+        # Sets the clock correction for an endpoint
+        # @param endpoint [URI / String]
+        # @param correction [Number]
+        def set_clock_correction(endpoint, correction)
+          @mutex.synchronize do
+            @endpoint_clock_corrections[endpoint.to_s] = correction
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/aws-sdk-core/plugins/retries/error_inspector.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Aws
+  module Plugins
+    module Retries
+      # @api private
+      # This class will be obsolete when APIs contain modeled exceptions
+      class ErrorInspector
+        EXPIRED_CREDS = Set.new(
+          [
+            'InvalidClientTokenId',        # query services
+            'UnrecognizedClientException', # json services
+            'InvalidAccessKeyId',          # s3
+            'AuthFailure',                 # ec2
+            'InvalidIdentityToken',        # sts
+            'ExpiredToken',                # route53
+            'ExpiredTokenException'        # kinesis
+          ]
+        )
+
+        THROTTLING_ERRORS = Set.new(
+          [
+            'Throttling',                             # query services
+            'ThrottlingException',                    # json services
+            'ThrottledException',                     # sns
+            'RequestThrottled',                       # sqs
+            'RequestThrottledException',              # generic service
+            'ProvisionedThroughputExceededException', # dynamodb
+            'TransactionInProgressException',         # dynamodb
+            'RequestLimitExceeded',                   # ec2
+            'BandwidthLimitExceeded',                 # cloud search
+            'LimitExceededException',                 # kinesis
+            'TooManyRequestsException',               # batch
+            'PriorRequestNotComplete',                # route53
+            'SlowDown',                               # s3
+            'EC2ThrottledException'                   # ec2
+          ]
+        )
+
+        CHECKSUM_ERRORS = Set.new(
+          [
+            'CRC32CheckFailed' # dynamodb
+          ]
+        )
+
+        NETWORKING_ERRORS = Set.new(
+          [
+            'RequestTimeout',          # s3
+            'InternalError',           # s3
+            'RequestTimeoutException', # glacier
+            'IDPCommunicationError'    # sts
+          ]
+        )
+
+        # See: https://github.com/aws/aws-sdk-net/blob/5810dfe401e0eac2e59d02276d4b479224b4538e/sdk/src/Core/Amazon.Runtime/Pipeline/RetryHandler/RetryPolicy.cs#L78
+        CLOCK_SKEW_ERRORS = Set.new(
+          [
+            'RequestTimeTooSkewed',
+            'RequestExpired',
+            'InvalidSignatureException',
+            'SignatureDoesNotMatch',
+            'AuthFailure',
+            'RequestInTheFuture'
+          ]
+        )
+
+        def initialize(error, http_status_code)
+          @error = error
+          @name = extract_name(@error)
+          @http_status_code = http_status_code
+        end
+
+        def expired_credentials?
+          !!(EXPIRED_CREDS.include?(@name) || @name.match(/expired/i))
+        end
+
+        def throttling_error?
+          !!(THROTTLING_ERRORS.include?(@name) ||
+            @name.match(/throttl/i) ||
+            @http_status_code == 429) ||
+            modeled_throttling?
+        end
+
+        def checksum?
+          CHECKSUM_ERRORS.include?(@name) || @error.is_a?(Errors::ChecksumError)
+        end
+
+        def networking?
+          @error.is_a?(Seahorse::Client::NetworkingError) ||
+            @error.is_a?(Errors::NoSuchEndpointError) ||
+            NETWORKING_ERRORS.include?(@name)
+        end
+
+        def server?
+          (500..599).cover?(@http_status_code)
+        end
+
+        def endpoint_discovery?(context)
+          return false unless context.operation.endpoint_discovery
+
+          @http_status_code == 421 ||
+            @name == 'InvalidEndpointException' ||
+            @error.is_a?(Errors::EndpointDiscoveryError)
+        end
+
+        def modeled_retryable?
+          @error.is_a?(Errors::ServiceError) && @error.retryable?
+        end
+
+        def modeled_throttling?
+          @error.is_a?(Errors::ServiceError) && @error.throttling?
+        end
+
+        def clock_skew?(context)
+          CLOCK_SKEW_ERRORS.include?(@name) &&
+            context.config.clock_skew.clock_skewed?(context)
+        end
+
+        def retryable?(context)
+          server? ||
+            modeled_retryable? ||
+            throttling_error? ||
+            networking? ||
+            checksum? ||
+            endpoint_discovery?(context) ||
+            (expired_credentials? && refreshable_credentials?(context)) ||
+            clock_skew?(context)
+        end
+
+        private
+
+        def refreshable_credentials?(context)
+          context.config.credentials.respond_to?(:refresh!)
+        end
+
+        def extract_name(error)
+          if error.is_a?(Errors::ServiceError)
+            error.class.code || error.class.name.to_s
+          else
+            error.class.name.to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/retries/retry_quota.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Aws
+  module Plugins
+    module Retries
+
+      # @api private
+      # Used in 'standard' and 'adaptive' retry modes.
+      class RetryQuota
+        INITIAL_RETRY_TOKENS = 500
+        RETRY_COST = 5
+        NO_RETRY_INCREMENT = 1
+        TIMEOUT_RETRY_COST = 10
+
+        def initialize(opts = {})
+          @mutex              = Mutex.new
+          @max_capacity       = opts.fetch(:max_capacity, INITIAL_RETRY_TOKENS)
+          @available_capacity = @max_capacity
+        end
+
+        # check if there is sufficient capacity to retry
+        # and return it.  If there is insufficient capacity
+        # return 0
+        # @return [Integer] The amount of capacity checked out
+        def checkout_capacity(error_inspector)
+          @mutex.synchronize do
+            capacity_amount = if error_inspector.networking?
+                                TIMEOUT_RETRY_COST
+                              else
+                                RETRY_COST
+                              end
+
+            # unable to acquire capacity
+            return 0 if capacity_amount > @available_capacity
+
+            @available_capacity -= capacity_amount
+            capacity_amount
+          end
+        end
+
+        # capacity_amount refers to the amount of capacity requested from
+        # the last retry.  It can either be RETRY_COST, TIMEOUT_RETRY_COST,
+        # or unset.
+        def release(capacity_amount)
+          # Implementation note:  The release() method is called for
+          # every API call.  In the common case where the request is
+          # successful and we're at full capacity, we can avoid locking.
+          # We can't exceed max capacity so there's no work we have to do.
+          return if @available_capacity == @max_capacity
+
+          @mutex.synchronize do
+            @available_capacity += capacity_amount || NO_RETRY_INCREMENT
+            @available_capacity = [@available_capacity, @max_capacity].min
+          end
+        end
+      end
+    end
+  end
+end