aws-sdk-core 3.173.0 → 3.254.1

data/lib/aws-sdk-core/plugins/regional_endpoint.rb

@@ -20,16 +20,31 @@ a default `:region` is searched for in the following locations:
 * `ENV['AWS_DEFAULT_REGION']`
 * `~/.aws/credentials`
 * `~/.aws/config`
-        DOCS
+             DOCS
         resolve_region(cfg)
       end
 
+      option(:sigv4a_signing_region_set,
+             doc_type: Array,
+             rbs_type: 'Array[String]',
+             docstring: <<-DOCS) do |cfg|
+A list of regions that should be signed with SigV4a signing. When
+not passed, a default `:sigv4a_signing_region_set` is searched for
+in the following locations:
+
+* `Aws.config[:sigv4a_signing_region_set]`
+* `ENV['AWS_SIGV4A_SIGNING_REGION_SET']`
+* `~/.aws/config`
+             DOCS
+        resolve_sigv4a_signing_region_set(cfg)
+      end
+
       option(:use_dualstack_endpoint,
         doc_type: 'Boolean',
         docstring: <<-DOCS) do |cfg|
 When set to `true`, dualstack enabled endpoints (with `.aws` TLD)
 will be used if available.
-        DOCS
+             DOCS
         resolve_use_dualstack_endpoint(cfg)
       end
 
@@ -39,7 +54,7 @@ will be used if available.
 When set to `true`, fips compatible endpoints will be used if available.
 When a `fips` region is used, the region is normalized and this config
 is set to `true`.
-        DOCS
+             DOCS
         resolve_use_fips_endpoint(cfg)
       end
 
@@ -47,50 +62,71 @@ is set to `true`.
       # Legacy endpoints must continue to be generated at client time.
       option(:regional_endpoint, false)
 
-      # NOTE: All of the defaults block code is effectively deprecated.
-      # Because old services can depend on this new core version, we must
-      # retain it.
+      option(:ignore_configured_endpoint_urls,
+        doc_type: 'Boolean',
+        docstring: <<-DOCS) do |cfg|
+Setting to true disables use of endpoint URLs provided via environment 
+variables and the shared configuration file.
+             DOCS
+        resolve_ignore_configured_endpoint_urls(cfg)
+      end
+
       option(:endpoint, doc_type: String, docstring: <<-DOCS) do |cfg|
 The client endpoint is normally constructed from the `:region`
 option. You should only configure an `:endpoint` when connecting
 to test or custom endpoints. This should be a valid HTTP(S) URI.
-        DOCS
-        endpoint_prefix = cfg.api.metadata['endpointPrefix']
-        if cfg.region && endpoint_prefix
-          if cfg.respond_to?(:sts_regional_endpoints)
-            sts_regional = cfg.sts_regional_endpoints
-          end
+      DOCS
+        resolve_endpoint(cfg)
+      end
 
-          # check region is a valid RFC host label
-          unless Seahorse::Util.host_label?(cfg.region)
-            raise Errors::InvalidRegionError
-          end
+      def after_initialize(client)
+        region = client.config.region
+        raise Errors::MissingRegionError if region.nil? || region == ''
 
-          region = cfg.region
-          new_region = region.gsub('fips-', '').gsub('-fips', '')
-          if region != new_region
-            warn("Legacy region #{region} was transformed to #{new_region}."\
-                 '`use_fips_endpoint` config was set to true.')
-            cfg.override_config(:use_fips_endpoint, true)
-            cfg.override_config(:region, new_region)
-          end
+        # resolve a default endpoint to preserve legacy behavior
+        initialize_default_endpoint(client) if client.config.endpoint.nil?
 
-          Aws::Partitions::EndpointProvider.resolve(
-            cfg.region,
-            endpoint_prefix,
-            sts_regional,
-            {
-              dualstack: cfg.use_dualstack_endpoint,
-              fips: cfg.use_fips_endpoint
-            }
-          )
-        end
+        region_set = client.config.sigv4a_signing_region_set
+        return if region_set.nil?
+        raise Errors::InvalidRegionSetError unless region_set.is_a?(Array)
+
+        region_set = region_set.compact.reject(&:empty?)
+        raise Errors::InvalidRegionSetError if region_set.empty?
+
+        client.config.sigv4a_signing_region_set = region_set
       end
 
-      def after_initialize(client)
-        if client.config.region.nil? || client.config.region == ''
-          raise Errors::MissingRegionError
+      private
+
+      def initialize_default_endpoint(client)
+        client_module = Object.const_get(client.class.name.rpartition('::').first)
+        param_class = client_module.const_get(:EndpointParameters)
+        endpoint_provider = client.config.endpoint_provider
+        params = param_class.create(client.config)
+        endpoint = endpoint_provider.resolve_endpoint(params)
+        client.config.endpoint = endpoint.url
+      rescue ArgumentError, NameError
+        # fallback to legacy
+        client.config.endpoint = resolve_legacy_endpoint(client.config)
+      end
+
+      # set a default endpoint in config using legacy (endpoints.json) resolver
+      def resolve_legacy_endpoint(cfg)
+        endpoint_prefix = cfg.api.metadata['endpointPrefix']
+        if cfg.respond_to?(:sts_regional_endpoints)
+          sts_regional = cfg.sts_regional_endpoints
         end
+
+        endpoint = Aws::Partitions::EndpointProvider.resolve(
+          cfg.region,
+          endpoint_prefix,
+          sts_regional,
+          {
+            dualstack: cfg.use_dualstack_endpoint,
+            fips: cfg.use_fips_endpoint
+          }
+        )
+        URI(endpoint)
       end
 
       class << self
@@ -104,6 +140,12 @@ to test or custom endpoints. This should be a valid HTTP(S) URI.
           env_region || cfg_region
         end
 
+        def resolve_sigv4a_signing_region_set(cfg)
+          value = ENV['AWS_SIGV4A_SIGNING_REGION_SET']
+          value ||= Aws.shared_config.sigv4a_signing_region_set(profile: cfg.profile)
+          value.split(',') if value
+        end
+
         def resolve_use_dualstack_endpoint(cfg)
           value = ENV['AWS_USE_DUALSTACK_ENDPOINT']
           value ||= Aws.shared_config.use_dualstack_endpoint(
@@ -117,6 +159,89 @@ to test or custom endpoints. This should be a valid HTTP(S) URI.
           value ||= Aws.shared_config.use_fips_endpoint(profile: cfg.profile)
           Aws::Util.str_2_bool(value) || false
         end
+
+        def resolve_ignore_configured_endpoint_urls(cfg)
+          value = ENV['AWS_IGNORE_CONFIGURED_ENDPOINT_URLS']
+          value ||= Aws.shared_config.ignore_configured_endpoint_urls(profile: cfg.profile)
+          Aws::Util.str_2_bool(value&.downcase) || false
+        end
+
+        # NOTE: with Endpoints 2.0, some of this logic is deprecated
+        # but because new old service gems may depend on new core versions
+        # we must preserve that behavior.
+        # Additional behavior controls the setting of the custom SDK::Endpoint
+        # parameter.
+        # When the `regional_endpoint` config is set to true - this indicates to
+        # Endpoints2.0 that a custom endpoint has NOT been configured by the user.
+        def resolve_endpoint(cfg)
+          endpoint = resolve_custom_config_endpoint(cfg)
+          endpoint_prefix = cfg.api.metadata['endpointPrefix']
+
+          return endpoint unless endpoint.nil? && cfg.region && endpoint_prefix
+
+          validate_region!(cfg.region)
+          handle_legacy_pseudo_regions(cfg)
+
+          # set regional_endpoint flag - this indicates to Endpoints 2.0
+          # that a custom endpoint has NOT been configured by the user
+          cfg.override_config(:regional_endpoint, true)
+
+          # a default endpoint is resolved in after_initialize
+          nil
+        end
+
+        # get a custom configured endpoint from ENV or configuration
+        def resolve_custom_config_endpoint(cfg)
+          return if cfg.ignore_configured_endpoint_urls
+
+
+          env_service_endpoint(cfg) || env_global_endpoint(cfg) || shared_config_endpoint(cfg)
+        end
+
+        def env_service_endpoint(cfg)
+          service_id = cfg.api.metadata['serviceId'] || cfg.api.metadata['endpointPrefix']
+          env_service_id = service_id.gsub(" ", "_").upcase
+          return unless endpoint = ENV["AWS_ENDPOINT_URL_#{env_service_id}"]
+
+          cfg.logger&.debug(
+            "Endpoint configured from ENV['AWS_ENDPOINT_URL_#{env_service_id}']: #{endpoint}\n")
+          endpoint
+        end
+
+        def env_global_endpoint(cfg)
+          return unless endpoint = ENV['AWS_ENDPOINT_URL']
+
+          cfg.logger&.debug(
+            "Endpoint configured from ENV['AWS_ENDPOINT_URL']: #{endpoint}\n")
+          endpoint
+        end
+
+        def shared_config_endpoint(cfg)
+          service_id = cfg.api.metadata['serviceId'] || cfg.api.metadata['endpointPrefix']
+          return unless endpoint = Aws.shared_config.configured_endpoint(profile: cfg.profile, service_id: service_id)
+
+          cfg.logger&.debug(
+            "Endpoint configured from shared config(profile: #{cfg.profile}): #{endpoint}\n")
+          endpoint
+        end
+
+        # check region is a valid RFC host label
+        def validate_region!(region)
+          unless Seahorse::Util.host_label?(region)
+            raise Errors::InvalidRegionError
+          end
+        end
+
+        def handle_legacy_pseudo_regions(cfg)
+          region = cfg.region
+          new_region = region.gsub('fips-', '').gsub('-fips', '')
+          if region != new_region
+            warn("Legacy region #{region} was transformed to #{new_region}."\
+                 '`use_fips_endpoint` config was set to true.')
+            cfg.override_config(:use_fips_endpoint, true)
+            cfg.override_config(:region, new_region)
+          end
+        end
       end
     end
   end

data/lib/aws-sdk-core/plugins/request_compression.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+module Aws
+  module Plugins
+    # @api private
+    class RequestCompression < Seahorse::Client::Plugin
+      DEFAULT_MIN_COMPRESSION_SIZE = 10_240
+      MIN_COMPRESSION_SIZE_LIMIT = 10_485_760
+      SUPPORTED_ENCODINGS = %w[gzip].freeze
+      CHUNK_SIZE = 1 * 1024 * 1024 # one MB
+
+      option(
+        :disable_request_compression,
+        default: false,
+        doc_type: 'Boolean',
+        docstring: <<-DOCS) do |cfg|
+When set to 'true' the request body will not be compressed
+for supported operations.
+      DOCS
+        resolve_disable_request_compression(cfg)
+      end
+
+      option(
+        :request_min_compression_size_bytes,
+        default: 10_240,
+        doc_type: 'Integer',
+        docstring: <<-DOCS) do |cfg|
+The minimum size in bytes that triggers compression for request
+bodies. The value must be non-negative integer value between 0
+and 10485780 bytes inclusive.
+      DOCS
+        resolve_request_min_compression_size_bytes(cfg)
+      end
+
+      def after_initialize(client)
+        validate_disable_request_compression_input(client.config)
+        validate_request_min_compression_size_bytes_input(client.config)
+      end
+
+      def validate_disable_request_compression_input(cfg)
+        unless [true, false].include?(cfg.disable_request_compression)
+          raise ArgumentError,
+                'Must provide either `true` or `false` for the '\
+                '`disable_request_compression` configuration option.'
+        end
+      end
+
+      def validate_request_min_compression_size_bytes_input(cfg)
+        value = Integer(cfg.request_min_compression_size_bytes)
+        unless value.between?(0, MIN_COMPRESSION_SIZE_LIMIT)
+          raise ArgumentError,
+                'Must provide a non-negative integer value between '\
+                  '`0` and `10485760` bytes inclusive for the '\
+                  '`request_min_compression_size_bytes` configuration option.'
+        end
+      end
+
+      def add_handlers(handlers, _config)
+        # priority set to ensure compression happens BEFORE checksum
+        handlers.add(CompressionHandler, priority: 16, step: :build)
+      end
+
+      class << self
+        private
+
+        def resolve_disable_request_compression(cfg)
+          value = ENV['AWS_DISABLE_REQUEST_COMPRESSION'] ||
+                  Aws.shared_config.disable_request_compression(profile: cfg.profile) ||
+                  'false'
+          Aws::Util.str_2_bool(value)
+        end
+
+        def resolve_request_min_compression_size_bytes(cfg)
+          value = ENV['AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES'] ||
+                  Aws.shared_config.request_min_compression_size_bytes(profile: cfg.profile) ||
+                  DEFAULT_MIN_COMPRESSION_SIZE.to_s
+          Integer(value)
+        end
+      end
+
+      # @api private
+      class CompressionHandler < Seahorse::Client::Handler
+        def call(context)
+          if should_compress?(context)
+            selected_encoding = request_encoding_selection(context)
+            if selected_encoding
+              if streaming?(context.operation.input)
+                process_streaming_compression(selected_encoding, context)
+              elsif context.http_request.body.size >= context.config.request_min_compression_size_bytes
+                process_compression(selected_encoding, context)
+              end
+            end
+          end
+          with_metric(selected_encoding) { @handler.call(context) }
+        end
+
+        private
+
+        def with_metric(encoding, &block)
+          case encoding
+          when 'gzip'
+            Aws::Plugins::UserAgent.metric('GZIP_REQUEST_COMPRESSION', &block)
+          else
+            block.call
+          end
+        end
+
+        def request_encoding_selection(context)
+          encoding_list = context.operation.request_compression['encodings']
+          encoding_list.find { |encoding| RequestCompression::SUPPORTED_ENCODINGS.include?(encoding) }
+        end
+
+        def update_content_encoding(encoding, context)
+          headers = context.http_request.headers
+          if headers['Content-Encoding']
+            headers['Content-Encoding'] += ", #{encoding}"
+          else
+            headers['Content-Encoding'] = encoding
+          end
+        end
+
+        def should_compress?(context)
+          context.operation.request_compression &&
+            !context.config.disable_request_compression
+        end
+
+        def streaming?(input)
+          if payload = input[:payload_member] # checking ref and shape
+            payload['streaming'] || payload.shape['streaming']
+          else
+            false
+          end
+        end
+
+        def process_compression(encoding, context)
+          case encoding
+          when 'gzip'
+            gzip_compress(context)
+          else
+            raise StandardError, "We currently do not support #{encoding} encoding"
+          end
+          update_content_encoding(encoding, context)
+        end
+
+        def gzip_compress(context)
+          compressed = StringIO.new
+          compressed.binmode
+          gzip_writer = Zlib::GzipWriter.new(compressed)
+          if context.http_request.body.respond_to?(:read)
+            update_in_chunks(gzip_writer, context.http_request.body)
+          else
+            gzip_writer.write(context.http_request.body)
+          end
+          gzip_writer.close
+          new_body = StringIO.new(compressed.string)
+          context.http_request.body = new_body
+        end
+
+        def update_in_chunks(compressor, io)
+          loop do
+            chunk = io.read(CHUNK_SIZE)
+            break unless chunk
+
+            compressor.write(chunk)
+          end
+        end
+
+        def process_streaming_compression(encoding, context)
+          case encoding
+          when 'gzip'
+            context.http_request.body = GzipIO.new(context.http_request.body)
+          else
+            raise StandardError, "We currently do not support #{encoding} encoding"
+          end
+          update_content_encoding(encoding, context)
+        end
+
+        # @api private
+        class GzipIO
+          def initialize(body)
+            @body = body
+            @buffer = ChunkBuffer.new
+            @gzip_writer = Zlib::GzipWriter.new(@buffer)
+          end
+
+          def read(length, buff = nil)
+            if @gzip_writer.closed?
+              # an empty string to signify an end as
+              # there will be nothing remaining to be read
+              StringIO.new('').read(length, buff)
+              return
+            end
+
+            chunk = @body.read(length)
+            if !chunk || chunk.empty?
+              # closing the writer will write one last chunk
+              # with a trailer (to be read from the @buffer)
+              @gzip_writer.close
+            else
+              # flush happens first to ensure that header fields
+              # are being sent over since write will override
+              @gzip_writer.flush
+              @gzip_writer.write(chunk)
+            end
+
+            StringIO.new(@buffer.last_chunk).read(length, buff)
+          end
+        end
+
+        # @api private
+        class ChunkBuffer
+          def initialize
+            @last_chunk = nil
+          end
+
+          attr_reader :last_chunk
+
+          def write(data)
+            @last_chunk = data
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/retries/clock_skew.rb

@@ -3,10 +3,8 @@
 module Aws
   module Plugins
     module Retries
-
       # @api private
       class ClockSkew
-
         CLOCK_SKEW_THRESHOLD = 5 * 60 # five minutes
 
         def initialize
@@ -22,9 +20,9 @@ module Aws
         end
 
         # Gets the clock_correction in seconds to apply to a given endpoint
-        # @param endpoint [URI / String]
+        # @param endpoint [URI, String]
         def clock_correction(endpoint)
-          @mutex.synchronize { @endpoint_clock_corrections[endpoint.to_s] }
+          @mutex.synchronize { @endpoint_clock_corrections[normalized_endpoint(endpoint)] }
         end
 
         # The estimated skew factors in any clock skew from
@@ -35,7 +33,7 @@ module Aws
         # Estimated Skew should not be used to correct clock skew errors
         # it should only be used to estimate TTL for a request
         def estimated_skew(endpoint)
-          @mutex.synchronize { @endpoint_estimated_skews[endpoint.to_s] }
+          @mutex.synchronize { @endpoint_estimated_skews[normalized_endpoint(endpoint)] }
         end
 
         # Determines whether a request has clock skew by comparing
@@ -55,9 +53,9 @@ module Aws
           endpoint = context.http_request.endpoint
           now_utc = Time.now.utc
           server_time = server_time(context.http_response)
-          if server_time && (now_utc - server_time).abs > CLOCK_SKEW_THRESHOLD
-            set_clock_correction(endpoint, server_time - now_utc)
-          end
+          return unless server_time && (now_utc - server_time).abs > CLOCK_SKEW_THRESHOLD
+
+          set_clock_correction(normalized_endpoint(endpoint), server_time - now_utc)
         end
 
         # Called for every request
@@ -69,20 +67,35 @@ module Aws
           now_utc = Time.now.utc
           server_time = server_time(context.http_response)
           return unless server_time
+
           @mutex.synchronize do
-            @endpoint_estimated_skews[endpoint.to_s] = server_time - now_utc
+            @endpoint_estimated_skews[normalized_endpoint(endpoint)] = server_time - now_utc
           end
         end
 
         private
 
+        ##
+        # @param endpoint [URI, String]
+        #     the endpoint to normalize
+        #
+        # @return [String]
+        #     the endpoint's schema, host, and port - without any path or query arguments
+        def normalized_endpoint(endpoint)
+          uri = endpoint.is_a?(URI::Generic) ? endpoint : URI(endpoint.to_s)
+
+          return endpoint.to_s unless uri.scheme && uri.host
+
+          "#{uri.scheme}://#{uri.host}:#{uri.port}"
+        rescue URI::InvalidURIError
+          endpoint.to_s
+        end
+
         # @param response [Seahorse::Client::Http::Response:]
         def server_time(response)
-          begin
-            Time.parse(response.headers['date']).utc
-          rescue
-            nil
-          end
+          Time.parse(response.headers['date']).utc
+        rescue StandardError
+          nil
         end
 
         # Sets the clock correction for an endpoint
@@ -90,11 +103,10 @@ module Aws
         # @param correction [Number]
         def set_clock_correction(endpoint, correction)
           @mutex.synchronize do
-            @endpoint_clock_corrections[endpoint.to_s] = correction
+            @endpoint_clock_corrections[normalized_endpoint(endpoint)] = correction
           end
         end
       end
     end
   end
 end
-

data/lib/aws-sdk-core/plugins/retries/retry_quota.rb

@@ -8,9 +8,11 @@ module Aws
       # Used in 'standard' and 'adaptive' retry modes.
       class RetryQuota
         INITIAL_RETRY_TOKENS = 500
-        RETRY_COST = 5
+        RETRY_COST = 14
+        LEGACY_RETRY_COST = 5 # TODO: Remove when new retries become default
         NO_RETRY_INCREMENT = 1
-        TIMEOUT_RETRY_COST = 10
+        THROTTLING_RETRY_COST = 5
+        TIMEOUT_RETRY_COST = 10 # TODO: Remove when new retries become default
 
         def initialize(opts = {})
           @mutex              = Mutex.new
@@ -19,15 +21,16 @@ module Aws
         end
 
         # check if there is sufficient capacity to retry
-        # and return it.  If there is insufficient capacity
+        # and return it. If there is insufficient capacity
         # return 0
         # @return [Integer] The amount of capacity checked out
         def checkout_capacity(error_inspector)
           @mutex.synchronize do
-            capacity_amount = if error_inspector.networking?
-                                TIMEOUT_RETRY_COST
+            # TODO: Remove gate and keep only the new_retries branch
+            capacity_amount = if RetryErrors.new_retries?
+                                error_inspector.throttling_error? ? THROTTLING_RETRY_COST : RETRY_COST
                               else
-                                RETRY_COST
+                                error_inspector.networking? ? TIMEOUT_RETRY_COST : LEGACY_RETRY_COST
                               end
 
             # unable to acquire capacity
@@ -39,8 +42,8 @@ module Aws
         end
 
         # capacity_amount refers to the amount of capacity requested from
-        # the last retry.  It can either be RETRY_COST, TIMEOUT_RETRY_COST,
-        # or unset.
+        # the last retry. It can either be RETRY_COST,
+        # THROTTLING_RETRY_COST/TIMEOUT_RETRY_COST, or unset.
         def release(capacity_amount)
           # Implementation note:  The release() method is called for
           # every API call.  In the common case where the request is