aws-sdk-core 3.168.4 → 3.190.3

data/lib/aws-sdk-core/plugins/regional_endpoint.rb

@@ -47,44 +47,21 @@ is set to `true`.
       # Legacy endpoints must continue to be generated at client time.
       option(:regional_endpoint, false)
 
-      # NOTE: All of the defaults block code is effectively deprecated.
-      # Because old services can depend on this new core version, we must
-      # retain it.
+      option(:ignore_configured_endpoint_urls,
+        doc_type: 'Boolean',
+        docstring: <<-DOCS) do |cfg|
+Setting to true disables use of endpoint URLs provided via environment 
+variables and the shared configuration file.
+        DOCS
+        resolve_ignore_configured_endpoint_urls(cfg)
+      end
+
       option(:endpoint, doc_type: String, docstring: <<-DOCS) do |cfg|
 The client endpoint is normally constructed from the `:region`
 option. You should only configure an `:endpoint` when connecting
 to test or custom endpoints. This should be a valid HTTP(S) URI.
         DOCS
-        endpoint_prefix = cfg.api.metadata['endpointPrefix']
-        if cfg.region && endpoint_prefix
-          if cfg.respond_to?(:sts_regional_endpoints)
-            sts_regional = cfg.sts_regional_endpoints
-          end
-
-          # check region is a valid RFC host label
-          unless Seahorse::Util.host_label?(cfg.region)
-            raise Errors::InvalidRegionError
-          end
-
-          region = cfg.region
-          new_region = region.gsub('fips-', '').gsub('-fips', '')
-          if region != new_region
-            warn("Legacy region #{region} was transformed to #{new_region}."\
-                 '`use_fips_endpoint` config was set to true.')
-            cfg.override_config(:use_fips_endpoint, true)
-            cfg.override_config(:region, new_region)
-          end
-
-          Aws::Partitions::EndpointProvider.resolve(
-            cfg.region,
-            endpoint_prefix,
-            sts_regional,
-            {
-              dualstack: cfg.use_dualstack_endpoint,
-              fips: cfg.use_fips_endpoint
-            }
-          )
-        end
+        resolve_endpoint(cfg)
       end
 
       def after_initialize(client)
@@ -117,6 +94,105 @@ to test or custom endpoints. This should be a valid HTTP(S) URI.
           value ||= Aws.shared_config.use_fips_endpoint(profile: cfg.profile)
           Aws::Util.str_2_bool(value) || false
         end
+
+        def resolve_ignore_configured_endpoint_urls(cfg)
+          value = ENV['AWS_IGNORE_CONFIGURED_ENDPOINT_URLS']
+          value ||= Aws.shared_config.ignore_configured_endpoint_urls(profile: cfg.profile)
+          Aws::Util.str_2_bool(value&.downcase) || false
+        end
+
+        # NOTE: with Endpoints 2.0, some of this logic is deprecated
+        # but because new old service gems may depend on new core versions
+        # we must preserve that behavior.
+        # Additional behavior controls the setting of the custom SDK::Endpoint
+        # parameter.
+        # When the `regional_endpoint` config is set to true - this indicates to
+        # Endpoints2.0 that a custom endpoint has NOT been configured by the user.
+        def resolve_endpoint(cfg)
+          endpoint = resolve_custom_config_endpoint(cfg)
+          endpoint_prefix = cfg.api.metadata['endpointPrefix']
+
+          return endpoint unless endpoint.nil? && cfg.region && endpoint_prefix
+
+          validate_region!(cfg.region)
+          handle_legacy_pseudo_regions(cfg)
+
+          # set regional_endpoint flag - this indicates to Endpoints 2.0
+          # that a custom endpoint has NOT been configured by the user
+          cfg.override_config(:regional_endpoint, true)
+
+          resolve_legacy_endpoint(cfg)
+        end
+
+        # get a custom configured endpoint from ENV or configuration
+        def resolve_custom_config_endpoint(cfg)
+          return if cfg.ignore_configured_endpoint_urls
+
+
+          env_service_endpoint(cfg) || env_global_endpoint(cfg) || shared_config_endpoint(cfg)
+        end
+
+        def env_service_endpoint(cfg)
+          service_id = cfg.api.metadata['serviceId'] || cfg.api.metadata['endpointPrefix']
+          env_service_id = service_id.gsub(" ", "_").upcase
+          return unless endpoint = ENV["AWS_ENDPOINT_URL_#{env_service_id}"]
+
+          cfg.logger&.debug(
+            "Endpoint configured from ENV['AWS_ENDPOINT_URL_#{env_service_id}']: #{endpoint}\n")
+          endpoint
+        end
+
+        def env_global_endpoint(cfg)
+          return unless endpoint = ENV['AWS_ENDPOINT_URL']
+
+          cfg.logger&.debug(
+            "Endpoint configured from ENV['AWS_ENDPOINT_URL']: #{endpoint}\n")
+          endpoint
+        end
+
+        def shared_config_endpoint(cfg)
+          service_id = cfg.api.metadata['serviceId'] || cfg.api.metadata['endpointPrefix']
+          return unless endpoint = Aws.shared_config.configured_endpoint(profile: cfg.profile, service_id: service_id)
+
+          cfg.logger&.debug(
+            "Endpoint configured from shared config(profile: #{cfg.profile}): #{endpoint}\n")
+          endpoint
+        end
+
+        # check region is a valid RFC host label
+        def validate_region!(region)
+          unless Seahorse::Util.host_label?(region)
+            raise Errors::InvalidRegionError
+          end
+        end
+
+        def handle_legacy_pseudo_regions(cfg)
+          region = cfg.region
+          new_region = region.gsub('fips-', '').gsub('-fips', '')
+          if region != new_region
+            warn("Legacy region #{region} was transformed to #{new_region}."\
+                 '`use_fips_endpoint` config was set to true.')
+            cfg.override_config(:use_fips_endpoint, true)
+            cfg.override_config(:region, new_region)
+          end
+        end
+        # set a default endpoint in config using legacy (endpoints.json) resolver
+        def resolve_legacy_endpoint(cfg)
+          endpoint_prefix = cfg.api.metadata['endpointPrefix']
+          if cfg.respond_to?(:sts_regional_endpoints)
+            sts_regional = cfg.sts_regional_endpoints
+          end
+
+          Aws::Partitions::EndpointProvider.resolve(
+            cfg.region,
+            endpoint_prefix,
+            sts_regional,
+            {
+              dualstack: cfg.use_dualstack_endpoint,
+              fips: cfg.use_fips_endpoint
+            }
+          )
+        end
       end
     end
   end

data/lib/aws-sdk-core/plugins/request_compression.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+module Aws
+  module Plugins
+    # @api private
+    class RequestCompression < Seahorse::Client::Plugin
+      DEFAULT_MIN_COMPRESSION_SIZE = 10_240
+      MIN_COMPRESSION_SIZE_LIMIT = 10_485_760
+      SUPPORTED_ENCODINGS = %w[gzip].freeze
+      CHUNK_SIZE = 1 * 1024 * 1024 # one MB
+
+      option(
+        :disable_request_compression,
+        default: false,
+        doc_type: 'Boolean',
+        docstring: <<-DOCS) do |cfg|
+When set to 'true' the request body will not be compressed
+for supported operations.
+      DOCS
+        resolve_disable_request_compression(cfg)
+      end
+
+      option(
+        :request_min_compression_size_bytes,
+        default: 10_240,
+        doc_type: 'Integer',
+        docstring: <<-DOCS) do |cfg|
+The minimum size in bytes that triggers compression for request
+bodies. The value must be non-negative integer value between 0
+and 10485780 bytes inclusive.
+      DOCS
+        resolve_request_min_compression_size_bytes(cfg)
+      end
+
+      def after_initialize(client)
+        validate_disable_request_compression_input(client.config)
+        validate_request_min_compression_size_bytes_input(client.config)
+      end
+
+      def validate_disable_request_compression_input(cfg)
+        unless [true, false].include?(cfg.disable_request_compression)
+          raise ArgumentError,
+                'Must provide either `true` or `false` for the '\
+                '`disable_request_compression` configuration option.'
+        end
+      end
+
+      def validate_request_min_compression_size_bytes_input(cfg)
+        value = Integer(cfg.request_min_compression_size_bytes)
+        unless value.between?(0, MIN_COMPRESSION_SIZE_LIMIT)
+          raise ArgumentError,
+                'Must provide a non-negative integer value between '\
+                  '`0` and `10485760` bytes inclusive for the '\
+                  '`request_min_compression_size_bytes` configuration option.'
+        end
+      end
+
+      def add_handlers(handlers, _config)
+        # priority set to ensure compression happens BEFORE checksum
+        handlers.add(CompressionHandler, priority: 16, step: :build)
+      end
+
+      class << self
+        private
+
+        def resolve_disable_request_compression(cfg)
+          value = ENV['AWS_DISABLE_REQUEST_COMPRESSION'] ||
+                  Aws.shared_config.disable_request_compression(profile: cfg.profile) ||
+                  'false'
+          Aws::Util.str_2_bool(value)
+        end
+
+        def resolve_request_min_compression_size_bytes(cfg)
+          value = ENV['AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES'] ||
+                  Aws.shared_config.request_min_compression_size_bytes(profile: cfg.profile) ||
+                  DEFAULT_MIN_COMPRESSION_SIZE.to_s
+          Integer(value)
+        end
+      end
+
+      # @api private
+      class CompressionHandler < Seahorse::Client::Handler
+        def call(context)
+          if should_compress?(context)
+            selected_encoding = request_encoding_selection(context)
+            if selected_encoding
+              if streaming?(context.operation.input)
+                process_streaming_compression(selected_encoding, context)
+              elsif context.http_request.body.size >= context.config.request_min_compression_size_bytes
+                process_compression(selected_encoding, context)
+              end
+            end
+          end
+          @handler.call(context)
+        end
+
+        private
+
+        def request_encoding_selection(context)
+          encoding_list = context.operation.request_compression['encodings']
+          encoding_list.find { |encoding| RequestCompression::SUPPORTED_ENCODINGS.include?(encoding) }
+        end
+
+        def update_content_encoding(encoding, context)
+          headers = context.http_request.headers
+          if headers['Content-Encoding']
+            headers['Content-Encoding'] += ',' + encoding
+          else
+            headers['Content-Encoding'] = encoding
+          end
+        end
+
+        def should_compress?(context)
+          context.operation.request_compression &&
+            !context.config.disable_request_compression
+        end
+
+        def streaming?(input)
+          if payload = input[:payload_member] # checking ref and shape
+            payload['streaming'] || payload.shape['streaming']
+          else
+            false
+          end
+        end
+
+        def process_compression(encoding, context)
+          case encoding
+          when 'gzip'
+            gzip_compress(context)
+          else
+            raise StandardError, "We currently do not support #{encoding} encoding"
+          end
+          update_content_encoding(encoding, context)
+        end
+
+        def gzip_compress(context)
+          compressed = StringIO.new
+          compressed.binmode
+          gzip_writer = Zlib::GzipWriter.new(compressed)
+          if context.http_request.body.respond_to?(:read)
+            update_in_chunks(gzip_writer, context.http_request.body)
+          else
+            gzip_writer.write(context.http_request.body)
+          end
+          gzip_writer.close
+          new_body = StringIO.new(compressed.string)
+          context.http_request.body = new_body
+        end
+
+        def update_in_chunks(compressor, io)
+          loop do
+            chunk = io.read(CHUNK_SIZE)
+            break unless chunk
+
+            compressor.write(chunk)
+          end
+        end
+
+        def process_streaming_compression(encoding, context)
+          case encoding
+          when 'gzip'
+            context.http_request.body = GzipIO.new(context.http_request.body)
+          else
+            raise StandardError, "We currently do not support #{encoding} encoding"
+          end
+          update_content_encoding(encoding, context)
+        end
+
+        # @api private
+        class GzipIO
+          def initialize(body)
+            @body = body
+            @buffer = ChunkBuffer.new
+            @gzip_writer = Zlib::GzipWriter.new(@buffer)
+          end
+
+          def read(length, buff = nil)
+            if @gzip_writer.closed?
+              # an empty string to signify an end as
+              # there will be nothing remaining to be read
+              StringIO.new('').read(length, buff)
+              return
+            end
+
+            chunk = @body.read(length)
+            if !chunk || chunk.empty?
+              # closing the writer will write one last chunk
+              # with a trailer (to be read from the @buffer)
+              @gzip_writer.close
+            else
+              # flush happens first to ensure that header fields
+              # are being sent over since write will override
+              @gzip_writer.flush
+              @gzip_writer.write(chunk)
+            end
+
+            StringIO.new(@buffer.last_chunk).read(length, buff)
+          end
+        end
+
+        # @api private
+        class ChunkBuffer
+          def initialize
+            @last_chunk = nil
+          end
+
+          attr_reader :last_chunk
+
+          def write(data)
+            @last_chunk = data
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/sign.rb

@@ -13,7 +13,7 @@ module Aws
       option(:sigv4_region)
       option(:unsigned_operations, default: [])
 
-      supported_auth_types = %w[sigv4 bearer none]
+      supported_auth_types = %w[sigv4 bearer sigv4-s3express none]
       supported_auth_types += ['sigv4a'] if Aws::Sigv4::Signer.use_crt?
       SUPPORTED_AUTH_TYPES = supported_auth_types.freeze
 
@@ -24,10 +24,14 @@ module Aws
 
       # @api private
       # Return a signer with the `sign(context)` method
-      def self.signer_for(auth_scheme, config, region_override = nil)
+      def self.signer_for(auth_scheme, config, sigv4_region_override = nil, sigv4_credentials_override = nil)
         case auth_scheme['name']
-        when 'sigv4', 'sigv4a'
-          SignatureV4.new(auth_scheme, config, region_override)
+        when 'sigv4', 'sigv4a', 'sigv4-s3express'
+          sigv4_overrides = {
+            region: sigv4_region_override,
+            credentials: sigv4_credentials_override
+          }
+          SignatureV4.new(auth_scheme, config, sigv4_overrides)
         when 'bearer'
           Bearer.new
         else
@@ -42,7 +46,8 @@ module Aws
             signer = Sign.signer_for(
               context[:auth_scheme],
               context.config,
-              context[:sigv4_region]
+              context[:sigv4_region],
+              context[:sigv4_credentials]
             )
             signer.sign(context)
           end
@@ -88,12 +93,12 @@ module Aws
 
       # @api private
       class SignatureV4
-        def initialize(auth_scheme, config, region_override = nil)
+        def initialize(auth_scheme, config, sigv4_overrides = {})
           scheme_name = auth_scheme['name']
 
-          unless %w[sigv4 sigv4a].include?(scheme_name)
+          unless %w[sigv4 sigv4a sigv4-s3express].include?(scheme_name)
             raise ArgumentError,
-                  "Expected sigv4 or sigv4a auth scheme, got #{scheme_name}"
+                  "Expected sigv4, sigv4a, or sigv4-s3express auth scheme, got #{scheme_name}"
           end
 
           region = if scheme_name == 'sigv4a'
@@ -104,10 +109,11 @@ module Aws
           begin
             @signer = Aws::Sigv4::Signer.new(
               service: config.sigv4_name || auth_scheme['signingName'],
-              region: region_override || config.sigv4_region || region,
-              credentials_provider: config.credentials,
+              region: sigv4_overrides[:region] || config.sigv4_region || region,
+              credentials_provider: sigv4_overrides[:credentials] || config.credentials,
               signing_algorithm: scheme_name.to_sym,
               uri_escape_path: !!!auth_scheme['disableDoubleEncoding'],
+              normalize_path: !!!auth_scheme['disableNormalizePath'],
               unsigned_headers: %w[content-length user-agent x-amzn-trace-id]
             )
           rescue Aws::Sigv4::Errors::MissingCredentialsError

data/lib/aws-sdk-core/plugins/user_agent.rb

@@ -4,7 +4,31 @@ module Aws
   module Plugins
     # @api private
     class UserAgent < Seahorse::Client::Plugin
+      # @api private
       option(:user_agent_suffix)
+      # @api private
+      option(:user_agent_frameworks, default: [])
+
+      option(
+        :sdk_ua_app_id,
+        doc_type: 'String',
+        docstring: <<-DOCS) do |cfg|
+A unique and opaque application ID that is appended to the
+User-Agent header as app/<sdk_ua_app_id>. It should have a
+maximum length of 50.
+        DOCS
+        app_id = ENV['AWS_SDK_UA_APP_ID']
+        app_id ||= Aws.shared_config.sdk_ua_app_id(profile: cfg.profile)
+        app_id
+      end
+
+      def self.feature(feature, &block)
+        Thread.current[:aws_sdk_core_user_agent_feature] ||= []
+        Thread.current[:aws_sdk_core_user_agent_feature] << "ft/#{feature}"
+        block.call
+      ensure
+        Thread.current[:aws_sdk_core_user_agent_feature].pop
+      end
 
       # @api private
       class Handler < Seahorse::Client::Handler
@@ -14,33 +38,112 @@ module Aws
         end
 
         def set_user_agent(context)
-          ua = "aws-sdk-ruby3/#{CORE_GEM_VERSION}"
+          context.http_request.headers['User-Agent'] = UserAgent.new(context).to_s
+        end
+
+        class UserAgent
+          def initialize(context)
+            @context = context
+          end
+
+          def to_s
+            ua = "aws-sdk-ruby3/#{CORE_GEM_VERSION}"
+            ua += ' ua/2.0'
+            ua += " #{api_metadata}" if api_metadata
+            ua += " #{os_metadata}"
+            ua += " #{language_metadata}"
+            ua += " #{env_metadata}" if env_metadata
+            ua += " #{config_metadata}" if config_metadata
+            ua += " #{app_id}" if app_id
+            ua += " #{feature_metadata}" if feature_metadata
+            ua += " #{framework_metadata}" if framework_metadata
+            if @context.config.user_agent_suffix
+              ua += " #{@context.config.user_agent_suffix}"
+            end
+            ua.strip
+          end
+
+          private
 
-          begin
-            ua += " #{RUBY_ENGINE}/#{RUBY_VERSION}"
-          rescue
-            ua += " RUBY_ENGINE_NA/#{RUBY_VERSION}"
+          # Used to be gem_name/gem_version
+          def api_metadata
+            service_id = @context.config.api.metadata['serviceId']
+            return unless service_id
+
+            service_id = service_id.gsub(' ', '_').downcase
+            gem_version = @context[:gem_version]
+            "api/#{service_id}##{gem_version}"
+          end
+
+          # Used to be RUBY_PLATFORM
+          def os_metadata
+            os =
+              case RbConfig::CONFIG['host_os']
+              when /mac|darwin/
+                'macos'
+              when /linux|cygwin/
+                'linux'
+              when /mingw|mswin/
+                'windows'
+              else
+                'other'
+              end
+            metadata = "os/#{os}"
+            local_version = Gem::Platform.local.version
+            metadata += "##{local_version}" if local_version
+            metadata += " md/#{RbConfig::CONFIG['host_cpu']}"
+            metadata
           end
 
-          ua += " #{RUBY_PLATFORM}"
+          # Used to be RUBY_ENGINE/RUBY_VERSION
+          def language_metadata
+            "lang/#{RUBY_ENGINE}##{RUBY_ENGINE_VERSION} md/#{RUBY_VERSION}"
+          end
+
+          def env_metadata
+            return unless (execution_env = ENV['AWS_EXECUTION_ENV'])
+
+            "exec-env/#{execution_env}"
+          end
 
-          if context[:gem_name] && context[:gem_version]
-            ua += " #{context[:gem_name]}/#{context[:gem_version]}"
+          def config_metadata
+            "cfg/retry-mode##{@context.config.retry_mode}"
           end
 
-          if (execution_env = ENV['AWS_EXECUTION_ENV'])
-            ua += " exec-env/#{execution_env}"
+          def app_id
+            return unless (app_id = @context.config.sdk_ua_app_id)
+
+            # Sanitize and only allow these characters
+            app_id = app_id.gsub(/[^!#$%&'*+\-.^_`|~0-9A-Za-z]/, '-')
+            "app/#{app_id}"
           end
 
-          if context.config.user_agent_suffix
-            ua += " #{context.config.user_agent_suffix}"
+          def feature_metadata
+            return unless Thread.current[:aws_sdk_core_user_agent_feature]
+
+            Thread.current[:aws_sdk_core_user_agent_feature].join(' ')
           end
 
-          context.http_request.headers['User-Agent'] = ua.strip
+          def framework_metadata
+            if (frameworks_cfg = @context.config.user_agent_frameworks).empty?
+              return
+            end
+
+            # Frameworks may be aws-record, aws-sdk-rails, etc.
+            regex = /gems\/(?<name>#{frameworks_cfg.join('|')})-(?<version>\d+\.\d+\.\d+)/.freeze
+            frameworks = {}
+            Kernel.caller.each do |line|
+              match = line.match(regex)
+              next unless match
+
+              frameworks[match[:name]] = match[:version]
+            end
+            frameworks.map { |n, v| "lib/#{n}##{v}" }.join(' ')
+          end
         end
       end
 
-      handler(Handler)
+      handler(Handler, priority: 1)
     end
   end
 end

data/lib/aws-sdk-core/refreshing_credentials.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'thread'
-
 module Aws
 
   # Base class used credential classes that can be refreshed. This
@@ -36,12 +34,6 @@ module Aws
       @credentials
     end
 
-    # @return [Time,nil]
-    def expiration
-      refresh_if_near_expiration!
-      @expiration
-    end
-
     # Refresh credentials.
     # @return [void]
     def refresh!
@@ -54,6 +46,14 @@ module Aws
 
     private
 
+    def sync_expiration_length
+      self.class::SYNC_EXPIRATION_LENGTH
+    end
+
+    def async_expiration_length
+      self.class::ASYNC_EXPIRATION_LENGTH
+    end
+
     # Refreshes credentials asynchronously and synchronously.
     # If we are near to expiration, block while getting new credentials.
     # Otherwise, if we're approaching expiration, use the existing credentials
@@ -62,18 +62,18 @@ module Aws
       # Note: This check is an optimization. Rather than acquire the mutex on every #refresh_if_near_expiration
       # call, we check before doing so, and then we check within the mutex to avoid a race condition.
       # See issue: https://github.com/aws/aws-sdk-ruby/issues/2641 for more info.
-      if near_expiration?(SYNC_EXPIRATION_LENGTH)
+      if near_expiration?(sync_expiration_length)
         @mutex.synchronize do
-          if near_expiration?(SYNC_EXPIRATION_LENGTH)
+          if near_expiration?(sync_expiration_length)
             @before_refresh.call(self) if @before_refresh
             refresh
           end
         end
-      elsif @async_refresh && near_expiration?(ASYNC_EXPIRATION_LENGTH)
+      elsif @async_refresh && near_expiration?(async_expiration_length)
         unless @mutex.locked?
           Thread.new do
             @mutex.synchronize do
-              if near_expiration?(ASYNC_EXPIRATION_LENGTH)
+              if near_expiration?(async_expiration_length)
                 @before_refresh.call(self) if @before_refresh
                 refresh
               end