aws-sdk-core 3.100.0 → 3.191.0

data/lib/aws-sdk-core/plugins/retry_errors.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'set'
 require_relative 'retries/error_inspector'
 require_relative 'retries/retry_quota'
@@ -71,6 +73,7 @@ is only used in the `legacy` retry mode.
         :retry_jitter,
         default: :none,
         doc_type: Symbol,
+        rbs_type: '(:none | :equal | :full | ^(Integer) -> Integer)',
         docstring: <<-DOCS)
 A delay randomiser function used by the default backoff function.
 Some predefined functions can be referenced by name - :none, :equal, :full,
@@ -95,6 +98,7 @@ This option is only used in the `legacy` retry mode.
         :retry_mode,
         default: 'legacy',
         doc_type: String,
+        rbs_type: '("legacy" | "standard" | "adaptive")',
         docstring: <<-DOCS) do |cfg|
 Specifies which retry algorithm to use. Values are:
 
@@ -161,9 +165,15 @@ a clock skew correction and retry requests with skewed client clocks.
       option(:clock_skew) { Retries::ClockSkew.new }
 
       def self.resolve_retry_mode(cfg)
-        value = ENV['AWS_RETRY_MODE'] ||
-                Aws.shared_config.retry_mode(profile: cfg.profile) ||
-                'legacy'
+        default_mode_value =
+          if cfg.respond_to?(:defaults_mode_config_resolver)
+            cfg.defaults_mode_config_resolver.resolve(:retry_mode)
+          end
+
+          value = ENV['AWS_RETRY_MODE'] ||
+                  Aws.shared_config.retry_mode(profile: cfg.profile) ||
+                  default_mode_value ||
+                  'legacy'
         # Raise if provided value is not one of the retry modes
         if value != 'legacy' && value != 'standard' && value != 'adaptive'
           raise ArgumentError,
@@ -174,11 +184,12 @@ a clock skew correction and retry requests with skewed client clocks.
       end
 
       def self.resolve_max_attempts(cfg)
-        value = (ENV['AWS_MAX_ATTEMPTS'] && ENV['AWS_MAX_ATTEMPTS'].to_i) ||
+        value = (ENV['AWS_MAX_ATTEMPTS']) ||
                 Aws.shared_config.max_attempts(profile: cfg.profile) ||
-                3
+                '3'
+        value = value.to_i
         # Raise if provided value is not a positive integer
-        if !value.is_a?(Integer) || value <= 0
+        if value <= 0
           raise ArgumentError,
             'Must provide a positive integer for max_attempts profile '\
             'option or for ENV[\'AWS_MAX_ATTEMPTS\']'
@@ -304,12 +315,17 @@ a clock skew correction and retry requests with skewed client clocks.
 
         def retry_request(context, error)
           context.retries += 1
-          context.config.credentials.refresh! if error.expired_credentials?
+          context.config.credentials.refresh! if refresh_credentials?(context, error)
           context.http_request.body.rewind
           context.http_response.reset
           call(context)
         end
 
+        def refresh_credentials?(context, error)
+          error.expired_credentials? &&
+            context.config.credentials.respond_to?(:refresh!)
+        end
+
         def add_retry_headers(context)
           request_pairs = {
             'attempt' => context.retries,
@@ -374,7 +390,7 @@ a clock skew correction and retry requests with skewed client clocks.
         def retry_request(context, error)
           delay_retry(context)
           context.retries += 1
-          context.config.credentials.refresh! if error.expired_credentials?
+          context.config.credentials.refresh! if refresh_credentials?(context, error)
           context.http_request.body.rewind
           context.http_response.reset
           call(context)
@@ -390,6 +406,11 @@ a clock skew correction and retry requests with skewed client clocks.
             response_truncatable?(context)
         end
 
+        def refresh_credentials?(context, error)
+          error.expired_credentials? &&
+            context.config.credentials.respond_to?(:refresh!)
+        end
+
         def retry_limit(context)
           context.config.retry_limit
         end

data/lib/aws-sdk-core/plugins/sign.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require 'aws-sigv4'
+
+module Aws
+  module Plugins
+    # @api private
+    class Sign < Seahorse::Client::Plugin
+      # These once had defaults. But now they are used as overrides to
+      # new endpoint and auth resolution.
+      option(:sigv4_signer)
+      option(:sigv4_name)
+      option(:sigv4_region)
+      option(:unsigned_operations, default: [])
+
+      supported_auth_types = %w[sigv4 bearer sigv4-s3express none]
+      supported_auth_types += ['sigv4a'] if Aws::Sigv4::Signer.use_crt?
+      SUPPORTED_AUTH_TYPES = supported_auth_types.freeze
+
+      def add_handlers(handlers, cfg)
+        operations = cfg.api.operation_names - cfg.unsigned_operations
+        handlers.add(Handler, step: :sign, operations: operations)
+      end
+
+      # @api private
+      # Return a signer with the `sign(context)` method
+      def self.signer_for(auth_scheme, config, sigv4_region_override = nil, sigv4_credentials_override = nil)
+        case auth_scheme['name']
+        when 'sigv4', 'sigv4a', 'sigv4-s3express'
+          sigv4_overrides = {
+            region: sigv4_region_override,
+            credentials: sigv4_credentials_override
+          }
+          SignatureV4.new(auth_scheme, config, sigv4_overrides)
+        when 'bearer'
+          Bearer.new
+        else
+          NullSigner.new
+        end
+      end
+
+      class Handler < Seahorse::Client::Handler
+        def call(context)
+          # Skip signing if using sigv2 signing from s3_signer in S3
+          unless v2_signing?(context.config)
+            signer = Sign.signer_for(
+              context[:auth_scheme],
+              context.config,
+              context[:sigv4_region],
+              context[:sigv4_credentials]
+            )
+            signer.sign(context)
+          end
+          @handler.call(context)
+        end
+
+        private
+
+        def v2_signing?(config)
+          # 's3' is legacy signing, 'v4' is default
+          config.respond_to?(:signature_version) &&
+            config.signature_version == 's3'
+        end
+      end
+
+      # @api private
+      class Bearer
+        def initialize
+        end
+
+        def sign(context)
+          if context.http_request.endpoint.scheme != 'https'
+            raise ArgumentError,
+                  'Unable to use bearer authorization on non https endpoint.'
+          end
+
+          token_provider = context.config.token_provider
+
+          raise Errors::MissingBearerTokenError unless token_provider&.set?
+
+          context.http_request.headers['Authorization'] =
+            "Bearer #{token_provider.token.token}"
+        end
+
+        def presign_url(*args)
+          raise ArgumentError, 'Bearer auth does not support presigned urls'
+        end
+
+        def sign_event(*args)
+          raise ArgumentError, 'Bearer auth does not support event signing'
+        end
+      end
+
+      # @api private
+      class SignatureV4
+        def initialize(auth_scheme, config, sigv4_overrides = {})
+          scheme_name = auth_scheme['name']
+
+          unless %w[sigv4 sigv4a sigv4-s3express].include?(scheme_name)
+            raise ArgumentError,
+                  "Expected sigv4, sigv4a, or sigv4-s3express auth scheme, got #{scheme_name}"
+          end
+
+          region = if scheme_name == 'sigv4a'
+                     auth_scheme['signingRegionSet'].first
+                   else
+                     auth_scheme['signingRegion']
+                   end
+          begin
+            @signer = Aws::Sigv4::Signer.new(
+              service: config.sigv4_name || auth_scheme['signingName'],
+              region: sigv4_overrides[:region] || config.sigv4_region || region,
+              credentials_provider: sigv4_overrides[:credentials] || config.credentials,
+              signing_algorithm: scheme_name.to_sym,
+              uri_escape_path: !!!auth_scheme['disableDoubleEncoding'],
+              normalize_path: !!!auth_scheme['disableNormalizePath'],
+              unsigned_headers: %w[content-length user-agent x-amzn-trace-id]
+            )
+          rescue Aws::Sigv4::Errors::MissingCredentialsError
+            raise Aws::Errors::MissingCredentialsError
+          end
+        end
+
+        def sign(context)
+          req = context.http_request
+
+          apply_authtype(context, req)
+          reset_signature(req)
+          apply_clock_skew(context, req)
+
+          # compute the signature
+          begin
+            signature = @signer.sign_request(
+              http_method: req.http_method,
+              url: req.endpoint,
+              headers: req.headers,
+              body: req.body
+            )
+          rescue Aws::Sigv4::Errors::MissingCredentialsError
+            # Necessary for when credentials is explicitly set to nil
+            raise Aws::Errors::MissingCredentialsError
+          end
+          # apply signature headers
+          req.headers.update(signature.headers)
+
+          # add request metadata with signature components for debugging
+          context[:canonical_request] = signature.canonical_request
+          context[:string_to_sign] = signature.string_to_sign
+        end
+
+        def presign_url(*args)
+          @signer.presign_url(*args)
+        end
+
+        def sign_event(*args)
+          @signer.sign_event(*args)
+        end
+
+        private
+
+        def apply_authtype(context, req)
+          if context.operation['authtype'].eql?('v4-unsigned-body') &&
+             req.endpoint.scheme.eql?('https')
+            req.headers['X-Amz-Content-Sha256'] ||= 'UNSIGNED-PAYLOAD'
+          end
+        end
+
+        def reset_signature(req)
+          # in case this request is being re-signed
+          req.headers.delete('Authorization')
+          req.headers.delete('X-Amz-Security-Token')
+          req.headers.delete('X-Amz-Date')
+          req.headers.delete('x-Amz-Region-Set')
+        end
+
+        def apply_clock_skew(context, req)
+          if context.config.respond_to?(:clock_skew) &&
+             context.config.clock_skew &&
+             context.config.correct_clock_skew
+
+            endpoint = context.http_request.endpoint
+            skew = context.config.clock_skew.clock_correction(endpoint)
+            if skew.abs.positive?
+              req.headers['X-Amz-Date'] =
+                (Time.now.utc + skew).strftime('%Y%m%dT%H%M%SZ')
+            end
+          end
+        end
+
+      end
+
+      # @api private
+      class NullSigner
+
+        def sign(context)
+        end
+
+        def presign_url(*args)
+        end
+
+        def sign_event(*args)
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/signature_v2.rb

@@ -1,6 +1,9 @@
+# frozen_string_literal: true
+
 module Aws
   module Plugins
     # @api private
+    # Necessary to keep after Endpoints 2.0
     class SignatureV2 < Seahorse::Client::Plugin
 
       option(:v2_signer) do |cfg|

data/lib/aws-sdk-core/plugins/signature_v4.rb

@@ -1,52 +1,50 @@
+# frozen_string_literal: true
+
 require 'aws-sigv4'
 
 module Aws
   module Plugins
     # @api private
+    # Necessary to exist after endpoints 2.0
     class SignatureV4 < Seahorse::Client::Plugin
 
+      V4_AUTH = %w[v4 v4-unsigned-payload v4-unsigned-body]
+
       option(:sigv4_signer) do |cfg|
         SignatureV4.build_signer(cfg)
       end
 
       option(:sigv4_name) do |cfg|
-        cfg.api.metadata['signingName'] || cfg.api.metadata['endpointPrefix']
+        signingName = if cfg.region
+          Aws::Partitions::EndpointProvider.signing_service(
+            cfg.region, cfg.api.metadata['endpointPrefix']
+          )
+        end
+        signingName || cfg.api.metadata['signingName'] || cfg.api.metadata['endpointPrefix']
       end
 
       option(:sigv4_region) do |cfg|
-
-        # The signature version 4 signing region is most
-        # commonly the configured region. There are a few
-        # notable exceptions:
-        #
-        # * Some services have a global endpoint to the entire
-        #   partition. For example, when constructing a route53
-        #   client for a region like "us-west-2", we will
-        #   always use "route53.amazonaws.com". This endpoint
-        #   is actually global to the entire partition,
-        #   and must be signed as "us-east-1".
-        #
-        # * When the region is configured, but it is configured
-        #   to a non region, such as "aws-global". This is similar
-        #   to the previous case. We use the Aws::Partitions::EndpointProvider
-        #   to resolve to the actual signing region.
-        #
-        prefix = cfg.api.metadata['endpointPrefix']
-        if prefix && cfg.endpoint.to_s.match(/#{prefix}\.amazonaws\.com/)
-          'us-east-1'
-        elsif cfg.region
-          Aws::Partitions::EndpointProvider.signing_region(cfg.region, cfg.sigv4_name)
+        if cfg.region
+          if cfg.respond_to?(:sts_regional_endpoints)
+            sts_regional = cfg.sts_regional_endpoints
+          end
+          Aws::Partitions::EndpointProvider.signing_region(
+            cfg.region, cfg.api.metadata['endpointPrefix'], sts_regional
+          )
         end
       end
 
       option(:unsigned_operations) do |cfg|
-        cfg.api.operation_names.inject([]) do |unsigned, operation_name|
-          if cfg.api.operation(operation_name)['authtype'] == 'none' ||
-             cfg.api.operation(operation_name)['authtype'] == 'custom'
-            # Unsign requests that has custom apigateway authorizer as well
-            unsigned << operation_name
-          else
-            unsigned
+        if cfg.api.metadata['signatureVersion'] == 'v4'
+          # select operations where authtype is set and is not v4
+          cfg.api.operation_names.select do |o|
+            cfg.api.operation(o)['authtype'] && !V4_AUTH.include?(cfg.api.operation(o)['authtype'])
+          end
+        else # service is not v4 auth
+          # select all operations where authtype is not v4
+          # (includes operations with no explicit authtype)
+          cfg.api.operation_names.select do |o|
+            !V4_AUTH.include?(cfg.api.operation(o)['authtype'])
           end
         end
       end
@@ -106,6 +104,7 @@ module Aws
           req.headers.delete('Authorization')
           req.headers.delete('X-Amz-Security-Token')
           req.headers.delete('X-Amz-Date')
+          req.headers.delete('x-Amz-Region-Set')
 
           if context.config.respond_to?(:clock_skew) &&
              context.config.clock_skew &&
@@ -142,7 +141,7 @@ module Aws
         def apply_authtype(context)
           if context.operation['authtype'].eql?('v4-unsigned-body') &&
              context.http_request.endpoint.scheme.eql?('https')
-            context.http_request.headers['X-Amz-Content-Sha256'] = 'UNSIGNED-PAYLOAD'
+            context.http_request.headers['X-Amz-Content-Sha256'] ||= 'UNSIGNED-PAYLOAD'
           end
           context
         end

data/lib/aws-sdk-core/plugins/stub_responses.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module Plugins
     # @api private
@@ -6,6 +8,7 @@ module Aws
       option(:stub_responses,
         default: false,
         doc_type: 'Boolean',
+        rbs_type: 'untyped',
         docstring: <<-DOCS)
 Causes the client to return stubbed responses. By default
 fake responses are generated and returned. You can specify
@@ -38,6 +41,8 @@ requests are made, and retries are disabled.
           client.handlers.remove(ClientMetricsPlugin::Handler)
           client.handlers.remove(ClientMetricsSendPlugin::LatencyHandler)
           client.handlers.remove(ClientMetricsSendPlugin::AttemptHandler)
+          client.handlers.remove(Seahorse::Client::Plugins::RequestCallback::OptionHandler)
+          client.handlers.remove(Seahorse::Client::Plugins::RequestCallback::ReadCallbackHandler)
         end
       end
 
@@ -47,7 +52,11 @@ requests are made, and retries are disabled.
           stub = context.client.next_stub(context)
           resp = Seahorse::Client::Response.new(context: context)
           async_mode = context.client.is_a? Seahorse::Client::AsyncBase
-          apply_stub(stub, resp, async_mode)
+          if Hash === stub && stub[:mutex]
+            stub[:mutex].synchronize { apply_stub(stub, resp, async_mode) }
+          else
+            apply_stub(stub, resp, async_mode)
+          end
 
           async_mode ? Seahorse::Client::AsyncResponse.new(
             context: context, stream: context[:input_event_stream_handler].event_emitter.stream, sync_queue: Queue.new) : resp

data/lib/aws-sdk-core/plugins/transfer_encoding.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module Plugins
 

data/lib/aws-sdk-core/plugins/user_agent.rb

@@ -1,8 +1,34 @@
+# frozen_string_literal: true
+
 module Aws
   module Plugins
     # @api private
     class UserAgent < Seahorse::Client::Plugin
+      # @api private
       option(:user_agent_suffix)
+      # @api private
+      option(:user_agent_frameworks, default: [])
+
+      option(
+        :sdk_ua_app_id,
+        doc_type: 'String',
+        docstring: <<-DOCS) do |cfg|
+A unique and opaque application ID that is appended to the
+User-Agent header as app/<sdk_ua_app_id>. It should have a
+maximum length of 50.
+        DOCS
+        app_id = ENV['AWS_SDK_UA_APP_ID']
+        app_id ||= Aws.shared_config.sdk_ua_app_id(profile: cfg.profile)
+        app_id
+      end
+
+      def self.feature(feature, &block)
+        Thread.current[:aws_sdk_core_user_agent_feature] ||= []
+        Thread.current[:aws_sdk_core_user_agent_feature] << "ft/#{feature}"
+        block.call
+      ensure
+        Thread.current[:aws_sdk_core_user_agent_feature].pop
+      end
 
       # @api private
       class Handler < Seahorse::Client::Handler
@@ -12,33 +38,112 @@ module Aws
         end
 
         def set_user_agent(context)
-          ua = "aws-sdk-ruby3/#{CORE_GEM_VERSION}"
+          context.http_request.headers['User-Agent'] = UserAgent.new(context).to_s
+        end
+
+        class UserAgent
+          def initialize(context)
+            @context = context
+          end
 
-          begin
-            ua += " #{RUBY_ENGINE}/#{RUBY_VERSION}"
-          rescue
-            ua += " RUBY_ENGINE_NA/#{RUBY_VERSION}"
+          def to_s
+            ua = "aws-sdk-ruby3/#{CORE_GEM_VERSION}"
+            ua += ' ua/2.0'
+            ua += " #{api_metadata}" if api_metadata
+            ua += " #{os_metadata}"
+            ua += " #{language_metadata}"
+            ua += " #{env_metadata}" if env_metadata
+            ua += " #{config_metadata}" if config_metadata
+            ua += " #{app_id}" if app_id
+            ua += " #{feature_metadata}" if feature_metadata
+            ua += " #{framework_metadata}" if framework_metadata
+            if @context.config.user_agent_suffix
+              ua += " #{@context.config.user_agent_suffix}"
+            end
+            ua.strip
           end
 
-          ua += " #{RUBY_PLATFORM}"
+          private
 
-          if context[:gem_name] && context[:gem_version]
-            ua += " #{context[:gem_name]}/#{context[:gem_version]}"
+          # Used to be gem_name/gem_version
+          def api_metadata
+            service_id = @context.config.api.metadata['serviceId']
+            return unless service_id
+
+            service_id = service_id.gsub(' ', '_').downcase
+            gem_version = @context[:gem_version]
+            "api/#{service_id}##{gem_version}"
           end
 
-          if (execution_env = ENV['AWS_EXECUTION_ENV'])
-            ua += " exec-env/#{execution_env}"
+          # Used to be RUBY_PLATFORM
+          def os_metadata
+            os =
+              case RbConfig::CONFIG['host_os']
+              when /mac|darwin/
+                'macos'
+              when /linux|cygwin/
+                'linux'
+              when /mingw|mswin/
+                'windows'
+              else
+                'other'
+              end
+            metadata = "os/#{os}"
+            local_version = Gem::Platform.local.version
+            metadata += "##{local_version}" if local_version
+            metadata += " md/#{RbConfig::CONFIG['host_cpu']}"
+            metadata
           end
 
-          if context.config.user_agent_suffix
-            ua += " #{context.config.user_agent_suffix}"
+          # Used to be RUBY_ENGINE/RUBY_VERSION
+          def language_metadata
+            "lang/#{RUBY_ENGINE}##{RUBY_ENGINE_VERSION} md/#{RUBY_VERSION}"
           end
 
-          context.http_request.headers['User-Agent'] = ua.strip
+          def env_metadata
+            return unless (execution_env = ENV['AWS_EXECUTION_ENV'])
+
+            "exec-env/#{execution_env}"
+          end
+
+          def config_metadata
+            "cfg/retry-mode##{@context.config.retry_mode}"
+          end
+
+          def app_id
+            return unless (app_id = @context.config.sdk_ua_app_id)
+
+            # Sanitize and only allow these characters
+            app_id = app_id.gsub(/[^!#$%&'*+\-.^_`|~0-9A-Za-z]/, '-')
+            "app/#{app_id}"
+          end
+
+          def feature_metadata
+            return unless Thread.current[:aws_sdk_core_user_agent_feature]
+
+            Thread.current[:aws_sdk_core_user_agent_feature].join(' ')
+          end
+
+          def framework_metadata
+            if (frameworks_cfg = @context.config.user_agent_frameworks).empty?
+              return
+            end
+
+            # Frameworks may be aws-record, aws-sdk-rails, etc.
+            regex = /gems\/(?<name>#{frameworks_cfg.join('|')})-(?<version>\d+\.\d+\.\d+)/.freeze
+            frameworks = {}
+            Kernel.caller.each do |line|
+              match = line.match(regex)
+              next unless match
+
+              frameworks[match[:name]] = match[:version]
+            end
+            frameworks.map { |n, v| "lib/#{n}##{v}" }.join(' ')
+          end
         end
       end
 
-      handler(Handler)
+      handler(Handler, priority: 1)
     end
   end
 end

data/lib/aws-sdk-core/process_credentials.rb

@@ -1,19 +1,16 @@
-require 'open3'
+# frozen_string_literal: true
 
 module Aws
-
   # A credential provider that executes a given process and attempts
-  # to read its stdout to recieve a JSON payload containing the credentials
-  #
-  # Automatically handles refreshing credentials if an Expiration time is
-  # provided in the credentials payload
-  #
-  #     credentials = Aws::ProcessCredentials.new('/usr/bin/credential_proc').credentials
+  # to read its stdout to recieve a JSON payload containing the credentials.
   #
+  #     credentials = Aws::ProcessCredentials.new('/usr/bin/credential_proc')
   #     ec2 = Aws::EC2::Client.new(credentials: credentials)
   #
-  # More documentation on process based credentials can be found here:
-  # https://docs.aws.amazon.com/cli/latest/topic/config-vars.html#sourcing-credentials-from-external-processes
+  # Automatically handles refreshing credentials if an Expiration time is
+  # provided in the credentials payload.
+  #
+  # @see https://docs.aws.amazon.com/cli/latest/topic/config-vars.html#sourcing-credentials-from-external-processes
   class ProcessCredentials
 
     include CredentialProvider
@@ -27,6 +24,7 @@ module Aws
     def initialize(process)
       @process = process
       @credentials = credentials_from_process(@process)
+      @async_refresh = false
 
       super
     end
@@ -34,15 +32,16 @@ module Aws
     private
     def credentials_from_process(proc_invocation)
       begin
-        raw_out, process_status = Open3.capture2(proc_invocation)
+        raw_out = `#{proc_invocation}`
+        process_status = $?
       rescue Errno::ENOENT
         raise Errors::InvalidProcessCredentialsPayload.new("Could not find process #{proc_invocation}")
       end
 
       if process_status.success?
         begin
-          creds_json = JSON.parse(raw_out)
-        rescue JSON::ParserError
+          creds_json = Aws::Json.load(raw_out)
+        rescue Aws::Json::ParseError
           raise Errors::InvalidProcessCredentialsPayload.new("Invalid JSON response")
         end
         payload_version = creds_json['Version']
@@ -72,9 +71,9 @@ module Aws
       @credentials = credentials_from_process(@process)
     end
 
-    def near_expiration?
+    def near_expiration?(expiration_length)
       # are we within 5 minutes of expiration?
-      @expiration && (Time.now.to_i + 5 * 60) > @expiration.to_i
+      @expiration && (Time.now.to_i + expiration_length) > @expiration.to_i
     end
   end
 end

data/lib/aws-sdk-core/query/ec2_param_builder.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 
 module Aws