aws-sdk-core 3.178.0 → 3.233.0

data/lib/aws-sdk-core/instance_profile_credentials.rb

@@ -4,11 +4,23 @@ require 'time'
 require 'net/http'
 
 module Aws
-  # An auto-refreshing credential provider that loads credentials from
-  # EC2 instances.
+  # An auto-refreshing credential provider that loads credentials from EC2 instances.
   #
   #     instance_credentials = Aws::InstanceProfileCredentials.new
   #     ec2 = Aws::EC2::Client.new(credentials: instance_credentials)
+  #
+  # ## Retries
+  # When initialized from the default credential chain, this provider defaults to `0` retries.
+  # Breakdown of retries is as follows:
+  #
+  #  * **Configurable retries** (defaults to `1`): these retries handle errors when communicating
+  #     with the IMDS endpoint. There are two separate retry mechanisms within the provider:
+  #       * Entire token fetch and credential retrieval process
+  #       * Token fetching
+  #  * **JSON parsing retries**: Fixed at 3 attempts to handle cases when IMDS returns malformed JSON
+  #     responses. These retries are separate from configurable retries.
+  #
+  # @see https://docs.aws.amazon.com/sdkref/latest/guide/feature-imds-credentials.html IMDS Credential Provider
   class InstanceProfileCredentials
     include CredentialProvider
     include RefreshingCredentials
@@ -22,10 +34,8 @@ module Aws
     # @api private
     class TokenExpiredError < RuntimeError; end
 
-    # These are the errors we trap when attempting to talk to the
-    # instance metadata service.  Any of these imply the service
-    # is not present, no responding or some other non-recoverable
-    # error.
+    # These are the errors we trap when attempting to talk to the instance metadata service.
+    # Any of these imply the service is not present, no responding or some other non-recoverable error.
     # @api private
     NETWORK_ERRORS = [
       Errno::EHOSTUNREACH,
@@ -46,84 +56,113 @@ module Aws
     METADATA_TOKEN_PATH = '/latest/api/token'.freeze
 
     # @param [Hash] options
-    # @option options [Integer] :retries (1) Number of times to retry
-    #   when retrieving credentials.
-    # @option options [String] :endpoint ('http://169.254.169.254') The IMDS
-    #   endpoint. This option has precedence over the :endpoint_mode.
-    # @option options [String] :endpoint_mode ('IPv4') The endpoint mode for
-    #   the instance metadata service. This is either 'IPv4' ('169.254.169.254')
-    #   or 'IPv6' ('[fd00:ec2::254]').
-    # @option options [String] :ip_address ('169.254.169.254') Deprecated. Use
-    #   :endpoint instead. The IP address for the endpoint.
+    # @option options [Integer] :retries (1) Number of times to retry when retrieving credentials.
+    # @option options [String] :endpoint ('http://169.254.169.254') The IMDS endpoint. This option has precedence
+    #    over the `:endpoint_mode`.
+    # @option options [String] :endpoint_mode ('IPv4') The endpoint mode for the instance metadata service. This is
+    #   either 'IPv4' (`169.254.169.254`) or IPv6' (`[fd00:ec2::254]`).
+    # @option options [Boolean] :disable_imds_v1 (false) Disable the use of the legacy EC2 Metadata Service v1.
+    # @option options [String] :ip_address ('169.254.169.254') Deprecated. Use `:endpoint` instead.
+    #   The IP address for the endpoint.
     # @option options [Integer] :port (80)
     # @option options [Float] :http_open_timeout (1)
     # @option options [Float] :http_read_timeout (1)
-    # @option options [Numeric, Proc] :delay By default, failures are retried
-    #   with exponential back-off, i.e. `sleep(1.2 ** num_failures)`. You can
-    #   pass a number of seconds to sleep between failed attempts, or
-    #   a Proc that accepts the number of failures.
-    # @option options [IO] :http_debug_output (nil) HTTP wire
-    #   traces are sent to this object.  You can specify something
-    #   like $stdout.
-    # @option options [Integer] :token_ttl Time-to-Live in seconds for EC2
-    #   Metadata Token used for fetching Metadata Profile Credentials, defaults
-    #   to 21600 seconds
-    # @option options [Callable] before_refresh Proc called before
-    #   credentials are refreshed. `before_refresh` is called
-    #   with an instance of this object when
-    #   AWS credentials are required and need to be refreshed.
+    # @option options [Numeric, Proc] :delay By default, failures are retried with exponential back-off, i.e.
+    #   `sleep(1.2 ** num_failures)`. You can pass a number of seconds to sleep between failed attempts, or a Proc
+    #   that accepts the number of failures.
+    # @option options [IO] :http_debug_output (nil) HTTP wire traces are sent to this object.
+    #   You can specify something like `$stdout`.
+    # @option options [Integer] :token_ttl Time-to-Live in seconds for EC2 Metadata Token used for fetching
+    #   Metadata Profile Credentials, defaults to 21600 seconds.
+    # @option options [Callable] :before_refresh Proc called before credentials are refreshed. `before_refresh`
+    #   is called with an instance of this object when AWS credentials are required and need to be refreshed.
     def initialize(options = {})
-      @retries = options[:retries] || 1
-      endpoint_mode = resolve_endpoint_mode(options)
-      @endpoint = resolve_endpoint(options, endpoint_mode)
-      @port = options[:port] || 80
+      @backoff = resolve_backoff(options[:backoff])
+      @disable_imds_v1 = resolve_disable_v1(options)
+      @endpoint = resolve_endpoint(options)
       @http_open_timeout = options[:http_open_timeout] || 1
       @http_read_timeout = options[:http_read_timeout] || 1
       @http_debug_output = options[:http_debug_output]
-      @backoff = backoff(options[:backoff])
+      @port = options[:port] || 80
+      @retries = options[:retries] || 1
       @token_ttl = options[:token_ttl] || 21_600
-      @token = nil
-      @no_refresh_until = nil
+
       @async_refresh = false
+      @imds_v1_fallback = false
+      @no_refresh_until = nil
+      @token = nil
+      @metrics = ['CREDENTIALS_IMDS']
       super
     end
 
-    # @return [Integer] Number of times to retry when retrieving credentials
-    #   from the instance metadata service. Defaults to 0 when resolving from
-    #   the default credential chain ({Aws::CredentialProviderChain}).
+    # @return [Boolean0
+    attr_reader :disable_imds_v1
+
+    # @return [Integer]
+    attr_reader :token_ttl
+
+    # @return [Integer]
     attr_reader :retries
 
+    # @return [Proc]
+    attr_reader :backoff
+
+    # @return [String]
+    attr_reader :endpoint
+
+    # @return [Integer]
+    attr_reader :port
+
+    # @return [Integer]
+    attr_reader :http_open_timeout
+
+    # @return [Integer]
+    attr_reader :http_read_timeout
+
+    # @return [IO, nil]
+    attr_reader :http_debug_output
+
     private
 
     def resolve_endpoint_mode(options)
-      value = options[:endpoint_mode]
-      value ||= ENV['AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE']
-      value ||= Aws.shared_config.ec2_metadata_service_endpoint_mode(
-        profile: options[:profile]
-      )
-      value || 'IPv4'
+      options[:endpoint_mode] ||
+        ENV['AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE'] ||
+        Aws.shared_config.ec2_metadata_service_endpoint_mode(profile: options[:profile]) ||
+        'IPv4'
     end
 
-    def resolve_endpoint(options, endpoint_mode)
-      value = options[:endpoint] || options[:ip_address]
-      value ||= ENV['AWS_EC2_METADATA_SERVICE_ENDPOINT']
-      value ||= Aws.shared_config.ec2_metadata_service_endpoint(
-        profile: options[:profile]
-      )
+    def resolve_endpoint(options)
+      if (value = options[:ip_address])
+        warn('The `:ip_address` option is deprecated. Use `:endpoint` instead.')
+        return value
+      end
 
+      value =
+        options[:endpoint] ||
+        ENV['AWS_EC2_METADATA_SERVICE_ENDPOINT'] ||
+        Aws.shared_config.ec2_metadata_service_endpoint(profile: options[:profile]) ||
+        nil
       return value if value
 
+      endpoint_mode = resolve_endpoint_mode(options)
       case endpoint_mode.downcase
       when 'ipv4' then 'http://169.254.169.254'
       when 'ipv6' then 'http://[fd00:ec2::254]'
       else
-        raise ArgumentError,
-              ':endpoint_mode is not valid, expected IPv4 or IPv6, '\
-              "got: #{endpoint_mode}"
+        raise ArgumentError, ":endpoint_mode is not valid, expected IPv4 or IPv6, got: #{endpoint_mode}"
       end
     end
 
-    def backoff(backoff)
+    def resolve_disable_v1(options)
+      value =
+        options[:disable_imds_v1] ||
+        ENV['AWS_EC2_METADATA_V1_DISABLED'] ||
+        Aws.shared_config.ec2_metadata_v1_disabled(profile: options[:profile]) ||
+        'false'
+      Aws::Util.str_2_bool(value.to_s.downcase)
+    end
+
+    def resolve_backoff(backoff)
       case backoff
       when Proc then backoff
       when Numeric then ->(_) { sleep(backoff) }
@@ -137,107 +176,89 @@ module Aws
         return
       end
 
-      # Retry loading credentials up to 3 times is the instance metadata
-      # service is responding but is returning invalid JSON documents
-      # in response to the GET profile credentials call.
-      begin
-        retry_errors([Aws::Json::ParseError, StandardError], max_retries: 3) do
-          c = Aws::Json.load(get_credentials.to_s)
-          if empty_credentials?(@credentials)
-            @credentials = Credentials.new(
-              c['AccessKeyId'],
-              c['SecretAccessKey'],
-              c['Token']
-            )
-            @expiration = c['Expiration'] ? Time.iso8601(c['Expiration']) : nil
-            if @expiration && @expiration < Time.now
-              @no_refresh_until = Time.now + refresh_offset
-              warn_expired_credentials
-            end
-          else
-            #  credentials are already set, update them only if the new ones are not empty
-            if !c['AccessKeyId'] || c['AccessKeyId'].empty?
-              # error getting new credentials
-              @no_refresh_until = Time.now + refresh_offset
-              warn_expired_credentials
-            else
-              @credentials = Credentials.new(
-                c['AccessKeyId'],
-                c['SecretAccessKey'],
-                c['Token']
-              )
-              @expiration = c['Expiration'] ? Time.iso8601(c['Expiration']) : nil
-              if @expiration && @expiration < Time.now
-                @no_refresh_until = Time.now + refresh_offset
-                warn_expired_credentials
-              end
-            end
+      new_creds =
+        begin
+          # Retry loading credentials up to 3 times is the instance metadata
+          # service is responding but is returning invalid JSON documents
+          # in response to the GET profile credentials call.
+          retry_errors([Aws::Json::ParseError], max_retries: 3) do
+            Aws::Json.load(retrieve_credentials.to_s)
           end
-
+        rescue Aws::Json::ParseError
+          raise Aws::Errors::MetadataParserError
         end
-      rescue Aws::Json::ParseError
-        raise Aws::Errors::MetadataParserError
+
+      if @credentials&.set? && empty_credentials?(new_creds)
+        # credentials are already set, but there was an error getting new credentials
+        # so don't update the credentials and use stale ones (static stability)
+        @no_refresh_until = Time.now + rand(300..360)
+        warn_expired_credentials
+      else
+        # credentials are empty or successfully retrieved, update them
+        update_credentials(new_creds)
       end
     end
 
-    def get_credentials
+    def retrieve_credentials
       # Retry loading credentials a configurable number of times if
       # the instance metadata service is not responding.
-      if _metadata_disabled?
-        '{}'
-      else
-        begin
-          retry_errors(NETWORK_ERRORS, max_retries: @retries) do
-            open_connection do |conn|
-              # attempt to fetch token to start secure flow first
-              # and rescue to failover
-              begin
-                retry_errors(NETWORK_ERRORS, max_retries: @retries) do
-                  unless token_set?
-                    created_time = Time.now
-                    token_value, ttl = http_put(
-                      conn, METADATA_TOKEN_PATH, @token_ttl
-                    )
-                    @token = Token.new(token_value, ttl, created_time) if token_value && ttl
-                  end
-                end
-              rescue *NETWORK_ERRORS
-                # token attempt failed, reset token
-                # fallback to non-token mode
-                @token = nil
-              end
-
-              token = @token.value if token_set?
-
-              begin
-                metadata = http_get(conn, METADATA_PATH_BASE, token)
-                profile_name = metadata.lines.first.strip
-                http_get(conn, METADATA_PATH_BASE + profile_name, token)
-              rescue TokenExpiredError
-                # Token has expired, reset it
-                # The next retry should fetch it
-                @token = nil
-                raise Non200Response
-              end
-            end
+      begin
+        retry_errors(NETWORK_ERRORS, max_retries: @retries) do
+          open_connection do |conn|
+            # attempt to fetch token to start secure flow first
+            # and rescue to failover
+            fetch_token(conn) unless @imds_v1_fallback || (@token && !@token.expired?)
+
+            # disable insecure flow if we couldn't get token and imds v1 is disabled
+            raise TokenRetrivalError if @token.nil? && @disable_imds_v1
+
+            fetch_credentials(conn)
           end
-        rescue
-          '{}'
         end
+      rescue StandardError => e
+        warn("Error retrieving instance profile credentials: #{e}")
+        '{}'
       end
     end
 
-    def token_set?
-      @token && !@token.expired?
+    def update_credentials(creds)
+      @credentials = Credentials.new(creds['AccessKeyId'], creds['SecretAccessKey'], creds['Token'])
+      @expiration = creds['Expiration'] ? Time.iso8601(creds['Expiration']) : nil
+      return unless @expiration && @expiration < Time.now
+
+      @no_refresh_until = Time.now + rand(300..360)
+      warn_expired_credentials
     end
 
-    def _metadata_disabled?
-      ENV.fetch('AWS_EC2_METADATA_DISABLED', 'false').downcase == 'true'
+    def fetch_token(conn)
+      created_time = Time.now
+      token_value, ttl = http_put(conn)
+      @token = Token.new(token_value, ttl, created_time) if token_value && ttl
+    rescue *NETWORK_ERRORS
+      # token attempt failed, reset token
+      # fallback to non-token mode
+      @imds_v1_fallback = true
+    end
+
+    def fetch_credentials(conn)
+      metadata = http_get(conn, METADATA_PATH_BASE)
+      profile_name = metadata.lines.first.strip
+      http_get(conn, METADATA_PATH_BASE + profile_name)
+    rescue TokenExpiredError
+      # Token has expired, reset it
+      # The next retry should fetch it
+      @token = nil
+      @imds_v1_fallback = false
+      raise Non200Response
+    end
+
+    def token_set?
+      @token && !@token.expired?
     end
 
     def open_connection
       uri = URI.parse(@endpoint)
-      http = Net::HTTP.new(uri.hostname || @endpoint, @port || uri.port)
+      http = Net::HTTP.new(uri.hostname || @endpoint, uri.port || @port)
       http.open_timeout = @http_open_timeout
       http.read_timeout = @http_read_timeout
       http.set_debug_output(@http_debug_output) if @http_debug_output
@@ -246,9 +267,9 @@ module Aws
     end
 
     # GET request fetch profile and credentials
-    def http_get(connection, path, token = nil)
+    def http_get(connection, path)
       headers = { 'User-Agent' => "aws-sdk-ruby3/#{CORE_GEM_VERSION}" }
-      headers['x-aws-ec2-metadata-token'] = token if token
+      headers['x-aws-ec2-metadata-token'] = @token.value if @token
       response = connection.request(Net::HTTP::Get.new(path, headers))
 
       case response.code.to_i
@@ -262,12 +283,12 @@ module Aws
     end
 
     # PUT request fetch token with ttl
-    def http_put(connection, path, ttl)
+    def http_put(connection)
       headers = {
         'User-Agent' => "aws-sdk-ruby3/#{CORE_GEM_VERSION}",
-        'x-aws-ec2-metadata-token-ttl-seconds' => ttl.to_s
+        'x-aws-ec2-metadata-token-ttl-seconds' => @token_ttl.to_s
       }
-      response = connection.request(Net::HTTP::Put.new(path, headers))
+      response = connection.request(Net::HTTP::Put.new(METADATA_TOKEN_PATH, headers))
       case response.code.to_i
       when 200
         [
@@ -276,8 +297,6 @@ module Aws
         ]
       when 400
         raise TokenRetrivalError
-      when 401
-        raise TokenExpiredError
       else
         raise Non200Response
       end
@@ -298,18 +317,12 @@ module Aws
     end
 
     def warn_expired_credentials
-      warn("Attempting credential expiration extension due to a credential "\
-        "service availability issue. A refresh of these credentials "\
-        "will be attempted again in 5 minutes.")
-    end
-
-    def empty_credentials?(creds)
-      !creds || !creds.access_key_id || creds.access_key_id.empty?
+      warn('Attempting credential expiration extension due to a credential service availability issue. '\
+             'A refresh of these credentials will be attempted again in 5 minutes.')
     end
 
-    # Compute an offset for refresh with jitter
-    def refresh_offset
-      300 + rand(0..60)
+    def empty_credentials?(creds_hash)
+      !creds_hash['AccessKeyId'] || creds_hash['AccessKeyId'].empty?
     end
 
     # @api private

data/lib/aws-sdk-core/json/builder.rb

@@ -8,7 +8,7 @@ module Aws
 
       include Seahorse::Model::Shapes
 
-      def initialize(rules)
+      def initialize(rules, _options = {})
         @rules = rules
       end
 
@@ -20,6 +20,8 @@ module Aws
       private
 
       def structure(ref, values)
+        return nil if values.nil?
+
         shape = ref.shape
         values.each_pair.with_object({}) do |(key, value), data|
           if shape.member?(key) && !value.nil?
@@ -31,11 +33,15 @@ module Aws
       end
 
       def list(ref, values)
+        return nil if values.nil?
+
         member_ref = ref.shape.member
         values.collect { |value| format(member_ref, value) }
       end
 
       def map(ref, values)
+        return nil if values.nil?
+
         value_ref = ref.shape.value
         values.each.with_object({}) do |(key, value), data|
           data[key] = format(value_ref, value)
@@ -49,6 +55,7 @@ module Aws
         when MapShape       then map(ref, value)
         when TimestampShape then timestamp(ref, value)
         when BlobShape      then encode(value)
+        when FloatShape     then Util.serialize_number(value)
         else value
         end
       end

data/lib/aws-sdk-core/json/error_handler.rb

@@ -2,10 +2,9 @@
 
 module Aws
   module Json
-    class ErrorHandler < Xml::ErrorHandler
+    # @api private
+    class ErrorHandler < Aws::ErrorHandler
 
-      # @param [Seahorse::Client::RequestContext] context
-      # @return [Seahorse::Client::Response]
       def call(context)
         @handler.call(context).on(300..599) do |response|
           response.error = error(context)
@@ -19,24 +18,28 @@ module Aws
         json = Json.load(body)
         code = error_code(json, context)
         message = error_message(code, json)
-        data = parse_error_data(context, code)
+        data = parse_error_data(context, body, code)
         [code, message, data]
       rescue Json::ParseError
         [http_status_error_code(context), '', EmptyStructure.new]
       end
 
       def error_code(json, context)
+        # This is not correct per protocol tests. awsQueryError is intended to populate the
+        # error code of the error class. The error class should come from __type. Query and
+        # query compatible services currently have dynamic errors raised from error codes instead
+        # of the modeled error class. However, changing this in this major version would break
+        # existing usage.
         code =
           if aws_query_error?(context)
-            error = context.http_response.headers['x-amzn-query-error'].split(';')[0]
-            remove_prefix(error, context)
+            aws_query_error_code(context)
           else
             json['__type']
           end
         code ||= json['code']
         code ||= context.http_response.headers['x-amzn-errortype']
         if code
-          code.split('#').last
+          code.split('#').last.split(':').first
         else
           http_status_error_code(context)
         end
@@ -47,8 +50,14 @@ module Aws
           context.http_response.headers['x-amzn-query-error']
       end
 
+      def aws_query_error_code(context)
+        query_header = context.http_response.headers['x-amzn-query-error']
+        error, _type = query_header.split(';') # type not supported
+        remove_prefix(error, context)
+      end
+
       def remove_prefix(error_code, context)
-        if prefix = context.config.api.metadata['errorPrefix']
+        if (prefix = context.config.api.metadata['errorPrefix'])
           error_code.sub(/^#{prefix}/, '')
         else
           error_code
@@ -63,22 +72,29 @@ module Aws
         end
       end
 
-      def parse_error_data(context, code)
+      def parse_error_data(context, body, code)
         data = EmptyStructure.new
-        if error_rules = context.operation.errors
+        if (error_rules = context.operation.errors)
           error_rules.each do |rule|
             # match modeled shape name with the type(code) only
             # some type(code) might contains invalid characters
             # such as ':' (efs) etc
             match = rule.shape.name == code.gsub(/[^^a-zA-Z0-9]/, '')
-            if match && rule.shape.members.any?
-              data = Parser.new(rule).parse(context.http_response.body_contents)
-            end
+            next unless match && rule.shape.members.any?
+
+            data = Parser.new(rule).parse(body)
+            # errors support HTTP bindings
+            apply_error_headers(rule, context, data)
           end
         end
         data
       end
 
+      def apply_error_headers(rule, context, data)
+        headers = Aws::Rest::Response::Headers.new(rule)
+        headers.apply(context.http_response, data)
+      end
+
     end
   end
 end

data/lib/aws-sdk-core/json/handler.rb

@@ -12,8 +12,7 @@ module Aws
         build_request(context)
         response = @handler.call(context)
         response.on(200..299) { |resp| parse_response(resp) }
-        response.on(200..599) { |resp| apply_request_id(context) }
-        response
+        response.on(200..599) { |_resp| apply_request_id(context) }
       end
 
       private
@@ -22,6 +21,7 @@ module Aws
         context.http_request.http_method = 'POST'
         context.http_request.headers['Content-Type'] = content_type(context)
         context.http_request.headers['X-Amz-Target'] = target(context)
+        context.http_request.headers['X-Amzn-Query-Mode'] = 'true' if query_compatible?(context)
         context.http_request.body = build_body(context)
       end
 
@@ -38,10 +38,10 @@ module Aws
       end
 
       def parse_body(context)
+        json = context.http_response.body_contents
         if simple_json?(context)
-          Json.load(context.http_response.body_contents)
-        elsif rules = context.operation.output
-          json = context.http_response.body_contents
+          Json.load(json)
+        elsif (rules = context.operation.output)
           if json.is_a?(Array)
             # an array of emitted events
             if json[0].respond_to?(:response)
@@ -59,7 +59,10 @@ module Aws
             end
             resp_struct
           else
-            Parser.new(rules).parse(json == '' ? '{}' : json)
+            Parser.new(
+              rules,
+              query_compatible: query_compatible?(context)
+            ).parse(json)
           end
         else
           EmptyStructure.new
@@ -83,6 +86,10 @@ module Aws
         context.config.simple_json
       end
 
+      def query_compatible?(context)
+        context.config.api.metadata.key?('awsQueryCompatible')
+      end
+
     end
   end
 end

data/lib/aws-sdk-core/json/json_engine.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module Aws
   module Json
-    module JSONEngine
+    module JsonEngine
       class << self
         def load(json)
           JSON.parse(json)

data/lib/aws-sdk-core/json/oj_engine.rb

@@ -1,10 +1,16 @@
 # frozen_string_literal: true
 
+require 'oj'
+
 module Aws
   module Json
     module OjEngine
       # @api private
-      LOAD_OPTIONS = { mode: :compat, symbol_keys: false, empty_string: false }.freeze
+      LOAD_OPTIONS = {
+        mode: :compat,
+        symbol_keys: false,
+        empty_string: false
+      }.freeze
 
       # @api private
       DUMP_OPTIONS = { mode: :compat }.freeze