aws-sdk-core 3.100.0 → 3.191.0

data/lib/aws-sdk-core/instance_profile_credentials.rb

@@ -1,10 +1,15 @@
-require 'json'
+# frozen_string_literal: true
+
 require 'time'
 require 'net/http'
 
 module Aws
+  # An auto-refreshing credential provider that loads credentials from
+  # EC2 instances.
+  #
+  #     instance_credentials = Aws::InstanceProfileCredentials.new
+  #     ec2 = Aws::EC2::Client.new(credentials: instance_credentials)
   class InstanceProfileCredentials
-
     include CredentialProvider
     include RefreshingCredentials
 
@@ -43,7 +48,15 @@ module Aws
     # @param [Hash] options
     # @option options [Integer] :retries (1) Number of times to retry
     #   when retrieving credentials.
-    # @option options [String] :ip_address ('169.254.169.254')
+    # @option options [String] :endpoint ('http://169.254.169.254') The IMDS
+    #   endpoint. This option has precedence over the :endpoint_mode.
+    # @option options [String] :endpoint_mode ('IPv4') The endpoint mode for
+    #   the instance metadata service. This is either 'IPv4' ('169.254.169.254')
+    #   or 'IPv6' ('[fd00:ec2::254]').
+    # @option options [Boolean] :disable_imds_v1 (false) Disable the use of the
+    #  legacy EC2 Metadata Service v1.
+    # @option options [String] :ip_address ('169.254.169.254') Deprecated. Use
+    #   :endpoint instead. The IP address for the endpoint.
     # @option options [Integer] :port (80)
     # @option options [Float] :http_open_timeout (1)
     # @option options [Float] :http_read_timeout (1)
@@ -57,16 +70,26 @@ module Aws
     # @option options [Integer] :token_ttl Time-to-Live in seconds for EC2
     #   Metadata Token used for fetching Metadata Profile Credentials, defaults
     #   to 21600 seconds
+    # @option options [Callable] before_refresh Proc called before
+    #   credentials are refreshed. `before_refresh` is called
+    #   with an instance of this object when
+    #   AWS credentials are required and need to be refreshed.
     def initialize(options = {})
       @retries = options[:retries] || 1
-      @ip_address = options[:ip_address] || '169.254.169.254'
+      endpoint_mode = resolve_endpoint_mode(options)
+      @endpoint = resolve_endpoint(options, endpoint_mode)
       @port = options[:port] || 80
+      @disable_imds_v1 = resolve_disable_v1(options)
+      # Flag for if v2 flow fails, skip future attempts
+      @imds_v1_fallback = false
       @http_open_timeout = options[:http_open_timeout] || 1
       @http_read_timeout = options[:http_read_timeout] || 1
       @http_debug_output = options[:http_debug_output]
       @backoff = backoff(options[:backoff])
       @token_ttl = options[:token_ttl] || 21_600
       @token = nil
+      @no_refresh_until = nil
+      @async_refresh = false
       super
     end
 
@@ -77,6 +100,44 @@ module Aws
 
     private
 
+    def resolve_endpoint_mode(options)
+      value = options[:endpoint_mode]
+      value ||= ENV['AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE']
+      value ||= Aws.shared_config.ec2_metadata_service_endpoint_mode(
+        profile: options[:profile]
+      )
+      value || 'IPv4'
+    end
+
+    def resolve_endpoint(options, endpoint_mode)
+      value = options[:endpoint] || options[:ip_address]
+      value ||= ENV['AWS_EC2_METADATA_SERVICE_ENDPOINT']
+      value ||= Aws.shared_config.ec2_metadata_service_endpoint(
+        profile: options[:profile]
+      )
+
+      return value if value
+
+      case endpoint_mode.downcase
+      when 'ipv4' then 'http://169.254.169.254'
+      when 'ipv6' then 'http://[fd00:ec2::254]'
+      else
+        raise ArgumentError,
+              ':endpoint_mode is not valid, expected IPv4 or IPv6, '\
+              "got: #{endpoint_mode}"
+      end
+    end
+
+    def resolve_disable_v1(options)
+      value = options[:disable_imds_v1]
+      value ||= ENV['AWS_EC2_METADATA_V1_DISABLED']
+      value ||= Aws.shared_config.ec2_metadata_v1_disabled(
+        profile: options[:profile]
+      )
+      value = value.to_s.downcase if value
+      Aws::Util.str_2_bool(value) || false
+    end
+
     def backoff(backoff)
       case backoff
       when Proc then backoff
@@ -86,20 +147,49 @@ module Aws
     end
 
     def refresh
+      if @no_refresh_until && @no_refresh_until > Time.now
+        warn_expired_credentials
+        return
+      end
+
       # Retry loading credentials up to 3 times is the instance metadata
       # service is responding but is returning invalid JSON documents
       # in response to the GET profile credentials call.
       begin
-        retry_errors([JSON::ParserError, StandardError], max_retries: 3) do
-          c = JSON.parse(get_credentials.to_s)
-          @credentials = Credentials.new(
-            c['AccessKeyId'],
-            c['SecretAccessKey'],
-            c['Token']
-          )
-          @expiration = c['Expiration'] ? Time.iso8601(c['Expiration']) : nil
+        retry_errors([Aws::Json::ParseError], max_retries: 3) do
+          c = Aws::Json.load(get_credentials.to_s)
+          if empty_credentials?(@credentials)
+            @credentials = Credentials.new(
+              c['AccessKeyId'],
+              c['SecretAccessKey'],
+              c['Token']
+            )
+            @expiration = c['Expiration'] ? Time.iso8601(c['Expiration']) : nil
+            if @expiration && @expiration < Time.now
+              @no_refresh_until = Time.now + refresh_offset
+              warn_expired_credentials
+            end
+          else
+            #  credentials are already set, update them only if the new ones are not empty
+            if !c['AccessKeyId'] || c['AccessKeyId'].empty?
+              # error getting new credentials
+              @no_refresh_until = Time.now + refresh_offset
+              warn_expired_credentials
+            else
+              @credentials = Credentials.new(
+                c['AccessKeyId'],
+                c['SecretAccessKey'],
+                c['Token']
+              )
+              @expiration = c['Expiration'] ? Time.iso8601(c['Expiration']) : nil
+              if @expiration && @expiration < Time.now
+                @no_refresh_until = Time.now + refresh_offset
+                warn_expired_credentials
+              end
+            end
+          end
         end
-      rescue JSON::ParserError
+      rescue Aws::Json::ParseError
         raise Aws::Errors::MetadataParserError
       end
     end
@@ -115,25 +205,14 @@ module Aws
             open_connection do |conn|
               # attempt to fetch token to start secure flow first
               # and rescue to failover
-              begin
-                retry_errors(NETWORK_ERRORS, max_retries: @retries) do
-                  unless token_set?
-                    token_value, ttl = http_put(
-                      conn, METADATA_TOKEN_PATH, @token_ttl
-                    )
-                    @token = Token.new(token_value, ttl) if token_value && ttl
-                  end
-                end
-              rescue *NETWORK_ERRORS
-                # token attempt failed, reset token
-                # fallback to non-token mode
-                @token = nil
-              end
-
+              fetch_token(conn) unless @imds_v1_fallback
               token = @token.value if token_set?
-              metadata = http_get(conn, METADATA_PATH_BASE, token)
-              profile_name = metadata.lines.first.strip
-              http_get(conn, METADATA_PATH_BASE + profile_name, token)
+
+              # disable insecure flow if we couldn't get token
+              # and imds v1 is disabled
+              raise TokenRetrivalError if token.nil? && @disable_imds_v1
+
+              _get_credentials(conn, token)
             end
           end
         rescue
@@ -142,6 +221,36 @@ module Aws
       end
     end
 
+    def fetch_token(conn)
+      retry_errors(NETWORK_ERRORS, max_retries: @retries) do
+        unless token_set?
+          created_time = Time.now
+          token_value, ttl = http_put(
+            conn, METADATA_TOKEN_PATH, @token_ttl
+          )
+          @token = Token.new(token_value, ttl, created_time) if token_value && ttl
+        end
+      end
+    rescue *NETWORK_ERRORS
+      # token attempt failed, reset token
+      # fallback to non-token mode
+      @token = nil
+      @imds_v1_fallback = true
+    end
+
+    # token is optional - if nil, uses v1 (insecure) flow
+    def _get_credentials(conn, token)
+      metadata = http_get(conn, METADATA_PATH_BASE, token)
+      profile_name = metadata.lines.first.strip
+      http_get(conn, METADATA_PATH_BASE + profile_name, token)
+    rescue TokenExpiredError
+      # Token has expired, reset it
+      # The next retry should fetch it
+      @token = nil
+      @imds_v1_fallback = false
+      raise Non200Response
+    end
+
     def token_set?
       @token && !@token.expired?
     end
@@ -151,7 +260,8 @@ module Aws
     end
 
     def open_connection
-      http = Net::HTTP.new(@ip_address, @port, nil)
+      uri = URI.parse(@endpoint)
+      http = Net::HTTP.new(uri.hostname || @endpoint, @port || uri.port)
       http.open_timeout = @http_open_timeout
       http.read_timeout = @http_read_timeout
       http.set_debug_output(@http_debug_output) if @http_debug_output
@@ -164,9 +274,15 @@ module Aws
       headers = { 'User-Agent' => "aws-sdk-ruby3/#{CORE_GEM_VERSION}" }
       headers['x-aws-ec2-metadata-token'] = token if token
       response = connection.request(Net::HTTP::Get.new(path, headers))
-      raise Non200Response unless response.code.to_i == 200
 
-      response.body
+      case response.code.to_i
+      when 200
+        response.body
+      when 401
+        raise TokenExpiredError
+      else
+        raise Non200Response
+      end
     end
 
     # PUT request fetch token with ttl
@@ -184,8 +300,6 @@ module Aws
         ]
       when 400
         raise TokenRetrivalError
-      when 401
-        raise TokenExpiredError
       else
         raise Non200Response
       end
@@ -205,13 +319,28 @@ module Aws
       end
     end
 
+    def warn_expired_credentials
+      warn("Attempting credential expiration extension due to a credential "\
+        "service availability issue. A refresh of these credentials "\
+        "will be attempted again in 5 minutes.")
+    end
+
+    def empty_credentials?(creds)
+      !creds || !creds.access_key_id || creds.access_key_id.empty?
+    end
+
+    # Compute an offset for refresh with jitter
+    def refresh_offset
+      300 + rand(0..60)
+    end
+
     # @api private
     # Token used to fetch IMDS profile and credentials
     class Token
-      def initialize(value, ttl)
+      def initialize(value, ttl, created_time = Time.now)
         @ttl = ttl
         @value = value
-        @created_time = Time.now
+        @created_time = created_time
       end
 
       # [String] token value

data/lib/aws-sdk-core/json/builder.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 
 module Aws

data/lib/aws-sdk-core/json/error_handler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module Json
     class ErrorHandler < Xml::ErrorHandler
@@ -24,7 +26,13 @@ module Aws
       end
 
       def error_code(json, context)
-        code = json['__type']
+        code =
+          if aws_query_error?(context)
+            error = context.http_response.headers['x-amzn-query-error'].split(';')[0]
+            remove_prefix(error, context)
+          else
+            json['__type']
+          end
         code ||= json['code']
         code ||= context.http_response.headers['x-amzn-errortype']
         if code
@@ -34,6 +42,19 @@ module Aws
         end
       end
 
+      def aws_query_error?(context)
+        context.config.api.metadata['awsQueryCompatible'] &&
+          context.http_response.headers['x-amzn-query-error']
+      end
+
+      def remove_prefix(error_code, context)
+        if prefix = context.config.api.metadata['errorPrefix']
+          error_code.sub(/^#{prefix}/, '')
+        else
+          error_code
+        end
+      end
+
       def error_message(code, json)
         if code == 'RequestEntityTooLarge'
           'Request body must be less than 1 MB'

data/lib/aws-sdk-core/json/handler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module Json
     class Handler < Seahorse::Client::Handler
@@ -57,7 +59,10 @@ module Aws
             end
             resp_struct
           else
-            Parser.new(rules).parse(json == '' ? '{}' : json)
+            Parser.new(
+              rules,
+              query_compatible: query_compatible?(context)
+            ).parse(json == '' ? '{}' : json)
           end
         else
           EmptyStructure.new
@@ -81,6 +86,10 @@ module Aws
         context.config.simple_json
       end
 
+      def query_compatible?(context)
+        context.config.api.metadata.key?('awsQueryCompatible')
+      end
+
     end
   end
 end

data/lib/aws-sdk-core/json/json_engine.rb

@@ -1,15 +1,19 @@
+# frozen_string_literal: true
+
 module Aws
   module Json
-    class OjEngine
-
-      def self.load(json)
-        Oj.load(json)
-      end
+    module JSONEngine
+      class << self
+        def load(json)
+          JSON.parse(json)
+        rescue JSON::ParserError => e
+          raise ParseError.new(e)
+        end
 
-      def self.dump(value)
-        Oj.dump(value)
+        def dump(value)
+          JSON.dump(value)
+        end
       end
-
     end
   end
 end

data/lib/aws-sdk-core/json/oj_engine.rb

@@ -1,15 +1,44 @@
+# frozen_string_literal: true
+
 module Aws
   module Json
-    class JSONEngine
+    module OjEngine
+      # @api private
+      LOAD_OPTIONS = { mode: :compat, symbol_keys: false, empty_string: false }.freeze
 
-      def self.load(json)
-        JSON.load(json)
-      end
+      # @api private
+      DUMP_OPTIONS = { mode: :compat }.freeze
+
+      class << self
+        def load(json)
+          Oj.load(json, LOAD_OPTIONS)
+        rescue *PARSE_ERRORS => e
+          raise ParseError.new(e)
+        end
+
+        def dump(value)
+          Oj.dump(value, DUMP_OPTIONS)
+        end
+
+        private
+
+        # Oj before 1.4.0 does not define Oj::ParseError and instead raises
+        # SyntaxError on failure
+        def detect_oj_parse_errors
+          require 'oj'
 
-      def self.dump(value)
-        JSON.dump(value)
+          if Oj.const_defined?(:ParseError)
+            [Oj::ParseError, EncodingError, JSON::ParserError]
+          else
+            [SyntaxError]
+          end
+        rescue LoadError
+          nil
+        end
       end
 
+      # @api private
+      PARSE_ERRORS = detect_oj_parse_errors
     end
   end
 end

data/lib/aws-sdk-core/json/parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 require 'time'
 
@@ -8,8 +10,9 @@ module Aws
       include Seahorse::Model::Shapes
 
       # @param [Seahorse::Model::ShapeRef] rules
-      def initialize(rules)
+      def initialize(rules, query_compatible: false)
         @rules = rules
+        @query_compatible = query_compatible
       end
 
       # @param [String<JSON>] json
@@ -26,8 +29,32 @@ module Aws
           member_name, member_ref = shape.member_by_location_name(key)
           if member_ref
             target[member_name] = parse_ref(member_ref, value)
+          elsif shape.union && key != '__type'
+            target[:unknown] = { 'name' => key, 'value' => value }
           end
         end
+        # In services that were previously Query/XML, members that were
+        # "flattened" defaulted to empty lists. In JSON, these values are nil,
+        # which is backwards incompatible. To preserve backwards compatibility,
+        # we set a default value of [] for these members.
+        if @query_compatible
+          ref.shape.members.each do |member_name, member_target|
+            next unless target[member_name].nil?
+
+            if flattened_list?(member_target.shape)
+              target[member_name] = []
+            elsif flattened_map?(member_target.shape)
+              target[member_name] = {}
+            end
+          end
+        end
+
+        if shape.union
+          # convert to subclass
+          member_subclass = shape.member_subclass(target.member).new
+          member_subclass[target.member] = target.value
+          target = member_subclass
+        end
         target
       end
 
@@ -69,6 +96,14 @@ module Aws
         value.is_a?(Numeric) ? Time.at(value) : Time.parse(value)
       end
 
+      def flattened_list?(shape)
+        shape.is_a?(ListShape) && shape.flattened
+      end
+
+      def flattened_map?(shape)
+        shape.is_a?(MapShape) && shape.flattened
+      end
+
     end
   end
 end

data/lib/aws-sdk-core/json.rb

@@ -1,8 +1,12 @@
+# frozen_string_literal: true
+
 require 'json'
 require_relative 'json/builder'
 require_relative 'json/error_handler'
 require_relative 'json/handler'
 require_relative 'json/parser'
+require_relative 'json/json_engine'
+require_relative 'json/oj_engine'
 
 module Aws
   # @api private
@@ -18,9 +22,7 @@ module Aws
 
     class << self
       def load(json)
-        ENGINE.load(json, *ENGINE_LOAD_OPTIONS)
-      rescue *ENGINE_ERRORS => e
-        raise ParseError, e
+        ENGINE.load(json)
       end
 
       def load_file(path)
@@ -28,38 +30,20 @@ module Aws
       end
 
       def dump(value)
-        ENGINE.dump(value, *ENGINE_DUMP_OPTIONS)
+        ENGINE.dump(value)
       end
 
       private
 
-      def oj_engine
+      def select_engine
         require 'oj'
-        [
-          Oj,
-          [{ mode: :compat, symbol_keys: false }],
-          [{ mode: :compat }],
-          oj_parse_error
-        ]
+        OjEngine
       rescue LoadError
-        false
-      end
-
-      def json_engine
-        [JSON, [], [], [JSON::ParserError]]
-      end
-
-      def oj_parse_error
-        if Oj.const_defined?('ParseError')
-          [Oj::ParseError, EncodingError, JSON::ParserError]
-        else
-          [SyntaxError]
-        end
+        JSONEngine
       end
     end
 
     # @api private
-    ENGINE, ENGINE_LOAD_OPTIONS, ENGINE_DUMP_OPTIONS, ENGINE_ERRORS =
-      oj_engine || json_engine
+    ENGINE = select_engine
   end
 end

data/lib/aws-sdk-core/log/formatter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'pathname'
 
 module Aws
@@ -24,6 +26,8 @@ module Aws
     #
     # You can put any of these placeholders into you pattern.
     #
+    #   * `:region` - The region configured for the client.
+    #
     #   * `:client_class` - The name of the client class.
     #
     #   * `:operation` - The name of the client request method.
@@ -83,6 +87,9 @@ module Aws
       #   The default list of filtered parameters is documented on the
       #   {ParamFilter} class.
       #
+      # @option options [Boolean] :filter_sensitive_params (true) Set to false
+      #   to disable the sensitive parameter filtering when logging
+      #   `:request_params`.
       def initialize(pattern, options = {})
         @pattern = pattern
         @param_formatter = ParamFormatter.new(options)
@@ -92,12 +99,12 @@ module Aws
       # @return [String]
       attr_reader :pattern
 
-      # Given a resopnse, this will format a log message and return it as a
+      # Given a response, this will format a log message and return it as a
       #   string according to {#pattern}.
       # @param [Seahorse::Client::Response] response
       # @return [String]
       def format(response)
-        pattern.gsub(/:(\w+)/) {|sym| send("_#{sym[1..-1]}", response) }
+        pattern.gsub(/:(\w+)/) { |sym| send("_#{sym[1..-1]}", response) }
       end
 
       # @api private
@@ -111,6 +118,10 @@ module Aws
 
       private
 
+      def _region(response)
+        response.context.config.region
+      end
+
       def _client_class(response)
         response.context.client.class.name
       end
@@ -121,7 +132,8 @@ module Aws
 
       def _request_params(response)
         params = response.context.params
-        @param_formatter.summarize(@param_filter.filter(params))
+        type = response.context.operation.input.shape.struct_class
+        @param_formatter.summarize(@param_filter.filter(params, type))
       end
 
       def _time(response)

data/lib/aws-sdk-core/log/handler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Seahorse
   module Client
     module Logging