aws-sdk-core 3.54.2 → 3.126.0

data/lib/aws-sdk-core/instance_profile_credentials.rb

@@ -1,16 +1,22 @@
-require 'json'
+# frozen_string_literal: true
+
 require 'time'
 require 'net/http'
 
 module Aws
   class InstanceProfileCredentials
-
     include CredentialProvider
     include RefreshingCredentials
 
     # @api private
     class Non200Response < RuntimeError; end
 
+    # @api private
+    class TokenRetrivalError < RuntimeError; end
+
+    # @api private
+    class TokenExpiredError < RuntimeError; end
+
     # These are the errors we trap when attempting to talk to the
     # instance metadata service.  Any of these imply the service
     # is not present, no responding or some other non-recoverable
@@ -23,16 +29,30 @@ module Aws
       Errno::ENETUNREACH,
       SocketError,
       Timeout::Error,
-      Non200Response,
-    ]
+      Non200Response
+    ].freeze
+
+    # Path base for GET request for profile and credentials
+    # @api private
+    METADATA_PATH_BASE = '/latest/meta-data/iam/security-credentials/'.freeze
+
+    # Path for PUT request for token
+    # @api private
+    METADATA_TOKEN_PATH = '/latest/api/token'.freeze
 
     # @param [Hash] options
-    # @option options [Integer] :retries (5) Number of times to retry
+    # @option options [Integer] :retries (1) Number of times to retry
     #   when retrieving credentials.
-    # @option options [String] :ip_address ('169.254.169.254')
+    # @option options [String] :endpoint ('http://169.254.169.254') The IMDS
+    #   endpoint. This option has precedence over the :endpoint_mode.
+    # @option options [String] :endpoint_mode ('IPv4') The endpoint mode for
+    #   the instance metadata service. This is either 'IPv4' ('169.254.169.254')
+    #   or 'IPv6' ('[fd00:ec2::254]').
+    # @option options [String] :ip_address ('169.254.169.254') Deprecated. Use
+    #   :endpoint instead. The IP address for the endpoint.
     # @option options [Integer] :port (80)
-    # @option options [Float] :http_open_timeout (5)
-    # @option options [Float] :http_read_timeout (5)
+    # @option options [Float] :http_open_timeout (1)
+    # @option options [Float] :http_read_timeout (1)
     # @option options [Numeric, Proc] :delay By default, failures are retried
     #   with exponential back-off, i.e. `sleep(1.2 ** num_failures)`. You can
     #   pass a number of seconds to sleep between failed attempts, or
@@ -40,28 +60,63 @@ module Aws
     # @option options [IO] :http_debug_output (nil) HTTP wire
     #   traces are sent to this object.  You can specify something
     #   like $stdout.
-    def initialize options = {}
-      @retries = options[:retries] || 5
-      @ip_address = options[:ip_address] || '169.254.169.254'
+    # @option options [Integer] :token_ttl Time-to-Live in seconds for EC2
+    #   Metadata Token used for fetching Metadata Profile Credentials, defaults
+    #   to 21600 seconds
+    def initialize(options = {})
+      @retries = options[:retries] || 1
+      endpoint_mode = resolve_endpoint_mode(options)
+      @endpoint = resolve_endpoint(options, endpoint_mode)
       @port = options[:port] || 80
-      @http_open_timeout = options[:http_open_timeout] || 5
-      @http_read_timeout = options[:http_read_timeout] || 5
+      @http_open_timeout = options[:http_open_timeout] || 1
+      @http_read_timeout = options[:http_read_timeout] || 1
       @http_debug_output = options[:http_debug_output]
       @backoff = backoff(options[:backoff])
+      @token_ttl = options[:token_ttl] || 21_600
+      @token = nil
       super
     end
 
-    # @return [Integer] The number of times to retry failed attempts to
-    #   fetch credentials from the instance metadata service. Defaults to 0.
+    # @return [Integer] Number of times to retry when retrieving credentials
+    #   from the instance metadata service. Defaults to 0 when resolving from
+    #   the default credential chain ({Aws::CredentialProviderChain}).
     attr_reader :retries
 
     private
 
+    def resolve_endpoint_mode(options)
+      value = options[:endpoint_mode]
+      value ||= ENV['AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE']
+      value ||= Aws.shared_config.ec2_metadata_service_endpoint_mode(
+        profile: options[:profile]
+      )
+      value || 'IPv4'
+    end
+
+    def resolve_endpoint(options, endpoint_mode)
+      value = options[:endpoint] || options[:ip_address]
+      value ||= ENV['AWS_EC2_METADATA_SERVICE_ENDPOINT']
+      value ||= Aws.shared_config.ec2_metadata_service_endpoint(
+        profile: options[:profile]
+      )
+
+      return value if value
+
+      case endpoint_mode.downcase
+      when 'ipv4' then 'http://169.254.169.254'
+      when 'ipv6' then 'http://[fd00:ec2::254]'
+      else
+        raise ArgumentError,
+              ':endpoint_mode is not valid, expected IPv4 or IPv6, '\
+              "got: #{endpoint_mode}"
+      end
+    end
+
     def backoff(backoff)
       case backoff
       when Proc then backoff
-      when Numeric then lambda { |_| sleep(backoff) }
-      else lambda { |num_failures| Kernel.sleep(1.2 ** num_failures) }
+      when Numeric then ->(_) { sleep(backoff) }
+      else ->(num_failures) { Kernel.sleep(1.2**num_failures) }
       end
     end
 
@@ -70,8 +125,8 @@ module Aws
       # service is responding but is returning invalid JSON documents
       # in response to the GET profile credentials call.
       begin
-        retry_errors([JSON::ParserError, StandardError], max_retries: 3) do
-          c = JSON.parse(get_credentials.to_s)
+        retry_errors([Aws::Json::ParseError, StandardError], max_retries: 3) do
+          c = Aws::Json.load(get_credentials.to_s)
           @credentials = Credentials.new(
             c['AccessKeyId'],
             c['SecretAccessKey'],
@@ -79,8 +134,8 @@ module Aws
           )
           @expiration = c['Expiration'] ? Time.iso8601(c['Expiration']) : nil
         end
-      rescue JSON::ParserError
-        raise Aws::Errors::MetadataParserError.new
+      rescue Aws::Json::ParseError
+        raise Aws::Errors::MetadataParserError
       end
     end
 
@@ -93,9 +148,27 @@ module Aws
         begin
           retry_errors(NETWORK_ERRORS, max_retries: @retries) do
             open_connection do |conn|
-              path = '/latest/meta-data/iam/security-credentials/'
-              profile_name = http_get(conn, path).lines.first.strip
-              http_get(conn, path + profile_name)
+              # attempt to fetch token to start secure flow first
+              # and rescue to failover
+              begin
+                retry_errors(NETWORK_ERRORS, max_retries: @retries) do
+                  unless token_set?
+                    token_value, ttl = http_put(
+                      conn, METADATA_TOKEN_PATH, @token_ttl
+                    )
+                    @token = Token.new(token_value, ttl) if token_value && ttl
+                  end
+                end
+              rescue *NETWORK_ERRORS
+                # token attempt failed, reset token
+                # fallback to non-token mode
+                @token = nil
+              end
+
+              token = @token.value if token_set?
+              metadata = http_get(conn, METADATA_PATH_BASE, token)
+              profile_name = metadata.lines.first.strip
+              http_get(conn, METADATA_PATH_BASE + profile_name, token)
             end
           end
         rescue
@@ -104,13 +177,17 @@ module Aws
       end
     end
 
+    def token_set?
+      @token && !@token.expired?
+    end
+
     def _metadata_disabled?
-      flag = ENV["AWS_EC2_METADATA_DISABLED"]
-      !flag.nil? && flag.downcase == "true"
+      ENV.fetch('AWS_EC2_METADATA_DISABLED', 'false').downcase == 'true'
     end
 
     def open_connection
-      http = Net::HTTP.new(@ip_address, @port, nil)
+      uri = URI.parse(@endpoint)
+      http = Net::HTTP.new(uri.hostname || @endpoint, @port || uri.port)
       http.open_timeout = @http_open_timeout
       http.read_timeout = @http_read_timeout
       http.set_debug_output(@http_debug_output) if @http_debug_output
@@ -118,30 +195,67 @@ module Aws
       yield(http).tap { http.finish }
     end
 
-    def http_get(connection, path)
-      response = connection.request(Net::HTTP::Get.new(path, {"User-Agent" => "aws-sdk-ruby3/#{CORE_GEM_VERSION}"}))
-      if response.code.to_i == 200
-        response.body
+    # GET request fetch profile and credentials
+    def http_get(connection, path, token = nil)
+      headers = { 'User-Agent' => "aws-sdk-ruby3/#{CORE_GEM_VERSION}" }
+      headers['x-aws-ec2-metadata-token'] = token if token
+      response = connection.request(Net::HTTP::Get.new(path, headers))
+      raise Non200Response unless response.code.to_i == 200
+
+      response.body
+    end
+
+    # PUT request fetch token with ttl
+    def http_put(connection, path, ttl)
+      headers = {
+        'User-Agent' => "aws-sdk-ruby3/#{CORE_GEM_VERSION}",
+        'x-aws-ec2-metadata-token-ttl-seconds' => ttl.to_s
+      }
+      response = connection.request(Net::HTTP::Put.new(path, headers))
+      case response.code.to_i
+      when 200
+        [
+          response.body,
+          response.header['x-aws-ec2-metadata-token-ttl-seconds'].to_i
+        ]
+      when 400
+        raise TokenRetrivalError
+      when 401
+        raise TokenExpiredError
       else
         raise Non200Response
       end
     end
 
-    def retry_errors(error_classes, options = {}, &block)
+    def retry_errors(error_classes, options = {}, &_block)
       max_retries = options[:max_retries]
       retries = 0
       begin
         yield
       rescue *error_classes
-        if retries < max_retries
-          @backoff.call(retries)
-          retries += 1
-          retry
-        else
-          raise
-        end
+        raise unless retries < max_retries
+
+        @backoff.call(retries)
+        retries += 1
+        retry
       end
     end
 
+    # @api private
+    # Token used to fetch IMDS profile and credentials
+    class Token
+      def initialize(value, ttl)
+        @ttl = ttl
+        @value = value
+        @created_time = Time.now
+      end
+
+      # [String] token value
+      attr_reader :value
+
+      def expired?
+        Time.now - @created_time > @ttl
+      end
+    end
   end
 end

data/lib/aws-sdk-core/json/builder.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 
 module Aws

data/lib/aws-sdk-core/json/error_handler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module Json
     class ErrorHandler < Xml::ErrorHandler

data/lib/aws-sdk-core/json/handler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module Json
     class Handler < Seahorse::Client::Handler

data/lib/aws-sdk-core/json/json_engine.rb

@@ -1,15 +1,19 @@
+# frozen_string_literal: true
+
 module Aws
   module Json
-    class OjEngine
-
-      def self.load(json)
-        Oj.load(json)
-      end
+    module JSONEngine
+      class << self
+        def load(json)
+          JSON.parse(json)
+        rescue JSON::ParserError => e
+          raise ParseError.new(e)
+        end
 
-      def self.dump(value)
-        Oj.dump(value)
+        def dump(value)
+          JSON.dump(value)
+        end
       end
-
     end
   end
 end

data/lib/aws-sdk-core/json/oj_engine.rb

@@ -1,15 +1,44 @@
+# frozen_string_literal: true
+
 module Aws
   module Json
-    class JSONEngine
+    module OjEngine
+      # @api private
+      LOAD_OPTIONS = { mode: :compat, symbol_keys: false, empty_string: false }.freeze
 
-      def self.load(json)
-        JSON.load(json)
-      end
+      # @api private
+      DUMP_OPTIONS = { mode: :compat }.freeze
+
+      class << self
+        def load(json)
+          Oj.load(json, LOAD_OPTIONS)
+        rescue *PARSE_ERRORS => e
+          raise ParseError.new(e)
+        end
+
+        def dump(value)
+          Oj.dump(value, DUMP_OPTIONS)
+        end
+
+        private
+
+        # Oj before 1.4.0 does not define Oj::ParseError and instead raises
+        # SyntaxError on failure
+        def detect_oj_parse_errors
+          require 'oj'
 
-      def self.dump(value)
-        JSON.dump(value)
+          if Oj.const_defined?(:ParseError)
+            [Oj::ParseError, EncodingError, JSON::ParserError]
+          else
+            [SyntaxError]
+          end
+        rescue LoadError
+          nil
+        end
       end
 
+      # @api private
+      PARSE_ERRORS = detect_oj_parse_errors
     end
   end
 end

data/lib/aws-sdk-core/json/parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 require 'time'
 
@@ -26,8 +28,16 @@ module Aws
           member_name, member_ref = shape.member_by_location_name(key)
           if member_ref
             target[member_name] = parse_ref(member_ref, value)
+          elsif shape.union
+            target[:unknown] = { 'name' => key, 'value' => value }
           end
         end
+        if shape.union
+          # convert to subclass
+          member_subclass = shape.member_subclass(target.member).new
+          member_subclass[target.member] = target.value
+          target = member_subclass
+        end
         target
       end
 

data/lib/aws-sdk-core/json.rb

@@ -1,66 +1,49 @@
+# frozen_string_literal: true
+
 require 'json'
 require_relative 'json/builder'
 require_relative 'json/error_handler'
 require_relative 'json/handler'
 require_relative 'json/parser'
+require_relative 'json/json_engine'
+require_relative 'json/oj_engine'
 
 module Aws
   # @api private
   module Json
-
     class ParseError < StandardError
-
       def initialize(error)
         @error = error
         super(error.message)
       end
 
       attr_reader :error
-
     end
 
     class << self
-
       def load(json)
-        ENGINE.load(json, *ENGINE_LOAD_OPTIONS)
-      rescue ENGINE_ERROR => e
-        raise ParseError.new(e)
+        ENGINE.load(json)
       end
 
       def load_file(path)
-        self.load(File.open(path, 'r', encoding: 'UTF-8') { |f| f.read })
+        load(File.open(path, 'r', encoding: 'UTF-8', &:read))
       end
 
       def dump(value)
-        ENGINE.dump(value, *ENGINE_DUMP_OPTIONS)
+        ENGINE.dump(value)
       end
 
       private
 
-      def oj_engine
+      def select_engine
         require 'oj'
-        [Oj, [{mode: :compat, symbol_keys: false}], [{ mode: :compat }], oj_parse_error]
+        OjEngine
       rescue LoadError
-        false
+        JSONEngine
       end
-
-      def json_engine
-        [JSON, [], [], JSON::ParserError]
-      end
-
-      def oj_parse_error
-        if Oj.const_defined?('ParseError')
-          Oj::ParseError
-        else
-          SyntaxError
-        end
-      end
-
     end
 
     # @api private
-    ENGINE, ENGINE_LOAD_OPTIONS, ENGINE_DUMP_OPTIONS, ENGINE_ERROR =
-      oj_engine || json_engine
-
+    ENGINE = select_engine
   end
 end

data/lib/aws-sdk-core/log/formatter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'pathname'
 
 module Aws
@@ -83,6 +85,9 @@ module Aws
       #   The default list of filtered parameters is documented on the
       #   {ParamFilter} class.
       #
+      # @option options [Boolean] :filter_sensitive_params (true) Set to false
+      #   to disable the sensitive parameter filtering when logging
+      #   `:request_params`.
       def initialize(pattern, options = {})
         @pattern = pattern
         @param_formatter = ParamFormatter.new(options)
@@ -92,12 +97,12 @@ module Aws
       # @return [String]
       attr_reader :pattern
 
-      # Given a resopnse, this will format a log message and return it as a
+      # Given a response, this will format a log message and return it as a
       #   string according to {#pattern}.
       # @param [Seahorse::Client::Response] response
       # @return [String]
       def format(response)
-        pattern.gsub(/:(\w+)/) {|sym| send("_#{sym[1..-1]}", response) }
+        pattern.gsub(/:(\w+)/) { |sym| send("_#{sym[1..-1]}", response) }
       end
 
       # @api private
@@ -121,7 +126,8 @@ module Aws
 
       def _request_params(response)
         params = response.context.params
-        @param_formatter.summarize(@param_filter.filter(params))
+        type = response.context.operation.input.shape.struct_class
+        @param_formatter.summarize(@param_filter.filter(params, type))
       end
 
       def _time(response)
@@ -171,7 +177,13 @@ module Aws
       end
 
       def _http_response_body(response)
-        @param_formatter.summarize(response.context.http_response.body_contents)
+        if response.context.http_response.body.respond_to?(:rewind)
+          @param_formatter.summarize(
+            response.context.http_response.body_contents
+          )
+        else
+          ''
+        end
       end
 
       def _error_class(response)

data/lib/aws-sdk-core/log/handler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Seahorse
   module Client
     module Logging

data/lib/aws-sdk-core/log/param_filter.rb

@@ -1,43 +1,68 @@
+# frozen_string_literal: true
+
 require 'pathname'
 require 'set'
 
 module Aws
   module Log
     class ParamFilter
-
+      # DEPRECATED - This must exist for backwards compatibility. Sensitive
+      # members are now computed for each request/response type. This can be
+      # removed in a new major version. This list is no longer updated.
+      #
       # A managed list of sensitive parameters that should be filtered from
       # logs. This is updated automatically as part of each release. See the
-      # `tasks/sensitive.rake` for more information.
+      # `tasks/update-sensitive-params.rake` for more information.
       #
       # @api private
       # begin
-      SENSITIVE = [:access_token, :account_name, :account_password, :address, :admin_contact, :admin_password, :artifact_credentials, :auth_code, :authentication_token, :authorization_result, :backup_plan_tags, :backup_vault_tags, :base_32_string_seed, :body, :bot_configuration, :bot_email, :cause, :client_id, :client_secret, :configuration, :copy_source_sse_customer_key, :credentials, :current_password, :custom_attributes, :db_password, :default_phone_number, :definition, :description, :display_name, :e164_phone_number, :email, :email_address, :email_message, :embed_url, :error, :feedback_token, :file, :first_name, :id, :id_token, :input, :input_text, :key_id, :key_store_password, :kms_key_id, :kms_master_key_id, :lambda_function_arn, :last_name, :local_console_password, :master_account_email, :master_user_password, :message, :name, :new_password, :next_password, :notes, :old_password, :outbound_events_https_endpoint, :output, :owner_information, :parameters, :passphrase, :password, :payload, :phone_number, :plaintext, :previous_password, :primary_email, :primary_provisioned_number, :private_key, :proposed_password, :public_key, :qr_code_png, :query, :recovery_point_tags, :refresh_token, :registrant_contact, :request_attributes, :search_query, :secret_access_key, :secret_binary, :secret_code, :secret_hash, :secret_string, :secret_to_authenticate_initiator, :secret_to_authenticate_target, :security_token, :service_password, :session_attributes, :share_notes, :shared_secret, :slots, :sse_customer_key, :ssekms_key_id, :status_message, :tag_key_list, :tags, :task_parameters, :tech_contact, :temporary_password, :text, :token, :trust_password, :upload_credentials, :upload_url, :user_email, :user_name, :username, :value, :values, :variables, :zip_file]
+      SENSITIVE = [:access_token, :account_name, :account_password, :address, :admin_contact, :admin_password, :alexa_for_business_room_arn, :artifact_credentials, :auth_code, :auth_parameters, :authentication_token, :authorization_result, :backup_plan_tags, :backup_vault_tags, :base_32_string_seed, :basic_auth_credentials, :block, :block_address, :block_data, :blocks, :body, :bot_configuration, :bot_email, :calling_name, :cause, :client_id, :client_request_token, :client_secret, :comment, :configuration, :content, :copy_source_sse_customer_key, :credentials, :current_password, :custom_attributes, :custom_private_key, :db_password, :default_phone_number, :definition, :description, :destination_access_token, :digest_tip_address, :display_name, :domain_signing_private_key, :e164_phone_number, :email, :email_address, :email_message, :embed_url, :emergency_phone_number, :error, :external_meeting_id, :external_model_endpoint_data_blobs, :external_user_id, :fall_back_phone_number, :feedback_token, :file, :filter_expression, :first_name, :full_name, :host_key, :id, :id_token, :input, :input_text, :ion_text, :join_token, :key, :key_id, :key_material, :key_store_password, :kms_key_id, :kms_master_key_id, :lambda_function_arn, :last_name, :local_console_password, :master_account_email, :master_user_name, :master_user_password, :meeting_host_id, :message, :metadata, :name, :new_password, :next_password, :notes, :number, :oauth_token, :old_password, :outbound_events_https_endpoint, :output, :owner_information, :parameters, :passphrase, :password, :payload, :phone_number, :plaintext, :previous_password, :primary_email, :primary_provisioned_number, :private_key, :private_key_plaintext, :proof, :proposed_password, :proxy_phone_number, :public_key, :qr_code_png, :query, :random_password, :recovery_point_tags, :refresh_token, :registrant_contact, :request_attributes, :resource_arn, :restore_metadata, :revision, :saml_assertion, :search_query, :secret_access_key, :secret_binary, :secret_code, :secret_hash, :secret_string, :secret_to_authenticate_initiator, :secret_to_authenticate_target, :security_token, :service_password, :session_attributes, :session_token, :share_notes, :shared_secret, :slots, :sns_topic_arn, :source_access_token, :sqs_queue_arn, :sse_customer_key, :ssekms_encryption_context, :ssekms_key_id, :status_message, :tag_key_list, :tags, :target_address, :task_parameters, :tech_contact, :temporary_password, :test_phone_number, :text, :token, :trust_password, :type, :upload_credentials, :upload_url, :uri, :user_data, :user_email, :user_name, :user_password, :username, :value, :values, :variables, :vpn_psk, :web_identity_token, :zip_file]
       # end
 
       def initialize(options = {})
-        @filters = Set.new(SENSITIVE + Array(options[:filter]))
+        @enabled = options[:filter_sensitive_params] != false
+        @additional_filters = options[:filter] || []
       end
 
-      def filter(value)
-        case value
-        when Struct, Hash then filter_hash(value)
-        when Array then filter_array(value)
-        else value
+      def filter(values, type)
+        case values
+        when Struct then filter_struct(values, type)
+        when Hash then filter_hash(values, type)
+        when Array then filter_array(values, type)
+        else values
         end
       end
 
       private
 
-      def filter_hash(values)
+      def filter_struct(values, type)
+        if values.class.include? Aws::Structure::Union
+          values = { values.member => values.value }
+        end
+        filter_hash(values, type)
+      end
+
+      def filter_hash(values, type)
+        if type.const_defined?('SENSITIVE')
+          filters = type::SENSITIVE + @additional_filters
+        else
+          # Support backwards compatibility (new core + old service)
+          filters = SENSITIVE + @additional_filters
+        end
+
         filtered = {}
         values.each_pair do |key, value|
-          filtered[key] = @filters.include?(key) ? '[FILTERED]' : filter(value)
+          filtered[key] = if @enabled && filters.include?(key)
+            '[FILTERED]'
+          else
+            filter(value, type)
+          end
         end
         filtered
       end
 
-      def filter_array(values)
-        values.map { |value| filter(value) }
+      def filter_array(values, type)
+        values.map { |value| filter(value, type) }
       end
 
     end

data/lib/aws-sdk-core/log/param_formatter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'pathname'
 
 module Aws