aws-sdk-core 3.46.0 → 3.94.0

data/lib/aws-sdk-core/json.rb

@@ -7,28 +7,24 @@ require_relative 'json/parser'
 module Aws
   # @api private
   module Json
-
     class ParseError < StandardError
-
       def initialize(error)
         @error = error
         super(error.message)
       end
 
       attr_reader :error
-
     end
 
     class << self
-
       def load(json)
         ENGINE.load(json, *ENGINE_LOAD_OPTIONS)
-      rescue ENGINE_ERROR => e
-        raise ParseError.new(e)
+      rescue *ENGINE_ERRORS => e
+        raise ParseError, e
       end
 
       def load_file(path)
-        self.load(File.open(path, 'r', encoding: 'UTF-8') { |f| f.read })
+        load(File.open(path, 'r', encoding: 'UTF-8', &:read))
       end
 
       def dump(value)
@@ -39,28 +35,31 @@ module Aws
 
       def oj_engine
         require 'oj'
-        [Oj, [{mode: :compat, symbol_keys: false}], [{ mode: :compat }], oj_parse_error]
+        [
+          Oj,
+          [{ mode: :compat, symbol_keys: false }],
+          [{ mode: :compat }],
+          oj_parse_error
+        ]
       rescue LoadError
         false
       end
 
       def json_engine
-        [JSON, [], [], JSON::ParserError]
+        [JSON, [], [], [JSON::ParserError]]
       end
 
       def oj_parse_error
         if Oj.const_defined?('ParseError')
-          Oj::ParseError
+          [Oj::ParseError, EncodingError, JSON::ParserError]
         else
-          SyntaxError
+          [SyntaxError]
         end
       end
-
     end
 
     # @api private
-    ENGINE, ENGINE_LOAD_OPTIONS, ENGINE_DUMP_OPTIONS, ENGINE_ERROR =
+    ENGINE, ENGINE_LOAD_OPTIONS, ENGINE_DUMP_OPTIONS, ENGINE_ERRORS =
       oj_engine || json_engine
-
   end
 end

data/lib/aws-sdk-core/json/error_handler.rb

@@ -17,9 +17,10 @@ module Aws
         json = Json.load(body)
         code = error_code(json, context)
         message = error_message(code, json)
-        [code, message]
+        data = parse_error_data(context, code)
+        [code, message, data]
       rescue Json::ParseError
-        [http_status_error_code(context), '']
+        [http_status_error_code(context), '', EmptyStructure.new]
       end
 
       def error_code(json, context)
@@ -41,6 +42,22 @@ module Aws
         end
       end
 
+      def parse_error_data(context, code)
+        data = EmptyStructure.new
+        if error_rules = context.operation.errors
+          error_rules.each do |rule|
+            # match modeled shape name with the type(code) only
+            # some type(code) might contains invalid characters
+            # such as ':' (efs) etc
+            match = rule.shape.name == code.gsub(/[^^a-zA-Z0-9]/, '')
+            if match && rule.shape.members.any?
+              data = Parser.new(rule).parse(context.http_response.body_contents)
+            end
+          end
+        end
+        data
+      end
+
     end
   end
 end

data/lib/aws-sdk-core/json/handler.rb

@@ -40,7 +40,25 @@ module Aws
           Json.load(context.http_response.body_contents)
         elsif rules = context.operation.output
           json = context.http_response.body_contents
-          Parser.new(rules).parse(json == '' ? '{}' : json)
+          if json.is_a?(Array)
+            # an array of emitted events
+            if json[0].respond_to?(:response)
+              # initial response exists
+              # it must be the first event arrived
+              resp_struct = json.shift.response
+            else
+              resp_struct = context.operation.output.shape.struct_class.new
+            end
+
+            rules.shape.members.each do |name, ref|
+              if ref.eventstream
+                resp_struct.send("#{name}=", json.to_enum)
+              end
+            end
+            resp_struct
+          else
+            Parser.new(rules).parse(json == '' ? '{}' : json)
+          end
         else
           EmptyStructure.new
         end

data/lib/aws-sdk-core/log/formatter.rb

@@ -171,7 +171,13 @@ module Aws
       end
 
       def _http_response_body(response)
-        @param_formatter.summarize(response.context.http_response.body_contents)
+        if response.context.http_response.body.respond_to?(:rewind)
+          @param_formatter.summarize(
+            response.context.http_response.body_contents
+          )
+        else
+          ''
+        end
       end
 
       def _error_class(response)

data/lib/aws-sdk-core/log/param_filter.rb

@@ -7,11 +7,11 @@ module Aws
 
       # A managed list of sensitive parameters that should be filtered from
       # logs. This is updated automatically as part of each release. See the
-      # `tasks/sensitive.rake` for more information.
+      # `tasks/update-sensitive-params.rake` for more information.
       #
       # @api private
       # begin
-      SENSITIVE = [:access_token, :account_name, :account_password, :address, :admin_contact, :artifact_credentials, :auth_code, :authentication_token, :authorization_result, :backup_plan_tags, :backup_vault_tags, :base_32_string_seed, :body, :bot_configuration, :cause, :client_id, :client_secret, :configuration, :copy_source_sse_customer_key, :credentials, :custom_attributes, :db_password, :definition, :description, :display_name, :email, :email_address, :email_message, :embed_url, :error, :feedback_token, :file, :first_name, :id, :id_token, :input, :input_text, :key_id, :key_store_password, :kms_key_id, :kms_master_key_id, :last_name, :local_console_password, :master_account_email, :master_user_password, :message, :name, :new_password, :notes, :old_password, :output, :owner_information, :parameters, :passphrase, :password, :payload, :phone_number, :plaintext, :previous_password, :primary_email, :private_key, :proposed_password, :public_key, :qr_code_png, :query, :recovery_point_tags, :refresh_token, :registrant_contact, :request_attributes, :search_query, :secret_access_key, :secret_binary, :secret_code, :secret_hash, :secret_string, :service_password, :session_attributes, :share_notes, :shared_secret, :slots, :sse_customer_key, :ssekms_key_id, :status_message, :tag_key_list, :tags, :task_parameters, :tech_contact, :temporary_password, :text, :token, :trust_password, :upload_credentials, :upload_url, :user_email, :user_name, :username, :value, :values, :variables, :zip_file]
+      SENSITIVE = [:access_token, :account_name, :account_password, :address, :admin_contact, :admin_password, :alexa_for_business_room_arn, :artifact_credentials, :auth_code, :authentication_token, :authorization_result, :backup_plan_tags, :backup_vault_tags, :base_32_string_seed, :block, :block_address, :block_data, :blocks, :body, :bot_configuration, :bot_email, :calling_name, :cause, :client_id, :client_request_token, :client_secret, :comment, :configuration, :copy_source_sse_customer_key, :credentials, :current_password, :custom_attributes, :custom_private_key, :db_password, :default_phone_number, :definition, :description, :destination_access_token, :digest_tip_address, :display_name, :domain_signing_private_key, :e164_phone_number, :email, :email_address, :email_message, :embed_url, :error, :external_meeting_id, :external_model_endpoint_data_blobs, :external_user_id, :fall_back_phone_number, :feedback_token, :file, :first_name, :full_name, :host_key, :id, :id_token, :input, :input_text, :ion_text, :join_token, :key, :key_id, :key_material, :key_store_password, :kms_key_id, :kms_master_key_id, :lambda_function_arn, :last_name, :local_console_password, :master_account_email, :master_user_name, :master_user_password, :meeting_host_id, :message, :metadata, :name, :new_password, :next_password, :notes, :number, :old_password, :outbound_events_https_endpoint, :output, :owner_information, :parameters, :passphrase, :password, :payload, :phone_number, :plaintext, :previous_password, :primary_email, :primary_provisioned_number, :private_key, :private_key_plaintext, :proof, :proposed_password, :proxy_phone_number, :public_key, :qr_code_png, :query, :random_password, :recovery_point_tags, :refresh_token, :registrant_contact, :request_attributes, :resource_arn, :restore_metadata, :revision, :search_query, :secret_access_key, :secret_binary, :secret_code, :secret_hash, :secret_string, :secret_to_authenticate_initiator, :secret_to_authenticate_target, :security_token, :service_password, :session_attributes, :session_token, :share_notes, :shared_secret, :slots, :sns_topic_arn, :source_access_token, :sqs_queue_arn, :sse_customer_key, :ssekms_encryption_context, :ssekms_key_id, :status_message, :tag_key_list, :tags, :target_address, :task_parameters, :tech_contact, :temporary_password, :text, :token, :trust_password, :type, :upload_credentials, :upload_url, :uri, :user_data, :user_email, :user_name, :user_password, :username, :value, :values, :variables, :vpn_psk, :zip_file]
       # end
 
       def initialize(options = {})
@@ -31,7 +31,7 @@ module Aws
       def filter_hash(values)
         filtered = {}
         values.each_pair do |key, value|
-          filtered[key] = @filters.include?(key) ? '[FILTERED]' : filter(value)
+          filtered[key] = @filters.any? { |f| f.to_s.casecmp(key.to_s).zero? } ? '[FILTERED]' : filter(value)
         end
         filtered
       end

data/lib/aws-sdk-core/pageable_response.rb

@@ -1,32 +1,46 @@
 module Aws
 
-  # Decorates a {Seahorse::Client::Response} with paging methods:
+  # Decorates a {Seahorse::Client::Response} with paging convenience methods.
+  # Some AWS calls provide paged responses to limit the amount of data returned
+  # with each response.  To optimize for latency, some APIs may return an
+  # inconsistent number of responses per page.  You should rely on the values of
+  # the `next_page?` method or using enumerable methods such as `each`
+  # rather than the number of items returned to iterate through results.
+  # See below for examples.
   #
-  #     resp = s3.list_objects(params)
-  #     resp.last_page?
-  #     #=> false
+  # # Paged Responses Are Enumerable
+  # The simplest way to handle paged response data is to use the built-in
+  # enumerator in the response object, as shown in the following example.
   #
-  #     # sends a request to receive the next response page
-  #     resp = resp.next_page
-  #     resp.last_page?
-  #     #=> true
+  #     s3 = Aws::S3::Client.new
   #
-  #     resp.next_page
-  #     #=> raises PageableResponse::LastPageError
+  #     s3.list_objects(bucket:'aws-sdk').each do |response|
+  #       puts response.contents.map(&:key)
+  #     end
   #
-  # You can enumerate all response pages with a block
+  # This yields one response object per API call made, and enumerates objects
+  # in the named bucket. The SDK retrieves additional pages of data to
+  # complete the request.
   #
-  #     ec2.describe_instances(params).each do |page|
-  #       # yields once per page
-  #       page.reservations.each do |r|
-  #         # ...
-  #       end
-  #     end
+  # # Handling Paged Responses Manually
+  # To handle paging yourself, use the response’s `next_page?` method to verify
+  # there are more pages to retrieve, or use the last_page? method to verify
+  # there are no more pages to retrieve.
+  #
+  # If there are more pages, use the `next_page` method to retrieve the
+  # next page of results, as shown in the following example.
   #
-  # Or using {#next_page} and {#last_page?}:
+  #     s3 = Aws::S3::Client.new
   #
-  #     resp.last_page?
-  #     resp = resp.next_page until resp.last_page?
+  #     # Get the first page of data
+  #     response = s3.list_objects(bucket:'aws-sdk')
+  #
+  #     # Get additional pages
+  #     while response.next_page? do
+  #       response = response.next_page
+  #       # Use the response data here...
+  #       puts response.contents.map(&:key)
+  #     end
   #
   module PageableResponse
 

data/lib/aws-sdk-core/param_validator.rb

@@ -22,6 +22,7 @@ module Aws
         ShapeRef.new(shape: shape)
       end
       @validate_required = options[:validate_required] != false
+      @input = options[:input].nil? ? true : !!options[:input]
     end
 
     # @param [Hash] params
@@ -39,6 +40,7 @@ module Aws
       return unless correct_type?(ref, values, errors, context)
 
       if ref.eventstream
+        # input eventstream is provided from event signals
         values.each do |value|
           # each event is structure type
           case value[:message_type]
@@ -58,7 +60,8 @@ module Aws
         # ensure required members are present
         if @validate_required
           shape.required.each do |member_name|
-            if values[member_name].nil?
+            input_eventstream = ref.shape.member(member_name).eventstream && @input
+            if values[member_name].nil? && !input_eventstream
               param = "#{context}[#{member_name.inspect}]"
               errors << "missing required parameter #{param}"
             end
@@ -138,7 +141,7 @@ module Aws
           errors << expected_got(context, "true or false", value)
         end
       when BlobShape
-        unless io_like?(value) or value.is_a?(String)
+        unless value.is_a?(String) || io_like?(value)
           errors << expected_got(context, "a String or IO object", value)
         end
       else
@@ -147,6 +150,11 @@ module Aws
     end
 
     def correct_type?(ref, value, errors, context)
+      if ref.eventstream && @input
+        errors << "instead of providing value directly for eventstreams at input,"\
+          " expected to use #signal events per stream"
+        return false
+      end
       case value
       when Hash then true
       when ref.shape.struct_class then true
@@ -158,9 +166,7 @@ module Aws
     end
 
     def io_like?(value)
-      value.respond_to?(:read) &&
-      value.respond_to?(:rewind) &&
-      value.respond_to?(:size)
+      value.respond_to?(:read) && value.respond_to?(:rewind)
     end
 
     def error_messages(errors)

data/lib/aws-sdk-core/plugins/client_metrics_plugin.rb

@@ -1,4 +1,5 @@
 require 'date'
+require_relative 'retries/error_inspector'
 
 module Aws
   module Plugins
@@ -24,6 +25,16 @@ agent is running on, where client metrics will be published via UDP.
         resolve_client_side_monitoring_port(cfg)
       end
 
+      option(:client_side_monitoring_host,
+        default: "127.0.0.1",
+        doc_type: String,
+        docstring: <<-DOCS) do |cfg|
+Allows you to specify the DNS hostname or IPv4 or IPv6 address that the client
+side monitoring agent is running on, where client metrics will be published via UDP.
+      DOCS
+        resolve_client_side_monitoring_host(cfg)
+      end
+
       option(:client_side_monitoring_publisher,
         default: ClientSideMonitoring::Publisher,
         doc_type: Aws::ClientSideMonitoring::Publisher,
@@ -49,6 +60,7 @@ all generated client side metrics. Defaults to an empty string.
           handlers.add(Handler, step: :initialize)
           publisher = config.client_side_monitoring_publisher
           publisher.agent_port = config.client_side_monitoring_port
+          publisher.agent_host = config.client_side_monitoring_host
         end
       end
 
@@ -70,6 +82,19 @@ all generated client side metrics. Defaults to an empty string.
         end
       end
 
+      def self.resolve_client_side_monitoring_host(cfg)
+        env_source = ENV["AWS_CSM_HOST"]
+        env_source = nil if env_source == ""
+        cfg_source = Aws.shared_config.csm_host(profile: cfg.profile)
+        if env_source
+          env_source
+        elsif cfg_source
+          cfg_source
+        else
+          "127.0.0.1"
+        end
+      end
+
       def self.resolve_client_side_monitoring(cfg)
         env_source = ENV["AWS_CSM_ENABLED"]
         env_source = nil if env_source == ""
@@ -117,7 +142,7 @@ all generated client side metrics. Defaults to an empty string.
             @handler.call(context)
           rescue StandardError => e
             # Handle SDK Exceptions
-            inspector = Aws::Plugins::RetryErrors::ErrorInspector.new(
+            inspector = Retries::ErrorInspector.new(
               e,
               context.http_response.status_code
             )

data/lib/aws-sdk-core/plugins/endpoint_discovery.rb

@@ -151,7 +151,7 @@ the background every 60 secs (default). Defaults to `false`.
 
       def self.resolve_endpoint_discovery(cfg)
         env = ENV['AWS_ENABLE_ENDPOINT_DISCOVERY']
-        shared_cfg = Aws.shared_config.endpoint_discovery(profile: cfg.profile)
+        shared_cfg = Aws.shared_config.endpoint_discovery_enabled(profile: cfg.profile)
         Aws::Util.str_2_bool(env) || Aws::Util.str_2_bool(shared_cfg)
       end
 

data/lib/aws-sdk-core/plugins/event_stream_configuration.rb

@@ -10,6 +10,20 @@ module Aws
 When an EventStream or Proc object is provided, it will be used as callback for each chunk of event stream response received along the way.
       DOCS
 
+      option(:input_event_stream_handler,
+        default: nil,
+        doc_type: 'Proc',
+        docstring: <<-DOCS)
+When an EventStream or Proc object is provided, it can be used for sending events for the event stream.
+      DOCS
+
+      option(:output_event_stream_handler,
+        default: nil,
+        doc_type: 'Proc',
+        docstring: <<-DOCS)
+When an EventStream or Proc object is provided, it will be used as callback for each chunk of event stream response received along the way.
+      DOCS
+
     end
 
   end

data/lib/aws-sdk-core/plugins/invocation_id.rb

@@ -0,0 +1,33 @@
+require 'securerandom'
+
+module Aws
+  module Plugins
+
+    # @api private
+    class InvocationId < Seahorse::Client::Plugin
+
+      # @api private
+      class Handler < Seahorse::Client::Handler
+
+        def call(context)
+          apply_invocation_id(context)
+          @handler.call(context)
+        end
+
+        private
+
+        def apply_invocation_id(context)
+          context.http_request.headers['amz-sdk-invocation-id'] = SecureRandom.uuid
+          if context[:input_event_emitter]
+            # only used for eventstreaming at input
+            context.http_request.headers['x-amz-content-sha256'] = 'STREAMING-AWS4-HMAC-SHA256-EVENTS'
+          end
+        end
+
+      end
+
+      handler(Handler, step: :initialize)
+
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/regional_endpoint.rb

@@ -35,7 +35,14 @@ to test endpoints. This should be avalid HTTP(S) URI.
         DOCS
         endpoint_prefix = cfg.api.metadata['endpointPrefix']
         if cfg.region && endpoint_prefix
-          Aws::Partitions::EndpointProvider.resolve(cfg.region, endpoint_prefix)
+          if cfg.respond_to?(:sts_regional_endpoints)
+            sts_regional = cfg.sts_regional_endpoints
+          end
+          Aws::Partitions::EndpointProvider.resolve(
+            cfg.region,
+            endpoint_prefix,
+            sts_regional
+          )
         end
       end
 

data/lib/aws-sdk-core/plugins/retries/client_rate_limiter.rb

@@ -0,0 +1,137 @@
+module Aws
+  module Plugins
+    module Retries
+      # @api private
+      # Used only in 'adaptive' retry mode
+      class ClientRateLimiter
+        MIN_CAPACITY = 1
+        MIN_FILL_RATE = 0.5
+        SMOOTH = 0.8
+        # How much to scale back after a throttling response
+        BETA = 0.7
+        # Controls how aggressively we scale up after being throttled
+        SCALE_CONSTANT = 0.4
+
+        def initialize
+          @mutex                = Mutex.new
+          @fill_rate            = nil
+          @max_capacity         = nil
+          @current_capacity     = 0
+          @last_timestamp       = nil
+          @enabled              = false
+          @measured_tx_rate     = 0
+          @last_tx_rate_bucket  = Aws::Util.monotonic_seconds
+          @request_count        = 0
+          @last_max_rate        = 0
+          @last_throttle_time   = Aws::Util.monotonic_seconds
+          @calculated_rate      = nil
+        end
+
+        def token_bucket_acquire(amount, wait_to_fill = true)
+          # Client side throttling is not enabled until we see a
+          # throttling error
+          return unless @enabled
+
+          @mutex.synchronize do
+            token_bucket_refill
+
+            # Next see if we have enough capacity for the requested amount
+            while @current_capacity < amount
+              raise Aws::Errors::RetryCapacityNotAvailableError unless wait_to_fill
+              @mutex.sleep((amount - @current_capacity) / @fill_rate)
+              token_bucket_refill
+            end
+            @current_capacity -= amount
+          end
+        end
+
+        def update_sending_rate(is_throttling_error)
+          @mutex.synchronize do
+            update_measured_rate
+
+            if is_throttling_error
+              rate_to_use = if @enabled
+                              [@measured_tx_rate, @fill_rate].min
+                            else
+                              @measured_tx_rate
+                            end
+
+              # The fill_rate is from the token bucket
+              @last_max_rate = rate_to_use
+              calculate_time_window
+              @last_throttle_time = Aws::Util.monotonic_seconds
+              @calculated_rate = cubic_throttle(rate_to_use)
+              enable_token_bucket
+            else
+              calculate_time_window
+              @calculated_rate = cubic_success(Aws::Util.monotonic_seconds)
+            end
+
+            new_rate = [@calculated_rate, 2 * @measured_tx_rate].min
+            token_bucket_update_rate(new_rate)
+          end
+        end
+
+        private
+
+        def token_bucket_refill
+          timestamp = Aws::Util.monotonic_seconds
+          unless @last_timestamp
+            @last_timestamp = timestamp
+            return
+          end
+
+          fill_amount = (timestamp - @last_timestamp) * @fill_rate
+          @current_capacity = [
+            @max_capacity, @current_capacity + fill_amount
+          ].min
+
+          @last_timestamp = timestamp
+        end
+
+        def token_bucket_update_rate(new_rps)
+          # Refill based on our current rate before we update to the
+          # new fill rate
+          token_bucket_refill
+          @fill_rate = [new_rps, MIN_FILL_RATE].max
+          @max_capacity = [new_rps, MIN_CAPACITY].max
+          # When we scale down we can't have a current capacity that exceeds our
+          # max_capacity.
+          @current_capacity = [@current_capacity, @max_capacity].min
+        end
+
+        def enable_token_bucket
+          @enabled = true
+        end
+
+        def update_measured_rate
+          t = Aws::Util.monotonic_seconds
+          time_bucket = (t * 2).floor / 2.0
+          @request_count += 1
+          if time_bucket > @last_tx_rate_bucket
+            current_rate = @request_count / (time_bucket - @last_tx_rate_bucket)
+            @measured_tx_rate = (current_rate * SMOOTH) +
+              (@measured_tx_rate * (1 - SMOOTH))
+            @request_count = 0
+            @last_tx_rate_bucket = time_bucket
+          end
+        end
+
+        def calculate_time_window
+          # This is broken out into a separate calculation because it only
+          # gets updated when @last_max_rate changes so it can be cached.
+          @time_window = ((@last_max_rate * (1 - BETA)) / SCALE_CONSTANT)**(1.0 / 3)
+        end
+
+        def cubic_success(timestamp)
+          dt = timestamp - @last_throttle_time
+          (SCALE_CONSTANT * ((dt - @time_window)**3)) + @last_max_rate
+        end
+
+        def cubic_throttle(rate_to_use)
+          rate_to_use * BETA
+        end
+      end
+    end
+  end
+end