aws-sdk-core 3.125.6 → 3.188.0

data/lib/aws-sdk-core/plugins/request_compression.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+module Aws
+  module Plugins
+    # @api private
+    class RequestCompression < Seahorse::Client::Plugin
+      DEFAULT_MIN_COMPRESSION_SIZE = 10_240
+      MIN_COMPRESSION_SIZE_LIMIT = 10_485_760
+      SUPPORTED_ENCODINGS = %w[gzip].freeze
+      CHUNK_SIZE = 1 * 1024 * 1024 # one MB
+
+      option(
+        :disable_request_compression,
+        default: false,
+        doc_type: 'Boolean',
+        docstring: <<-DOCS) do |cfg|
+When set to 'true' the request body will not be compressed
+for supported operations.
+      DOCS
+        resolve_disable_request_compression(cfg)
+      end
+
+      option(
+        :request_min_compression_size_bytes,
+        default: 10_240,
+        doc_type: 'Integer',
+        docstring: <<-DOCS) do |cfg|
+The minimum size in bytes that triggers compression for request
+bodies. The value must be non-negative integer value between 0
+and 10485780 bytes inclusive.
+      DOCS
+        resolve_request_min_compression_size_bytes(cfg)
+      end
+
+      def after_initialize(client)
+        validate_disable_request_compression_input(client.config)
+        validate_request_min_compression_size_bytes_input(client.config)
+      end
+
+      def validate_disable_request_compression_input(cfg)
+        unless [true, false].include?(cfg.disable_request_compression)
+          raise ArgumentError,
+                'Must provide either `true` or `false` for the '\
+                '`disable_request_compression` configuration option.'
+        end
+      end
+
+      def validate_request_min_compression_size_bytes_input(cfg)
+        value = Integer(cfg.request_min_compression_size_bytes)
+        unless value.between?(0, MIN_COMPRESSION_SIZE_LIMIT)
+          raise ArgumentError,
+                'Must provide a non-negative integer value between '\
+                  '`0` and `10485760` bytes inclusive for the '\
+                  '`request_min_compression_size_bytes` configuration option.'
+        end
+      end
+
+      def add_handlers(handlers, _config)
+        # priority set to ensure compression happens BEFORE checksum
+        handlers.add(CompressionHandler, priority: 16, step: :build)
+      end
+
+      class << self
+        private
+
+        def resolve_disable_request_compression(cfg)
+          value = ENV['AWS_DISABLE_REQUEST_COMPRESSION'] ||
+                  Aws.shared_config.disable_request_compression(profile: cfg.profile) ||
+                  'false'
+          Aws::Util.str_2_bool(value)
+        end
+
+        def resolve_request_min_compression_size_bytes(cfg)
+          value = ENV['AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES'] ||
+                  Aws.shared_config.request_min_compression_size_bytes(profile: cfg.profile) ||
+                  DEFAULT_MIN_COMPRESSION_SIZE.to_s
+          Integer(value)
+        end
+      end
+
+      # @api private
+      class CompressionHandler < Seahorse::Client::Handler
+        def call(context)
+          if should_compress?(context)
+            selected_encoding = request_encoding_selection(context)
+            if selected_encoding
+              if streaming?(context.operation.input)
+                process_streaming_compression(selected_encoding, context)
+              elsif context.http_request.body.size >= context.config.request_min_compression_size_bytes
+                process_compression(selected_encoding, context)
+              end
+            end
+          end
+          @handler.call(context)
+        end
+
+        private
+
+        def request_encoding_selection(context)
+          encoding_list = context.operation.request_compression['encodings']
+          encoding_list.find { |encoding| RequestCompression::SUPPORTED_ENCODINGS.include?(encoding) }
+        end
+
+        def update_content_encoding(encoding, context)
+          headers = context.http_request.headers
+          if headers['Content-Encoding']
+            headers['Content-Encoding'] += ',' + encoding
+          else
+            headers['Content-Encoding'] = encoding
+          end
+        end
+
+        def should_compress?(context)
+          context.operation.request_compression &&
+            !context.config.disable_request_compression
+        end
+
+        def streaming?(input)
+          if payload = input[:payload_member] # checking ref and shape
+            payload['streaming'] || payload.shape['streaming']
+          else
+            false
+          end
+        end
+
+        def process_compression(encoding, context)
+          case encoding
+          when 'gzip'
+            gzip_compress(context)
+          else
+            raise StandardError, "We currently do not support #{encoding} encoding"
+          end
+          update_content_encoding(encoding, context)
+        end
+
+        def gzip_compress(context)
+          compressed = StringIO.new
+          compressed.binmode
+          gzip_writer = Zlib::GzipWriter.new(compressed)
+          if context.http_request.body.respond_to?(:read)
+            update_in_chunks(gzip_writer, context.http_request.body)
+          else
+            gzip_writer.write(context.http_request.body)
+          end
+          gzip_writer.close
+          new_body = StringIO.new(compressed.string)
+          context.http_request.body = new_body
+        end
+
+        def update_in_chunks(compressor, io)
+          loop do
+            chunk = io.read(CHUNK_SIZE)
+            break unless chunk
+
+            compressor.write(chunk)
+          end
+        end
+
+        def process_streaming_compression(encoding, context)
+          case encoding
+          when 'gzip'
+            context.http_request.body = GzipIO.new(context.http_request.body)
+          else
+            raise StandardError, "We currently do not support #{encoding} encoding"
+          end
+          update_content_encoding(encoding, context)
+        end
+
+        # @api private
+        class GzipIO
+          def initialize(body)
+            @body = body
+            @buffer = ChunkBuffer.new
+            @gzip_writer = Zlib::GzipWriter.new(@buffer)
+          end
+
+          def read(length, buff = nil)
+            if @gzip_writer.closed?
+              # an empty string to signify an end as
+              # there will be nothing remaining to be read
+              StringIO.new('').read(length, buff)
+              return
+            end
+
+            chunk = @body.read(length)
+            if !chunk || chunk.empty?
+              # closing the writer will write one last chunk
+              # with a trailer (to be read from the @buffer)
+              @gzip_writer.close
+            else
+              # flush happens first to ensure that header fields
+              # are being sent over since write will override
+              @gzip_writer.flush
+              @gzip_writer.write(chunk)
+            end
+
+            StringIO.new(@buffer.last_chunk).read(length, buff)
+          end
+        end
+
+        # @api private
+        class ChunkBuffer
+          def initialize
+            @last_chunk = nil
+          end
+
+          attr_reader :last_chunk
+
+          def write(data)
+            @last_chunk = data
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/response_paging.rb

@@ -10,7 +10,7 @@ module Aws
         def call(context)
           context[:original_params] = context.params
           resp = @handler.call(context)
-          resp.extend(PageableResponse)
+          PageableResponse.apply(resp)
           resp.pager = context.operation[:pager] || Aws::Pager::NullPager.new
           resp
         end

data/lib/aws-sdk-core/plugins/retries/error_inspector.rb

@@ -39,7 +39,8 @@ module Aws
 
         CHECKSUM_ERRORS = Set.new(
           [
-            'CRC32CheckFailed' # dynamodb
+            'CRC32CheckFailed', # dynamodb
+            'BadDigest' # s3
           ]
         )
 
@@ -82,7 +83,7 @@ module Aws
         end
 
         def checksum?
-          CHECKSUM_ERRORS.include?(@name) || @error.is_a?(Errors::ChecksumError)
+          CHECKSUM_ERRORS.include?(@name)
         end
 
         def networking?

data/lib/aws-sdk-core/plugins/retry_errors.rb

@@ -313,12 +313,17 @@ a clock skew correction and retry requests with skewed client clocks.
 
         def retry_request(context, error)
           context.retries += 1
-          context.config.credentials.refresh! if error.expired_credentials?
+          context.config.credentials.refresh! if refresh_credentials?(context, error)
           context.http_request.body.rewind
           context.http_response.reset
           call(context)
         end
 
+        def refresh_credentials?(context, error)
+          error.expired_credentials? &&
+            context.config.credentials.respond_to?(:refresh!)
+        end
+
         def add_retry_headers(context)
           request_pairs = {
             'attempt' => context.retries,
@@ -383,7 +388,7 @@ a clock skew correction and retry requests with skewed client clocks.
         def retry_request(context, error)
           delay_retry(context)
           context.retries += 1
-          context.config.credentials.refresh! if error.expired_credentials?
+          context.config.credentials.refresh! if refresh_credentials?(context, error)
           context.http_request.body.rewind
           context.http_response.reset
           call(context)
@@ -399,6 +404,11 @@ a clock skew correction and retry requests with skewed client clocks.
             response_truncatable?(context)
         end
 
+        def refresh_credentials?(context, error)
+          error.expired_credentials? &&
+            context.config.credentials.respond_to?(:refresh!)
+        end
+
         def retry_limit(context)
           context.config.retry_limit
         end

data/lib/aws-sdk-core/plugins/sign.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+require 'aws-sigv4'
+
+module Aws
+  module Plugins
+    # @api private
+    class Sign < Seahorse::Client::Plugin
+      # These once had defaults. But now they are used as overrides to
+      # new endpoint and auth resolution.
+      option(:sigv4_signer)
+      option(:sigv4_name)
+      option(:sigv4_region)
+      option(:unsigned_operations, default: [])
+
+      supported_auth_types = %w[sigv4 bearer none]
+      supported_auth_types += ['sigv4a'] if Aws::Sigv4::Signer.use_crt?
+      SUPPORTED_AUTH_TYPES = supported_auth_types.freeze
+
+      def add_handlers(handlers, cfg)
+        operations = cfg.api.operation_names - cfg.unsigned_operations
+        handlers.add(Handler, step: :sign, operations: operations)
+      end
+
+      # @api private
+      # Return a signer with the `sign(context)` method
+      def self.signer_for(auth_scheme, config, region_override = nil)
+        case auth_scheme['name']
+        when 'sigv4', 'sigv4a'
+          SignatureV4.new(auth_scheme, config, region_override)
+        when 'bearer'
+          Bearer.new
+        else
+          NullSigner.new
+        end
+      end
+
+      class Handler < Seahorse::Client::Handler
+        def call(context)
+          # Skip signing if using sigv2 signing from s3_signer in S3
+          unless v2_signing?(context.config)
+            signer = Sign.signer_for(
+              context[:auth_scheme],
+              context.config,
+              context[:sigv4_region]
+            )
+            signer.sign(context)
+          end
+          @handler.call(context)
+        end
+
+        private
+
+        def v2_signing?(config)
+          # 's3' is legacy signing, 'v4' is default
+          config.respond_to?(:signature_version) &&
+            config.signature_version == 's3'
+        end
+      end
+
+      # @api private
+      class Bearer
+        def initialize
+        end
+
+        def sign(context)
+          if context.http_request.endpoint.scheme != 'https'
+            raise ArgumentError,
+                  'Unable to use bearer authorization on non https endpoint.'
+          end
+
+          token_provider = context.config.token_provider
+
+          raise Errors::MissingBearerTokenError unless token_provider&.set?
+
+          context.http_request.headers['Authorization'] =
+            "Bearer #{token_provider.token.token}"
+        end
+
+        def presign_url(*args)
+          raise ArgumentError, 'Bearer auth does not support presigned urls'
+        end
+
+        def sign_event(*args)
+          raise ArgumentError, 'Bearer auth does not support event signing'
+        end
+      end
+
+      # @api private
+      class SignatureV4
+        def initialize(auth_scheme, config, region_override = nil)
+          scheme_name = auth_scheme['name']
+
+          unless %w[sigv4 sigv4a].include?(scheme_name)
+            raise ArgumentError,
+                  "Expected sigv4 or sigv4a auth scheme, got #{scheme_name}"
+          end
+
+          region = if scheme_name == 'sigv4a'
+                     auth_scheme['signingRegionSet'].first
+                   else
+                     auth_scheme['signingRegion']
+                   end
+          begin
+            @signer = Aws::Sigv4::Signer.new(
+              service: config.sigv4_name || auth_scheme['signingName'],
+              region: region_override || config.sigv4_region || region,
+              credentials_provider: config.credentials,
+              signing_algorithm: scheme_name.to_sym,
+              uri_escape_path: !!!auth_scheme['disableDoubleEncoding'],
+              normalize_path: !!!auth_scheme['disableNormalizePath'],
+              unsigned_headers: %w[content-length user-agent x-amzn-trace-id]
+            )
+          rescue Aws::Sigv4::Errors::MissingCredentialsError
+            raise Aws::Errors::MissingCredentialsError
+          end
+        end
+
+        def sign(context)
+          req = context.http_request
+
+          apply_authtype(context, req)
+          reset_signature(req)
+          apply_clock_skew(context, req)
+
+          # compute the signature
+          begin
+            signature = @signer.sign_request(
+              http_method: req.http_method,
+              url: req.endpoint,
+              headers: req.headers,
+              body: req.body
+            )
+          rescue Aws::Sigv4::Errors::MissingCredentialsError
+            # Necessary for when credentials is explicitly set to nil
+            raise Aws::Errors::MissingCredentialsError
+          end
+          # apply signature headers
+          req.headers.update(signature.headers)
+
+          # add request metadata with signature components for debugging
+          context[:canonical_request] = signature.canonical_request
+          context[:string_to_sign] = signature.string_to_sign
+        end
+
+        def presign_url(*args)
+          @signer.presign_url(*args)
+        end
+
+        def sign_event(*args)
+          @signer.sign_event(*args)
+        end
+
+        private
+
+        def apply_authtype(context, req)
+          if context.operation['authtype'].eql?('v4-unsigned-body') &&
+             req.endpoint.scheme.eql?('https')
+            req.headers['X-Amz-Content-Sha256'] ||= 'UNSIGNED-PAYLOAD'
+          end
+        end
+
+        def reset_signature(req)
+          # in case this request is being re-signed
+          req.headers.delete('Authorization')
+          req.headers.delete('X-Amz-Security-Token')
+          req.headers.delete('X-Amz-Date')
+          req.headers.delete('x-Amz-Region-Set')
+        end
+
+        def apply_clock_skew(context, req)
+          if context.config.respond_to?(:clock_skew) &&
+             context.config.clock_skew &&
+             context.config.correct_clock_skew
+
+            endpoint = context.http_request.endpoint
+            skew = context.config.clock_skew.clock_correction(endpoint)
+            if skew.abs.positive?
+              req.headers['X-Amz-Date'] =
+                (Time.now.utc + skew).strftime('%Y%m%dT%H%M%SZ')
+            end
+          end
+        end
+
+      end
+
+      # @api private
+      class NullSigner
+
+        def sign(context)
+        end
+
+        def presign_url(*args)
+        end
+
+        def sign_event(*args)
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/signature_v2.rb

@@ -3,6 +3,7 @@
 module Aws
   module Plugins
     # @api private
+    # Necessary to keep after Endpoints 2.0
     class SignatureV2 < Seahorse::Client::Plugin
 
       option(:v2_signer) do |cfg|

data/lib/aws-sdk-core/plugins/signature_v4.rb

@@ -5,8 +5,11 @@ require 'aws-sigv4'
 module Aws
   module Plugins
     # @api private
+    # Necessary to exist after endpoints 2.0
     class SignatureV4 < Seahorse::Client::Plugin
 
+      V4_AUTH = %w[v4 v4-unsigned-payload v4-unsigned-body]
+
       option(:sigv4_signer) do |cfg|
         SignatureV4.build_signer(cfg)
       end
@@ -32,13 +35,16 @@ module Aws
       end
 
       option(:unsigned_operations) do |cfg|
-        cfg.api.operation_names.inject([]) do |unsigned, operation_name|
-          if cfg.api.operation(operation_name)['authtype'] == 'none' ||
-             cfg.api.operation(operation_name)['authtype'] == 'custom'
-            # Unsign requests that has custom apigateway authorizer as well
-            unsigned << operation_name
-          else
-            unsigned
+        if cfg.api.metadata['signatureVersion'] == 'v4'
+          # select operations where authtype is set and is not v4
+          cfg.api.operation_names.select do |o|
+            cfg.api.operation(o)['authtype'] && !V4_AUTH.include?(cfg.api.operation(o)['authtype'])
+          end
+        else # service is not v4 auth
+          # select all operations where authtype is not v4
+          # (includes operations with no explicit authtype)
+          cfg.api.operation_names.select do |o|
+            !V4_AUTH.include?(cfg.api.operation(o)['authtype'])
           end
         end
       end
@@ -98,6 +104,7 @@ module Aws
           req.headers.delete('Authorization')
           req.headers.delete('X-Amz-Security-Token')
           req.headers.delete('X-Amz-Date')
+          req.headers.delete('x-Amz-Region-Set')
 
           if context.config.respond_to?(:clock_skew) &&
              context.config.clock_skew &&
@@ -134,7 +141,7 @@ module Aws
         def apply_authtype(context)
           if context.operation['authtype'].eql?('v4-unsigned-body') &&
              context.http_request.endpoint.scheme.eql?('https')
-            context.http_request.headers['X-Amz-Content-Sha256'] = 'UNSIGNED-PAYLOAD'
+            context.http_request.headers['X-Amz-Content-Sha256'] ||= 'UNSIGNED-PAYLOAD'
           end
           context
         end

data/lib/aws-sdk-core/plugins/stub_responses.rb

@@ -51,7 +51,11 @@ requests are made, and retries are disabled.
           stub = context.client.next_stub(context)
           resp = Seahorse::Client::Response.new(context: context)
           async_mode = context.client.is_a? Seahorse::Client::AsyncBase
-          apply_stub(stub, resp, async_mode)
+          if Hash === stub && stub[:mutex]
+            stub[:mutex].synchronize { apply_stub(stub, resp, async_mode) }
+          else
+            apply_stub(stub, resp, async_mode)
+          end
 
           async_mode ? Seahorse::Client::AsyncResponse.new(
             context: context, stream: context[:input_event_stream_handler].event_emitter.stream, sync_queue: Queue.new) : resp