aws-sdk-s3 1.147.0 → 1.163.0

data/lib/aws-sdk-s3/object_version.rb

@@ -243,7 +243,7 @@ module Aws::S3
           :retry
         end
       end
-      Aws::Plugins::UserAgent.feature('resource') do
+      Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
         Aws::Waiters::Waiter.new(options).wait({})
       end
     end
@@ -303,7 +303,7 @@ module Aws::S3
         key: @object_key,
         version_id: @id
       )
-      resp = Aws::Plugins::UserAgent.feature('resource') do
+      resp = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
         @client.delete_object(options)
       end
       resp.data
@@ -523,6 +523,15 @@ module Aws::S3
     #   fails with the HTTP status code `403 Forbidden` (access denied).
     # @option options [String] :checksum_mode
     #   To retrieve the checksum, this mode must be enabled.
+    #
+    #   **General purpose buckets** - In addition, if you enable checksum mode
+    #   and the object is uploaded with a [checksum][1] and encrypted with an
+    #   Key Management Service (KMS) key, you must have permission to use the
+    #   `kms:Decrypt` action to retrieve the checksum.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_Checksum.html
     # @return [Types::GetObjectOutput]
     def get(options = {}, &block)
       options = options.merge(
@@ -530,7 +539,7 @@ module Aws::S3
         key: @object_key,
         version_id: @id
       )
-      resp = Aws::Plugins::UserAgent.feature('resource') do
+      resp = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
         @client.get_object(options, &block)
       end
       resp.data
@@ -544,6 +553,12 @@ module Aws::S3
     #     if_none_match: "IfNoneMatch",
     #     if_unmodified_since: Time.now,
     #     range: "Range",
+    #     response_cache_control: "ResponseCacheControl",
+    #     response_content_disposition: "ResponseContentDisposition",
+    #     response_content_encoding: "ResponseContentEncoding",
+    #     response_content_language: "ResponseContentLanguage",
+    #     response_content_type: "ResponseContentType",
+    #     response_expires: Time.now,
     #     sse_customer_algorithm: "SSECustomerAlgorithm",
     #     sse_customer_key: "SSECustomerKey",
     #     sse_customer_key_md5: "SSECustomerKeyMD5",
@@ -630,6 +645,18 @@ module Aws::S3
     #   satisfiable, only the `ContentLength` is affected in the response. If
     #   the Range is not satisfiable, S3 returns a `416 - Requested Range Not
     #   Satisfiable` error.
+    # @option options [String] :response_cache_control
+    #   Sets the `Cache-Control` header of the response.
+    # @option options [String] :response_content_disposition
+    #   Sets the `Content-Disposition` header of the response.
+    # @option options [String] :response_content_encoding
+    #   Sets the `Content-Encoding` header of the response.
+    # @option options [String] :response_content_language
+    #   Sets the `Content-Language` header of the response.
+    # @option options [String] :response_content_type
+    #   Sets the `Content-Type` header of the response.
+    # @option options [Time,DateTime,Date,Integer,String] :response_expires
+    #   Sets the `Expires` header of the response.
     # @option options [String] :sse_customer_algorithm
     #   Specifies the algorithm to use when encrypting the object (for
     #   example, AES256).
@@ -683,10 +710,20 @@ module Aws::S3
     # @option options [String] :checksum_mode
     #   To retrieve the checksum, this parameter must be enabled.
     #
-    #   In addition, if you enable `ChecksumMode` and the object is encrypted
-    #   with Amazon Web Services Key Management Service (Amazon Web Services
-    #   KMS), you must have permission to use the `kms:Decrypt` action for the
-    #   request to succeed.
+    #   **General purpose buckets** - If you enable checksum mode and the
+    #   object is uploaded with a [checksum][1] and encrypted with an Key
+    #   Management Service (KMS) key, you must have permission to use the
+    #   `kms:Decrypt` action to retrieve the checksum.
+    #
+    #   **Directory buckets** - If you enable `ChecksumMode` and the object is
+    #   encrypted with Amazon Web Services Key Management Service (Amazon Web
+    #   Services KMS), you must also have the `kms:GenerateDataKey` and
+    #   `kms:Decrypt` permissions in IAM identity-based policies and KMS key
+    #   policies for the KMS key to retrieve the checksum of the object.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_Checksum.html
     # @return [Types::HeadObjectOutput]
     def head(options = {})
       options = options.merge(
@@ -694,7 +731,7 @@ module Aws::S3
         key: @object_key,
         version_id: @id
       )
-      resp = Aws::Plugins::UserAgent.feature('resource') do
+      resp = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
         @client.head_object(options)
       end
       resp.data
@@ -868,7 +905,7 @@ module Aws::S3
               version_id: item.id
             }
           end
-          Aws::Plugins::UserAgent.feature('resource') do
+          Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
             batch[0].client.delete_objects(params)
           end
         end

data/lib/aws-sdk-s3/plugins/access_grants.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+      # @api private
+      class AccessGrants < Seahorse::Client::Plugin
+        @s3control =
+          begin
+            require 'aws-sdk-s3control'
+            true
+          rescue LoadError
+            false
+          end
+
+        option(
+          :access_grants,
+          default: false,
+          doc_type: 'Boolean',
+          docstring: <<-DOCS)
+When `true`, the S3 client will use the S3 Access Grants feature to
+authenticate requests. Bucket credentials will be fetched from S3
+Control using the `get_data_access` API.
+          DOCS
+
+        option(:access_grants_credentials_provider,
+          doc_type: 'Aws::S3::AccessGrantsCredentialsProvider',
+          rbs_type: 'untyped',
+          docstring: <<-DOCS) do |_cfg|
+When `access_grants` is `true`, this option can be used to provide
+additional options to the credentials provider, including a privilege
+setting, caching, and fallback behavior.
+          DOCS
+          Aws::S3::AccessGrantsCredentialsProvider.new
+        end
+
+        # @api private
+        class Handler < Seahorse::Client::Handler
+          PERMISSION_MAP = {
+            head_object: 'READ',
+            get_object: 'READ',
+            get_object_acl: 'READ',
+            list_multipart_uploads: 'READ',
+            list_objects_v2: 'READ',
+            list_object_versions: 'READ',
+            list_parts: 'READ',
+            head_bucket: 'READ',
+            get_object_attributes: 'READ',
+            put_object: 'WRITE',
+            put_object_acl: 'WRITE',
+            delete_object: 'WRITE',
+            abort_multipart_upload: 'WRITE',
+            create_multipart_upload: 'WRITE',
+            upload_part: 'WRITE',
+            complete_multipart_upload: 'WRITE',
+            delete_objects: 'WRITE',
+            copy_object: 'READWRITE'
+          }.freeze
+
+          def call(context)
+            provider = context.config.access_grants_credentials_provider
+
+            if access_grants_operation?(context) &&
+               !s3_express_endpoint?(context) &&
+               !credentials_head_bucket_call?(provider)
+              params = context[:endpoint_params]
+              permission = PERMISSION_MAP[context.operation_name]
+
+              key =
+                case context.operation_name
+                when :delete_objects
+                  delete_params = context.params[:delete]
+                  common_prefixes(delete_params[:objects].map { |o| o[:key] })
+                when :copy_object
+                  source_bucket, source_key = params[:copy_source].split('/', 2)
+                  if params[:bucket] != source_bucket
+                    raise ArgumentError,
+                          'source and destination bucket must be the same'
+                  end
+                  common_prefixes([params[:key], source_key])
+                else
+                  params[:key]
+                end
+
+              credentials = provider.access_grants_credentials_for(
+                bucket: params[:bucket],
+                key: key,
+                prefix: params[:prefix],
+                permission: permission
+              )
+              context[:sigv4_credentials] = credentials # Sign will use this
+            end
+
+            with_metric(credentials) { @handler.call(context) }
+          end
+
+          private
+
+          def with_metric(credentials, &block)
+            return block.call unless credentials
+
+            Aws::Plugins::UserAgent.metric('S3_ACCESS_GRANTS', &block)
+          end
+
+          # HeadBucket is a supported call. When fetching credentials,
+          # this plugin is executed again, and becomes recursive.
+          def credentials_head_bucket_call?(provider)
+            provider.instance_variable_get(:@head_bucket_call)
+          end
+
+          def access_grants_operation?(context)
+            params = context[:endpoint_params]
+            params[:bucket] && PERMISSION_MAP[context.operation_name]
+          end
+
+          def s3_express_endpoint?(context)
+            context[:endpoint_properties]['backend'] == 'S3Express'
+          end
+
+          # Return the common prefix of the keys, regardless of the delimiter.
+          # For example, given keys ['foo/bar', 'foo/baz'], the common prefix
+          # is 'foo/ba'.
+          def common_prefixes(keys)
+            return '' if keys.empty?
+
+            first_key = keys[0]
+            common_ancestor = first_key
+            last_prefix = ''
+            keys.each do |k|
+              until common_ancestor.empty?
+                break if k.start_with?(common_ancestor)
+
+                last_index = common_ancestor.rindex('/')
+                return '' if last_index.nil?
+
+                last_prefix = common_ancestor[(last_index + 1)..-1]
+                common_ancestor = common_ancestor[0...last_index]
+              end
+            end
+            new_common_ancestor = "#{common_ancestor}/#{last_prefix}"
+            keys.each do |k|
+              until last_prefix.empty?
+                break if k.start_with?(new_common_ancestor)
+
+                last_prefix = last_prefix[0...-1]
+                new_common_ancestor = "#{common_ancestor}/#{last_prefix}"
+              end
+            end
+            if new_common_ancestor == "#{first_key}/"
+              first_key
+            else
+              new_common_ancestor
+            end
+          end
+        end
+
+        def add_handlers(handlers, config)
+          return unless AccessGrants.s3control? && config.access_grants
+
+          handlers.add(Handler)
+        end
+
+        def after_initialize(client)
+          return unless AccessGrants.s3control? && client.config.access_grants
+
+          provider = client.config.access_grants_credentials_provider
+          provider.s3_client = client unless provider.s3_client
+        end
+
+        class << self
+          def s3control?
+            @s3control
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/endpoints.rb

@@ -46,11 +46,20 @@ module Aws::S3
           context[:auth_scheme] =
             Aws::Endpoints.resolve_auth_scheme(context, endpoint)
 
-          @handler.call(context)
+          with_metrics(context) { @handler.call(context) }
         end
 
         private
 
+        def with_metrics(context, &block)
+          metrics = []
+          metrics << 'ENDPOINT_OVERRIDE' unless context.config.regional_endpoint
+          if context[:auth_scheme] && context[:auth_scheme]['name'] == 'sigv4a'
+            metrics << 'SIGV4A_SIGNING'
+          end
+          Aws::Plugins::UserAgent.metric(*metrics, &block)
+        end
+
         def apply_endpoint_headers(context, headers)
           headers.each do |key, values|
             value = values

data/lib/aws-sdk-s3/plugins/express_session_auth.rb

@@ -31,7 +31,7 @@ for different buckets.
           def call(context)
             if (props = context[:endpoint_properties])
               # S3 Express endpoint - turn off md5 and enable crc32 default
-              if (backend = props['backend']) && backend == 'S3Express'
+              if props['backend'] == 'S3Express'
                 if context.operation_name == :put_object || checksum_required?(context)
                   context[:default_request_checksum_algorithm] = 'CRC32'
                 end
@@ -47,11 +47,17 @@ for different buckets.
                 context[:sigv4_credentials] = credentials # Sign will use this
               end
             end
-            @handler.call(context)
+            with_metric(credentials) { @handler.call(context) }
           end
 
           private
 
+          def with_metric(credentials, &block)
+            return block.call unless credentials
+
+            Aws::Plugins::UserAgent.metric('S3_EXPRESS_BUCKET', &block)
+          end
+
           def checksum_required?(context)
             context.operation.http_checksum_required ||
               (context.operation.http_checksum &&

data/lib/aws-sdk-s3/plugins/http_200_errors.rb

@@ -15,22 +15,67 @@ module Aws
 
           def call(context)
             @handler.call(context).on(200) do |response|
-              if error = check_for_error(context)
-                context.http_response.status_code = 500
-                response.data = nil
-                response.error = error
+              return response if streaming_output?(context.operation.output)
+
+              error = check_for_error(context)
+              return response unless error
+
+              context.http_response.status_code = 500
+              response.data = nil
+              response.error = error
+            end
+          end
+
+          private
+
+          # Streaming outputs are not subject to 200 errors.
+          def streaming_output?(output)
+            if (payload = output[:payload_member])
+              # checking ref and shape
+              payload['streaming'] || payload.shape['streaming'] ||
+                payload.eventstream
+            else
+              false
+            end
+          end
+
+          # Checks if the output shape is a structure shape and has members that
+          # are in the body for the case of a payload and a normal structure. A
+          # non-structure shape will not have members in the body. In the case
+          # of a string or blob, the body contents would have been checked first
+          # before this method is called in incomplete_xml_body?.
+          def members_in_body?(output)
+            shape =
+              if output[:payload_member]
+                output[:payload_member].shape
+              else
+                output.shape
               end
+
+            if structure_shape?(shape)
+              shape.members.any? { |_, k| k.location.nil? }
+            else
+              false
             end
           end
 
+          def structure_shape?(shape)
+            shape.is_a?(Seahorse::Model::Shapes::StructureShape)
+          end
+
+          # Must have a member in the body and have the start of an XML Tag.
+          # Other incomplete xml bodies will result in an XML ParsingError.
+          def incomplete_xml_body?(xml, output)
+            members_in_body?(output) && !xml.match(/<\w/)
+          end
+
           def check_for_error(context)
             xml = context.http_response.body_contents
-            if xml.match(/<Error>/)
+            if xml.match(/\?>\s*<Error>/)
               error_code = xml.match(/<Code>(.+?)<\/Code>/)[1]
               error_message = xml.match(/<Message>(.+?)<\/Message>/)[1]
               S3::Errors.error_class(error_code).new(context, error_message)
-            elsif !xml.match(/<\w/) # Must have the start of an XML Tag
-              # Other incomplete xml bodies will result in XML ParsingError
+            elsif incomplete_xml_body?(xml, context.operation.output)
               Seahorse::Client::NetworkingError.new(
                 S3::Errors
                   .error_class('InternalError')
@@ -40,15 +85,7 @@ module Aws
           end
         end
 
-        handler(
-          Handler,
-          step: :sign,
-          operations: [
-            :complete_multipart_upload,
-            :copy_object,
-            :upload_part_copy,
-          ]
-        )
+        handler(Handler, step: :sign)
       end
     end
   end

data/lib/aws-sdk-s3/plugins/s3_signer.rb

@@ -4,6 +4,11 @@ require 'aws-sigv4'
 
 module Aws
   module S3
+    # @api private
+    def self.bucket_region_cache
+      @bucket_region_cache ||= BucketRegionCache.new
+    end
+
     module Plugins
       # This plugin used to have a V4 signer but it was removed in favor of
       # generic Sign plugin that uses endpoint auth scheme.
@@ -51,7 +56,7 @@ module Aws
           private
 
           def check_for_cached_region(context, bucket)
-            cached_region = S3::BUCKET_REGIONS[bucket]
+            cached_region = Aws::S3.bucket_region_cache[bucket]
             if cached_region &&
                cached_region != context.config.region &&
                !S3Signer.custom_endpoint?(context)
@@ -97,7 +102,7 @@ module Aws
           end
 
           def update_bucket_cache(context, actual_region)
-            S3::BUCKET_REGIONS[context.params[:bucket]] = actual_region
+            Aws::S3.bucket_region_cache[context.params[:bucket]] = actual_region
           end
 
           def fips_region?(resp)

data/lib/aws-sdk-s3/presigner.rb

@@ -200,6 +200,7 @@ module Aws
         req.handlers.remove(Aws::Plugins::Sign::Handler)
         req.handlers.remove(Seahorse::Client::Plugins::ContentLength::Handler)
         req.handlers.remove(Aws::Rest::ContentTypeHandler)
+        req.handlers.remove(Aws::Plugins::InvocationId::Handler)
 
         req.handle(step: :send) do |context|
           # if an endpoint was not provided, force secure or insecure

data/lib/aws-sdk-s3/resource.rb

@@ -166,7 +166,7 @@ module Aws::S3
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/about-object-ownership.html
     # @return [Bucket]
     def create_bucket(options = {})
-      Aws::Plugins::UserAgent.feature('resource') do
+      Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
         @client.create_bucket(options)
       end
       Bucket.new(
@@ -193,18 +193,20 @@ module Aws::S3
     # @return [Bucket::Collection]
     def buckets(options = {})
       batches = Enumerator.new do |y|
-        batch = []
-        resp = Aws::Plugins::UserAgent.feature('resource') do
+        resp = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
           @client.list_buckets(options)
         end
-        resp.data.buckets.each do |b|
-          batch << Bucket.new(
-            name: b.name,
-            data: b,
-            client: @client
-          )
+        resp.each_page do |page|
+          batch = []
+          page.data.buckets.each do |b|
+            batch << Bucket.new(
+              name: b.name,
+              data: b,
+              client: @client
+            )
+          end
+          y.yield(batch)
         end
-        y.yield(batch)
       end
       Bucket::Collection.new(batches)
     end