aws-sdk-s3 1.150.0 → 1.160.0

data/lib/aws-sdk-s3/plugins/access_grants.rb

@@ -44,36 +44,70 @@ setting, caching, and fallback behavior.
             list_objects_v2: 'READ',
             list_object_versions: 'READ',
             list_parts: 'READ',
+            head_bucket: 'READ',
+            get_object_attributes: 'READ',
             put_object: 'WRITE',
             put_object_acl: 'WRITE',
             delete_object: 'WRITE',
             abort_multipart_upload: 'WRITE',
             create_multipart_upload: 'WRITE',
             upload_part: 'WRITE',
-            complete_multipart_upload: 'WRITE'
+            complete_multipart_upload: 'WRITE',
+            delete_objects: 'WRITE',
+            copy_object: 'READWRITE'
           }.freeze
 
           def call(context)
+            provider = context.config.access_grants_credentials_provider
+
             if access_grants_operation?(context) &&
-               !s3_express_endpoint?(context)
+               !s3_express_endpoint?(context) &&
+               !credentials_head_bucket_call?(provider)
               params = context[:endpoint_params]
               permission = PERMISSION_MAP[context.operation_name]
 
-              provider = context.config.access_grants_credentials_provider
+              key =
+                case context.operation_name
+                when :delete_objects
+                  delete_params = context.params[:delete]
+                  common_prefixes(delete_params[:objects].map { |o| o[:key] })
+                when :copy_object
+                  source_bucket, source_key = params[:copy_source].split('/', 2)
+                  if params[:bucket] != source_bucket
+                    raise ArgumentError,
+                          'source and destination bucket must be the same'
+                  end
+                  common_prefixes([params[:key], source_key])
+                else
+                  params[:key]
+                end
+
               credentials = provider.access_grants_credentials_for(
                 bucket: params[:bucket],
-                key: params[:key],
+                key: key,
                 prefix: params[:prefix],
                 permission: permission
               )
               context[:sigv4_credentials] = credentials # Sign will use this
             end
 
-            @handler.call(context)
+            with_metric(credentials) { @handler.call(context) }
           end
 
           private
 
+          def with_metric(credentials, &block)
+            return block.call unless credentials
+
+            Aws::Plugins::UserAgent.metric('S3_ACCESS_GRANTS', &block)
+          end
+
+          # HeadBucket is a supported call. When fetching credentials,
+          # this plugin is executed again, and becomes recursive.
+          def credentials_head_bucket_call?(provider)
+            provider.instance_variable_get(:@head_bucket_call)
+          end
+
           def access_grants_operation?(context)
             params = context[:endpoint_params]
             params[:bucket] && PERMISSION_MAP[context.operation_name]
@@ -82,6 +116,42 @@ setting, caching, and fallback behavior.
           def s3_express_endpoint?(context)
             context[:endpoint_properties]['backend'] == 'S3Express'
           end
+
+          # Return the common prefix of the keys, regardless of the delimiter.
+          # For example, given keys ['foo/bar', 'foo/baz'], the common prefix
+          # is 'foo/ba'.
+          def common_prefixes(keys)
+            return '' if keys.empty?
+
+            first_key = keys[0]
+            common_ancestor = first_key
+            last_prefix = ''
+            keys.each do |k|
+              until common_ancestor.empty?
+                break if k.start_with?(common_ancestor)
+
+                last_index = common_ancestor.rindex('/')
+                return '' if last_index.nil?
+
+                last_prefix = common_ancestor[(last_index + 1)..-1]
+                common_ancestor = common_ancestor[0...last_index]
+              end
+            end
+            new_common_ancestor = "#{common_ancestor}/#{last_prefix}"
+            keys.each do |k|
+              until last_prefix.empty?
+                break if k.start_with?(new_common_ancestor)
+
+                last_prefix = last_prefix[0...-1]
+                new_common_ancestor = "#{common_ancestor}/#{last_prefix}"
+              end
+            end
+            if new_common_ancestor == "#{first_key}/"
+              first_key
+            else
+              new_common_ancestor
+            end
+          end
         end
 
         def add_handlers(handlers, config)

data/lib/aws-sdk-s3/plugins/express_session_auth.rb

@@ -47,11 +47,17 @@ for different buckets.
                 context[:sigv4_credentials] = credentials # Sign will use this
               end
             end
-            @handler.call(context)
+            with_metric(credentials) { @handler.call(context) }
           end
 
           private
 
+          def with_metric(credentials, &block)
+            return block.call unless credentials
+
+            Aws::Plugins::UserAgent.metric('S3_EXPRESS_BUCKET', &block)
+          end
+
           def checksum_required?(context)
             context.operation.http_checksum_required ||
               (context.operation.http_checksum &&

data/lib/aws-sdk-s3/plugins/http_200_errors.rb

@@ -15,22 +15,67 @@ module Aws
 
           def call(context)
             @handler.call(context).on(200) do |response|
-              if error = check_for_error(context)
-                context.http_response.status_code = 500
-                response.data = nil
-                response.error = error
+              return response if streaming_output?(context.operation.output)
+
+              error = check_for_error(context)
+              return response unless error
+
+              context.http_response.status_code = 500
+              response.data = nil
+              response.error = error
+            end
+          end
+
+          private
+
+          # Streaming outputs are not subject to 200 errors.
+          def streaming_output?(output)
+            if (payload = output[:payload_member])
+              # checking ref and shape
+              payload['streaming'] || payload.shape['streaming'] ||
+                payload.eventstream
+            else
+              false
+            end
+          end
+
+          # Checks if the output shape is a structure shape and has members that
+          # are in the body for the case of a payload and a normal structure. A
+          # non-structure shape will not have members in the body. In the case
+          # of a string or blob, the body contents would have been checked first
+          # before this method is called in incomplete_xml_body?.
+          def members_in_body?(output)
+            shape =
+              if output[:payload_member]
+                output[:payload_member].shape
+              else
+                output.shape
               end
+
+            if structure_shape?(shape)
+              shape.members.any? { |_, k| k.location.nil? }
+            else
+              false
             end
           end
 
+          def structure_shape?(shape)
+            shape.is_a?(Seahorse::Model::Shapes::StructureShape)
+          end
+
+          # Must have a member in the body and have the start of an XML Tag.
+          # Other incomplete xml bodies will result in an XML ParsingError.
+          def incomplete_xml_body?(xml, output)
+            members_in_body?(output) && !xml.match(/<\w/)
+          end
+
           def check_for_error(context)
             xml = context.http_response.body_contents
-            if xml.match(/<Error>/)
+            if xml.match(/\?>\s*<Error>/)
               error_code = xml.match(/<Code>(.+?)<\/Code>/)[1]
               error_message = xml.match(/<Message>(.+?)<\/Message>/)[1]
               S3::Errors.error_class(error_code).new(context, error_message)
-            elsif !xml.match(/<\w/) # Must have the start of an XML Tag
-              # Other incomplete xml bodies will result in XML ParsingError
+            elsif incomplete_xml_body?(xml, context.operation.output)
               Seahorse::Client::NetworkingError.new(
                 S3::Errors
                   .error_class('InternalError')
@@ -40,15 +85,7 @@ module Aws
           end
         end
 
-        handler(
-          Handler,
-          step: :sign,
-          operations: [
-            :complete_multipart_upload,
-            :copy_object,
-            :upload_part_copy,
-          ]
-        )
+        handler(Handler, step: :sign)
       end
     end
   end

data/lib/aws-sdk-s3/resource.rb

@@ -166,7 +166,7 @@ module Aws::S3
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/about-object-ownership.html
     # @return [Bucket]
     def create_bucket(options = {})
-      Aws::Plugins::UserAgent.feature('resource') do
+      Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
         @client.create_bucket(options)
       end
       Bucket.new(
@@ -193,18 +193,20 @@ module Aws::S3
     # @return [Bucket::Collection]
     def buckets(options = {})
       batches = Enumerator.new do |y|
-        batch = []
-        resp = Aws::Plugins::UserAgent.feature('resource') do
+        resp = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
           @client.list_buckets(options)
         end
-        resp.data.buckets.each do |b|
-          batch << Bucket.new(
-            name: b.name,
-            data: b,
-            client: @client
-          )
+        resp.each_page do |page|
+          batch = []
+          page.data.buckets.each do |b|
+            batch << Bucket.new(
+              name: b.name,
+              data: b,
+              client: @client
+            )
+          end
+          y.yield(batch)
         end
-        y.yield(batch)
       end
       Bucket::Collection.new(batches)
     end