aws-sdk-s3 1.211.0 → 1.217.0

data/lib/aws-sdk-s3/client_api.rb

@@ -24,6 +24,7 @@ module Aws::S3
     AcceptRanges = Shapes::StringShape.new(name: 'AcceptRanges')
     AccessControlPolicy = Shapes::StructureShape.new(name: 'AccessControlPolicy')
     AccessControlTranslation = Shapes::StructureShape.new(name: 'AccessControlTranslation')
+    AccessDenied = Shapes::StructureShape.new(name: 'AccessDenied')
     AccessKeyIdValue = Shapes::StringShape.new(name: 'AccessKeyIdValue')
     AccessPointAlias = Shapes::BooleanShape.new(name: 'AccessPointAlias')
     AccessPointArn = Shapes::StringShape.new(name: 'AccessPointArn')
@@ -60,6 +61,7 @@ module Aws::S3
     BucketLoggingStatus = Shapes::StructureShape.new(name: 'BucketLoggingStatus')
     BucketLogsPermission = Shapes::StringShape.new(name: 'BucketLogsPermission')
     BucketName = Shapes::StringShape.new(name: 'BucketName')
+    BucketNamespace = Shapes::StringShape.new(name: 'BucketNamespace')
     BucketRegion = Shapes::StringShape.new(name: 'BucketRegion')
     BucketType = Shapes::StringShape.new(name: 'BucketType')
     BucketVersioningStatus = Shapes::StringShape.new(name: 'BucketVersioningStatus')
@@ -438,6 +440,7 @@ module Aws::S3
     NoSuchBucket = Shapes::StructureShape.new(name: 'NoSuchBucket')
     NoSuchKey = Shapes::StructureShape.new(name: 'NoSuchKey')
     NoSuchUpload = Shapes::StructureShape.new(name: 'NoSuchUpload')
+    NonEmptyKmsKeyArnString = Shapes::StringShape.new(name: 'NonEmptyKmsKeyArnString')
     NoncurrentVersionExpiration = Shapes::StructureShape.new(name: 'NoncurrentVersionExpiration')
     NoncurrentVersionTransition = Shapes::StructureShape.new(name: 'NoncurrentVersionTransition')
     NoncurrentVersionTransitionList = Shapes::ListShape.new(name: 'NoncurrentVersionTransitionList', flattened: true)
@@ -450,6 +453,7 @@ module Aws::S3
     ObjectAttributes = Shapes::StringShape.new(name: 'ObjectAttributes')
     ObjectAttributesList = Shapes::ListShape.new(name: 'ObjectAttributesList')
     ObjectCannedACL = Shapes::StringShape.new(name: 'ObjectCannedACL')
+    ObjectEncryption = Shapes::UnionShape.new(name: 'ObjectEncryption')
     ObjectIdentifier = Shapes::StructureShape.new(name: 'ObjectIdentifier')
     ObjectIdentifierList = Shapes::ListShape.new(name: 'ObjectIdentifierList', flattened: true)
     ObjectKey = Shapes::StringShape.new(name: 'ObjectKey')
@@ -616,6 +620,7 @@ module Aws::S3
     SSECustomerKey = Shapes::StringShape.new(name: 'SSECustomerKey')
     SSECustomerKeyMD5 = Shapes::StringShape.new(name: 'SSECustomerKeyMD5')
     SSEKMS = Shapes::StructureShape.new(name: 'SSEKMS', locationName: "SSE-KMS")
+    SSEKMSEncryption = Shapes::StructureShape.new(name: 'SSEKMSEncryption', locationName: "SSE-KMS")
     SSEKMSEncryptionContext = Shapes::StringShape.new(name: 'SSEKMSEncryptionContext')
     SSEKMSKeyId = Shapes::StringShape.new(name: 'SSEKMSKeyId')
     SSES3 = Shapes::StructureShape.new(name: 'SSES3', locationName: "SSE-S3")
@@ -678,6 +683,8 @@ module Aws::S3
     URI = Shapes::StringShape.new(name: 'URI')
     UpdateBucketMetadataInventoryTableConfigurationRequest = Shapes::StructureShape.new(name: 'UpdateBucketMetadataInventoryTableConfigurationRequest')
     UpdateBucketMetadataJournalTableConfigurationRequest = Shapes::StructureShape.new(name: 'UpdateBucketMetadataJournalTableConfigurationRequest')
+    UpdateObjectEncryptionRequest = Shapes::StructureShape.new(name: 'UpdateObjectEncryptionRequest')
+    UpdateObjectEncryptionResponse = Shapes::StructureShape.new(name: 'UpdateObjectEncryptionResponse')
     UploadIdMarker = Shapes::StringShape.new(name: 'UploadIdMarker')
     UploadPartCopyOutput = Shapes::StructureShape.new(name: 'UploadPartCopyOutput')
     UploadPartCopyRequest = Shapes::StructureShape.new(name: 'UploadPartCopyRequest')
@@ -721,6 +728,8 @@ module Aws::S3
     AccessControlTranslation.add_member(:owner, Shapes::ShapeRef.new(shape: OwnerOverride, required: true, location_name: "Owner"))
     AccessControlTranslation.struct_class = Types::AccessControlTranslation
 
+    AccessDenied.struct_class = Types::AccessDenied
+
     AllowedHeaders.member = Shapes::ShapeRef.new(shape: AllowedHeader)
 
     AllowedMethods.member = Shapes::ShapeRef.new(shape: AllowedMethod)
@@ -1005,6 +1014,7 @@ module Aws::S3
     CreateBucketRequest.add_member(:grant_write_acp, Shapes::ShapeRef.new(shape: GrantWriteACP, location: "header", location_name: "x-amz-grant-write-acp"))
     CreateBucketRequest.add_member(:object_lock_enabled_for_bucket, Shapes::ShapeRef.new(shape: ObjectLockEnabledForBucket, location: "header", location_name: "x-amz-bucket-object-lock-enabled"))
     CreateBucketRequest.add_member(:object_ownership, Shapes::ShapeRef.new(shape: ObjectOwnership, location: "header", location_name: "x-amz-object-ownership"))
+    CreateBucketRequest.add_member(:bucket_namespace, Shapes::ShapeRef.new(shape: BucketNamespace, location: "header", location_name: "x-amz-bucket-namespace"))
     CreateBucketRequest.struct_class = Types::CreateBucketRequest
     CreateBucketRequest[:payload] = :create_bucket_configuration
     CreateBucketRequest[:payload_member] = CreateBucketRequest.member(:create_bucket_configuration)
@@ -2210,6 +2220,12 @@ module Aws::S3
 
     ObjectAttributesList.member = Shapes::ShapeRef.new(shape: ObjectAttributes)
 
+    ObjectEncryption.add_member(:ssekms, Shapes::ShapeRef.new(shape: SSEKMSEncryption, location_name: "SSE-KMS"))
+    ObjectEncryption.add_member(:unknown, Shapes::ShapeRef.new(shape: nil, location_name: 'unknown'))
+    ObjectEncryption.add_member_subclass(:ssekms, Types::ObjectEncryption::Ssekms)
+    ObjectEncryption.add_member_subclass(:unknown, Types::ObjectEncryption::Unknown)
+    ObjectEncryption.struct_class = Types::ObjectEncryption
+
     ObjectIdentifier.add_member(:key, Shapes::ShapeRef.new(shape: ObjectKey, required: true, location_name: "Key"))
     ObjectIdentifier.add_member(:version_id, Shapes::ShapeRef.new(shape: ObjectVersionId, location_name: "VersionId"))
     ObjectIdentifier.add_member(:etag, Shapes::ShapeRef.new(shape: ETag, location_name: "ETag"))
@@ -2830,6 +2846,10 @@ module Aws::S3
     SSEKMS.add_member(:key_id, Shapes::ShapeRef.new(shape: SSEKMSKeyId, required: true, location_name: "KeyId"))
     SSEKMS.struct_class = Types::SSEKMS
 
+    SSEKMSEncryption.add_member(:kms_key_arn, Shapes::ShapeRef.new(shape: NonEmptyKmsKeyArnString, required: true, location_name: "KMSKeyArn"))
+    SSEKMSEncryption.add_member(:bucket_key_enabled, Shapes::ShapeRef.new(shape: BucketKeyEnabled, location_name: "BucketKeyEnabled"))
+    SSEKMSEncryption.struct_class = Types::SSEKMSEncryption
+
     SSES3.struct_class = Types::SSES3
 
     ScanRange.add_member(:start, Shapes::ShapeRef.new(shape: Start, location_name: "Start"))
@@ -2978,6 +2998,21 @@ module Aws::S3
     UpdateBucketMetadataJournalTableConfigurationRequest[:payload] = :journal_table_configuration
     UpdateBucketMetadataJournalTableConfigurationRequest[:payload_member] = UpdateBucketMetadataJournalTableConfigurationRequest.member(:journal_table_configuration)
 
+    UpdateObjectEncryptionRequest.add_member(:bucket, Shapes::ShapeRef.new(shape: BucketName, required: true, location: "uri", location_name: "Bucket", metadata: {"contextParam" => {"name" => "Bucket"}}))
+    UpdateObjectEncryptionRequest.add_member(:key, Shapes::ShapeRef.new(shape: ObjectKey, required: true, location: "uri", location_name: "Key"))
+    UpdateObjectEncryptionRequest.add_member(:version_id, Shapes::ShapeRef.new(shape: ObjectVersionId, location: "querystring", location_name: "versionId"))
+    UpdateObjectEncryptionRequest.add_member(:object_encryption, Shapes::ShapeRef.new(shape: ObjectEncryption, required: true, location_name: "ObjectEncryption", metadata: {"xmlNamespace" => {"uri" => "http://s3.amazonaws.com/doc/2006-03-01/"}}))
+    UpdateObjectEncryptionRequest.add_member(:request_payer, Shapes::ShapeRef.new(shape: RequestPayer, location: "header", location_name: "x-amz-request-payer"))
+    UpdateObjectEncryptionRequest.add_member(:expected_bucket_owner, Shapes::ShapeRef.new(shape: AccountId, location: "header", location_name: "x-amz-expected-bucket-owner"))
+    UpdateObjectEncryptionRequest.add_member(:content_md5, Shapes::ShapeRef.new(shape: ContentMD5, location: "header", location_name: "Content-MD5"))
+    UpdateObjectEncryptionRequest.add_member(:checksum_algorithm, Shapes::ShapeRef.new(shape: ChecksumAlgorithm, location: "header", location_name: "x-amz-sdk-checksum-algorithm"))
+    UpdateObjectEncryptionRequest.struct_class = Types::UpdateObjectEncryptionRequest
+    UpdateObjectEncryptionRequest[:payload] = :object_encryption
+    UpdateObjectEncryptionRequest[:payload_member] = UpdateObjectEncryptionRequest.member(:object_encryption)
+
+    UpdateObjectEncryptionResponse.add_member(:request_charged, Shapes::ShapeRef.new(shape: RequestCharged, location: "header", location_name: "x-amz-request-charged"))
+    UpdateObjectEncryptionResponse.struct_class = Types::UpdateObjectEncryptionResponse
+
     UploadPartCopyOutput.add_member(:copy_source_version_id, Shapes::ShapeRef.new(shape: CopySourceVersionId, location: "header", location_name: "x-amz-copy-source-version-id"))
     UploadPartCopyOutput.add_member(:copy_part_result, Shapes::ShapeRef.new(shape: CopyPartResult, location_name: "CopyPartResult"))
     UploadPartCopyOutput.add_member(:server_side_encryption, Shapes::ShapeRef.new(shape: ServerSideEncryption, location: "header", location_name: "x-amz-server-side-encryption"))
@@ -4298,6 +4333,25 @@ module Aws::S3
         o.output = Shapes::ShapeRef.new(shape: Shapes::StructureShape.new(struct_class: Aws::EmptyStructure))
       end)
 
+      api.add_operation(:update_object_encryption, Seahorse::Model::Operation.new.tap do |o|
+        o.name = "UpdateObjectEncryption"
+        o.http_method = "PUT"
+        o.http_request_uri = "/{Key+}?encryption"
+        o.http_checksum = {
+          "requestAlgorithmMember" => "checksum_algorithm",
+          "requestChecksumRequired" => true,
+        }
+        o.http_checksum = {
+          "requestAlgorithmMember" => "checksum_algorithm",
+          "requestChecksumRequired" => true,
+        }
+        o.input = Shapes::ShapeRef.new(shape: UpdateObjectEncryptionRequest)
+        o.output = Shapes::ShapeRef.new(shape: UpdateObjectEncryptionResponse)
+        o.errors << Shapes::ShapeRef.new(shape: NoSuchKey)
+        o.errors << Shapes::ShapeRef.new(shape: InvalidRequest)
+        o.errors << Shapes::ShapeRef.new(shape: AccessDenied)
+      end)
+
       api.add_operation(:upload_part, Seahorse::Model::Operation.new.tap do |o|
         o.name = "UploadPart"
         o.http_method = "PUT"

data/lib/aws-sdk-s3/customizations.rb

@@ -7,15 +7,21 @@ module Aws
     autoload :Encryption, 'aws-sdk-s3/encryption'
     autoload :EncryptionV2, 'aws-sdk-s3/encryption_v2'
     autoload :EncryptionV3, 'aws-sdk-s3/encryption_v3'
-    autoload :FilePart, 'aws-sdk-s3/file_part'
+    autoload :LegacySigner, 'aws-sdk-s3/legacy_signer'
+
+    # transfer manager + multipart upload/download utilities
     autoload :DefaultExecutor, 'aws-sdk-s3/default_executor'
+    autoload :FilePart, 'aws-sdk-s3/file_part'
     autoload :FileUploader, 'aws-sdk-s3/file_uploader'
     autoload :FileDownloader, 'aws-sdk-s3/file_downloader'
-    autoload :LegacySigner, 'aws-sdk-s3/legacy_signer'
     autoload :MultipartDownloadError, 'aws-sdk-s3/multipart_download_error'
     autoload :MultipartFileUploader, 'aws-sdk-s3/multipart_file_uploader'
     autoload :MultipartStreamUploader, 'aws-sdk-s3/multipart_stream_uploader'
     autoload :MultipartUploadError, 'aws-sdk-s3/multipart_upload_error'
+    autoload :DirectoryDownloadError, 'aws-sdk-s3/directory_download_error'
+    autoload :DirectoryDownloader, 'aws-sdk-s3/directory_downloader'
+    autoload :DirectoryUploadError, 'aws-sdk-s3/directory_upload_error'
+    autoload :DirectoryUploader, 'aws-sdk-s3/directory_uploader'
     autoload :ObjectCopier, 'aws-sdk-s3/object_copier'
     autoload :ObjectMultipartCopier, 'aws-sdk-s3/object_multipart_copier'
     autoload :PresignedPost, 'aws-sdk-s3/presigned_post'

data/lib/aws-sdk-s3/directory_download_error.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    # Raised when DirectoryDownloader fails to download objects from S3 bucket
+    class DirectoryDownloadError < StandardError
+      def initialize(message, errors = [])
+        @errors = errors
+        super(message)
+      end
+
+      # @return [Array<StandardError>] The list of errors encountered when downloading objects
+      attr_reader :errors
+    end
+  end
+end

data/lib/aws-sdk-s3/directory_downloader.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    # @api private
+    # This is a one-shot class that downloads objects from a bucket to a local directory.
+    # This works as follows:
+    # * ObjectProducer runs in a background thread, calling `list_objects_v2` and
+    #   pushing entries into a SizedQueue (max: 100).
+    # * An internal executor pulls from that queue and posts work. Each task uses
+    #   FileDownloader to download objects then signals completion via `completion_queue`.
+    #
+    # We track how many tasks we posted, then pop that many times from `completion_queue`
+    # to wait for everything to finish.
+    #
+    # Errors are collected in a mutex-protected array. On failure (unless ignore_failure is set),
+    # we call abort which closes the queue - the producer catches ClosedQueueError and exits cleanly.
+    class DirectoryDownloader
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+        @executor = options[:executor] || DefaultExecutor.new
+        @logger = options[:logger]
+        @producer = nil
+        @mutex = Mutex.new
+      end
+
+      attr_reader :client, :executor
+
+      def abort
+        @producer&.close
+      end
+
+      def download(destination, bucket:, **options)
+        if File.exist?(destination)
+          raise ArgumentError, 'invalid destination, expected a directory' unless File.directory?(destination)
+        else
+          FileUtils.mkdir_p(destination)
+        end
+
+        download_opts = build_download_opts(destination, options)
+        @producer = ObjectProducer.new(build_producer_opts(destination, bucket, options))
+        downloader = FileDownloader.new(client: @client, executor: @executor)
+        downloads, errors = process_download_queue(downloader, download_opts)
+        build_result(downloads, errors)
+      end
+
+      private
+
+      def build_download_opts(destination, opts)
+        {
+          destination: destination,
+          ignore_failure: opts[:ignore_failure] || false
+        }
+      end
+
+      def build_producer_opts(destination, bucket, opts)
+        {
+          client: @client,
+          directory_downloader: self,
+          destination: destination,
+          bucket: bucket,
+          s3_prefix: opts[:s3_prefix],
+          filter_callback: opts[:filter_callback],
+          request_callback: opts[:request_callback]
+        }
+      end
+
+      def build_result(download_count, errors)
+        if @producer&.closed?
+          msg = "directory download failed: #{errors.map(&:message).join('; ')}"
+          raise DirectoryDownloadError.new(msg, errors)
+        else
+          {
+            completed_downloads: [download_count - errors.count, 0].max,
+            failed_downloads: errors.count,
+            errors: errors.any? ? errors : nil
+          }.compact
+        end
+      end
+
+      def download_object(entry, downloader, errors, opts)
+        raise entry.error if entry.error
+
+        FileUtils.mkdir_p(File.dirname(entry.path)) unless Dir.exist?(File.dirname(entry.path))
+        downloader.download(entry.path, entry.params)
+        @logger&.debug("Downloaded #{entry.params[:key]} from #{entry.params[:bucket]} to #{entry.path}")
+      rescue StandardError => e
+        @logger&.warn("Failed to download #{entry.params[:key]} from #{entry.params[:bucket]}: #{e.message}")
+        @mutex.synchronize { errors << e }
+        abort unless opts[:ignore_failure]
+      end
+
+      def process_download_queue(downloader, opts)
+        queue_executor = DefaultExecutor.new(max_threads: 2)
+        completion_queue = Queue.new
+        posted_count = 0
+        errors = []
+        begin
+          @producer.each do |object|
+            queue_executor.post(object) do |o|
+              download_object(o, downloader, errors, opts)
+            ensure
+              completion_queue << :done
+            end
+            posted_count += 1
+          end
+        rescue ClosedQueueError
+          # abort already requested
+        rescue StandardError => e
+          @mutex.synchronize { errors << e }
+          abort
+        end
+        posted_count.times { completion_queue.pop }
+        [posted_count, errors]
+      ensure
+        queue_executor&.shutdown
+      end
+
+      # @api private
+      class ObjectProducer
+        include Enumerable
+
+        DEFAULT_QUEUE_SIZE = 100
+        DONE_MARKER = :done
+
+        def initialize(opts = {})
+          @directory_downloader = opts[:directory_downloader]
+          @destination_dir = opts[:destination]
+          @bucket = opts[:bucket]
+          @client = opts[:client]
+          @s3_prefix = opts[:s3_prefix]
+          @filter_callback = opts[:filter_callback]
+          @request_callback = opts[:request_callback]
+          @object_queue = SizedQueue.new(DEFAULT_QUEUE_SIZE)
+        end
+
+        def closed?
+          @object_queue.closed?
+        end
+
+        def close
+          @object_queue.close
+          @object_queue.clear
+        end
+
+        def each
+          producer_thread = Thread.new do
+            stream_objects
+            @object_queue << DONE_MARKER
+          rescue ClosedQueueError
+            # abort requested
+          rescue StandardError => e
+            close
+            raise e
+          end
+
+          while (object = @object_queue.shift) && object != DONE_MARKER
+            yield object
+          end
+        ensure
+          producer_thread.value
+        end
+
+        private
+
+        def apply_request_callback(key, params)
+          callback_params = @request_callback.call(key, params.dup)
+          return params unless callback_params.is_a?(Hash) && callback_params.any?
+
+          params.merge(callback_params)
+        end
+
+        def build_object_entry(key)
+          params = { bucket: @bucket, key: key }
+          params = apply_request_callback(key, params) if @request_callback
+          error = validate_key(key)
+          return DownloadEntry.new(path: '', params: params, error: error) if error
+
+          full_path = normalize_path(File.join(@destination_dir, key))
+          DownloadEntry.new(path: full_path, params: params, error: error)
+        end
+
+        def include_object?(obj)
+          return true unless @filter_callback
+
+          @filter_callback.call(obj)
+        end
+
+        def directory_marker?(obj)
+          obj.key.end_with?('/') && obj.size.zero?
+        end
+
+        def normalize_path(path)
+          return path if File::SEPARATOR == '/'
+
+          path.tr('/', File::SEPARATOR)
+        end
+
+        def stream_objects(continuation_token: nil)
+          resp = @client.list_objects_v2(bucket: @bucket, prefix: @s3_prefix, continuation_token: continuation_token)
+          resp.contents&.each do |o|
+            next if directory_marker?(o)
+            next unless include_object?(o)
+
+            @object_queue << build_object_entry(o.key)
+          end
+          stream_objects(continuation_token: resp.next_continuation_token) if resp.next_continuation_token
+        end
+
+        def validate_key(key)
+          segments = key.split('/')
+          return unless segments.any? { |s| %w[. ..].include?(s) }
+
+          DirectoryDownloadError.new("invalid key '#{key}': contains '.' or '..' path segments")
+        end
+
+        # @api private
+        class DownloadEntry
+          def initialize(opts = {})
+            @path = opts[:path]
+            @params = opts[:params]
+            @error = opts[:error]
+          end
+
+          attr_reader :path, :params, :error
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/directory_upload_error.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    # Raised when DirectoryUploader fails to upload files to S3 bucket
+    class DirectoryUploadError < StandardError
+      def initialize(message, errors = [])
+        @errors = errors
+        super(message)
+      end
+
+      # @return [Array<StandardError>] The list of errors encountered when uploading files
+      attr_reader :errors
+    end
+  end
+end