aws-sdk-s3 1.156.0 → 1.190.0

data/lib/aws-sdk-s3/errors.rb

@@ -29,12 +29,17 @@ module Aws::S3
   # ## Error Classes
   # * {BucketAlreadyExists}
   # * {BucketAlreadyOwnedByYou}
+  # * {EncryptionTypeMismatch}
+  # * {IdempotencyParameterMismatch}
   # * {InvalidObjectState}
+  # * {InvalidRequest}
+  # * {InvalidWriteOffset}
   # * {NoSuchBucket}
   # * {NoSuchKey}
   # * {NoSuchUpload}
   # * {ObjectAlreadyInActiveTierError}
   # * {ObjectNotInActiveTierError}
+  # * {TooManyParts}
   #
   # Additionally, error classes are dynamically generated for service errors based on the error code
   # if they are not defined above.
@@ -62,6 +67,26 @@ module Aws::S3
       end
     end
 
+    class EncryptionTypeMismatch < ServiceError
+
+      # @param [Seahorse::Client::RequestContext] context
+      # @param [String] message
+      # @param [Aws::S3::Types::EncryptionTypeMismatch] data
+      def initialize(context, message, data = Aws::EmptyStructure.new)
+        super(context, message, data)
+      end
+    end
+
+    class IdempotencyParameterMismatch < ServiceError
+
+      # @param [Seahorse::Client::RequestContext] context
+      # @param [String] message
+      # @param [Aws::S3::Types::IdempotencyParameterMismatch] data
+      def initialize(context, message, data = Aws::EmptyStructure.new)
+        super(context, message, data)
+      end
+    end
+
     class InvalidObjectState < ServiceError
 
       # @param [Seahorse::Client::RequestContext] context
@@ -82,6 +107,26 @@ module Aws::S3
       end
     end
 
+    class InvalidRequest < ServiceError
+
+      # @param [Seahorse::Client::RequestContext] context
+      # @param [String] message
+      # @param [Aws::S3::Types::InvalidRequest] data
+      def initialize(context, message, data = Aws::EmptyStructure.new)
+        super(context, message, data)
+      end
+    end
+
+    class InvalidWriteOffset < ServiceError
+
+      # @param [Seahorse::Client::RequestContext] context
+      # @param [String] message
+      # @param [Aws::S3::Types::InvalidWriteOffset] data
+      def initialize(context, message, data = Aws::EmptyStructure.new)
+        super(context, message, data)
+      end
+    end
+
     class NoSuchBucket < ServiceError
 
       # @param [Seahorse::Client::RequestContext] context
@@ -132,5 +177,18 @@ module Aws::S3
       end
     end
 
+    class TooManyParts < ServiceError
+
+      # @param [Seahorse::Client::RequestContext] context
+      # @param [String] message
+      # @param [Aws::S3::Types::TooManyParts] data
+      def initialize(context, message, data = Aws::EmptyStructure.new)
+        super(context, message, data)
+      end
+    end
+
   end
 end
+
+# Load customizations if they exist
+require 'aws-sdk-s3/customizations/errors'

data/lib/aws-sdk-s3/file_downloader.rb

@@ -28,18 +28,10 @@ module Aws
         @chunk_size = options[:chunk_size]
         @params = {
           bucket: options[:bucket],
-          key: options[:key],
+          key: options[:key]
         }
         @params[:version_id] = options[:version_id] if options[:version_id]
-
-        # checksum_mode only supports the value "ENABLED"
-        # falsey values (false/nil) or "DISABLED" should be considered
-        # disabled and the api parameter should be unset.
-        if (checksum_mode = options.fetch(:checksum_mode, 'ENABLED'))
-          @params[:checksum_mode] = checksum_mode unless checksum_mode.upcase == 'DISABLED'
-        end
         @on_checksum_validated = options[:on_checksum_validated]
-
         @progress_callback = options[:progress_callback]
 
         validate!
@@ -51,7 +43,7 @@ module Aws
           when 'get_range'
             if @chunk_size
               resp = @client.head_object(@params)
-              multithreaded_get_by_ranges(resp.content_length)
+              multithreaded_get_by_ranges(resp.content_length, resp.etag)
             else
               msg = 'In :get_range mode, :chunk_size must be provided'
               raise ArgumentError, msg
@@ -67,11 +59,6 @@ module Aws
       private
 
       def validate!
-        if @on_checksum_validated && @params[:checksum_mode] != 'ENABLED'
-          raise ArgumentError, "You must set checksum_mode: 'ENABLED' " +
-            "when providing a on_checksum_validated callback"
-        end
-
         if @on_checksum_validated && !@on_checksum_validated.respond_to?(:call)
           raise ArgumentError, 'on_checksum_validated must be callable'
         end
@@ -84,7 +71,7 @@ module Aws
           if resp.content_length <= MIN_CHUNK_SIZE
             single_request
           else
-            multithreaded_get_by_ranges(resp.content_length)
+            multithreaded_get_by_ranges(resp.content_length, resp.etag)
           end
         else
           # partNumber is an option
@@ -92,18 +79,18 @@ module Aws
           if resp.content_length <= MIN_CHUNK_SIZE
             single_request
           else
-            compute_mode(resp.content_length, count)
+            compute_mode(resp.content_length, count, resp.etag)
           end
         end
       end
 
-      def compute_mode(file_size, count)
+      def compute_mode(file_size, count, etag)
         chunk_size = compute_chunk(file_size)
         part_size = (file_size.to_f / count.to_f).ceil
         if chunk_size < part_size
-          multithreaded_get_by_ranges(file_size)
+          multithreaded_get_by_ranges(file_size, etag)
         else
-          multithreaded_get_by_parts(count, file_size)
+          multithreaded_get_by_parts(count, file_size, etag)
         end
       end
 
@@ -135,7 +122,7 @@ module Aws
         chunks.each_slice(@thread_count).to_a
       end
 
-      def multithreaded_get_by_ranges(file_size)
+      def multithreaded_get_by_ranges(file_size, etag)
         offset = 0
         default_chunk_size = compute_chunk(file_size)
         chunks = []
@@ -147,7 +134,7 @@ module Aws
           chunks << Part.new(
             part_number: part_number,
             size: (progress-offset),
-            params: @params.merge(range: range)
+            params: @params.merge(range: range, if_match: etag)
           )
           part_number += 1
           offset = progress
@@ -155,18 +142,16 @@ module Aws
         download_in_threads(PartList.new(chunks), file_size)
       end
 
-      def multithreaded_get_by_parts(n_parts, total_size)
+      def multithreaded_get_by_parts(n_parts, total_size, etag)
         parts = (1..n_parts).map do |part|
-          Part.new(part_number: part, params: @params.merge(part_number: part))
+          Part.new(part_number: part, params: @params.merge(part_number: part, if_match: etag))
         end
         download_in_threads(PartList.new(parts), total_size)
       end
 
       def download_in_threads(pending, total_size)
         threads = []
-        if @progress_callback
-          progress = MultipartProgress.new(pending, total_size, @progress_callback)
-        end
+        progress = MultipartProgress.new(pending, total_size, @progress_callback) if @progress_callback
         @thread_count.times do
           thread = Thread.new do
             begin
@@ -208,9 +193,7 @@ module Aws
 
         return resp unless @on_checksum_validated
 
-        if resp.checksum_validated
-          @on_checksum_validated.call(resp.checksum_validated, resp)
-        end
+        @on_checksum_validated.call(resp.checksum_validated, resp) if resp.checksum_validated
 
         resp
       end
@@ -251,7 +234,7 @@ module Aws
       end
 
       # @api private
-      class  MultipartProgress
+      class MultipartProgress
         def initialize(parts, total_size, progress_callback)
           @bytes_received = Array.new(parts.size, 0)
           @part_sizes = parts.map(&:size)

data/lib/aws-sdk-s3/legacy_signer.rb

@@ -3,7 +3,8 @@
 require 'set'
 require 'time'
 require 'openssl'
-require 'cgi'
+require "cgi/escape"
+require "cgi/util" if RUBY_VERSION < "3.5"
 require 'aws-sdk-core/query'
 
 module Aws

data/lib/aws-sdk-s3/multipart_file_uploader.rb

@@ -16,7 +16,6 @@ module Aws
 
       THREAD_COUNT = 10
 
-      # @api private
       CREATE_OPTIONS = Set.new(
         Client.api.operation(:create_multipart_upload).input.shape.member_names
       )
@@ -25,11 +24,16 @@ module Aws
         Client.api.operation(:complete_multipart_upload).input.shape.member_names
       )
 
-      # @api private
       UPLOAD_PART_OPTIONS = Set.new(
         Client.api.operation(:upload_part).input.shape.member_names
       )
 
+      CHECKSUM_KEYS = Set.new(
+        Client.api.operation(:upload_part).input.shape.members.map do |n, s|
+          n if s.location == 'header' && s.location_name.start_with?('x-amz-checksum-')
+        end.compact
+      )
+
       # @option options [Client] :client
       # @option options [Integer] :thread_count (THREAD_COUNT)
       def initialize(options = {})
@@ -121,15 +125,27 @@ module Aws
         parts
       end
 
+      def checksum_key?(key)
+        CHECKSUM_KEYS.include?(key)
+      end
+
+      def has_checksum_key?(keys)
+        keys.any? { |key| checksum_key?(key) }
+      end
+
       def create_opts(options)
-        CREATE_OPTIONS.inject({}) do |hash, key|
+        opts = { checksum_algorithm: Aws::Plugins::ChecksumAlgorithm::DEFAULT_CHECKSUM }
+        opts[:checksum_type] = 'FULL_OBJECT' if has_checksum_key?(options.keys)
+        CREATE_OPTIONS.inject(opts) do |hash, key|
           hash[key] = options[key] if options.key?(key)
           hash
         end
       end
 
       def complete_opts(options)
-        COMPLETE_OPTIONS.inject({}) do |hash, key|
+        opts = {}
+        opts[:checksum_type] = 'FULL_OBJECT' if has_checksum_key?(options.keys)
+        COMPLETE_OPTIONS.inject(opts) do |hash, key|
           hash[key] = options[key] if options.key?(key)
           hash
         end
@@ -137,7 +153,10 @@ module Aws
 
       def upload_part_opts(options)
         UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
-          hash[key] = options[key] if options.key?(key)
+          if options.key?(key)
+            # don't pass through checksum calculations
+            hash[key] = options[key] unless checksum_key?(key)
+          end
           hash
         end
       end
@@ -159,14 +178,13 @@ module Aws
                 end
                 resp = @client.upload_part(part)
                 part[:body].close
-                completed_part = {etag: resp.etag, part_number: part[:part_number]}
-
-                # get the requested checksum from the response
-                if part[:checksum_algorithm]
-                  k = "checksum_#{part[:checksum_algorithm].downcase}".to_sym
-                  completed_part[k] = resp[k]
-                end
-
+                completed_part = {
+                  etag: resp.etag,
+                  part_number: part[:part_number]
+                }
+                algorithm = resp.context.params[:checksum_algorithm]
+                k = "checksum_#{algorithm.downcase}".to_sym
+                completed_part[k] = resp.send(k)
                 completed.push(completed_part)
               end
               nil

data/lib/aws-sdk-s3/multipart_upload.rb

@@ -70,8 +70,10 @@ module Aws::S3
 
     # The class of storage used to store the object.
     #
-    # <note markdown="1"> **Directory buckets** - Only the S3 Express One Zone storage class is
-    # supported by directory buckets to store objects.
+    # <note markdown="1"> **Directory buckets** - Directory buckets only support
+    # `EXPRESS_ONEZONE` (the S3 Express One Zone storage class) in
+    # Availability Zones and `ONEZONE_IA` (the S3 One Zone-Infrequent Access
+    # storage class) in Dedicated Local Zones.
     #
     #  </note>
     # @return [String]
@@ -103,6 +105,18 @@ module Aws::S3
       data[:checksum_algorithm]
     end
 
+    # The checksum type that is used to calculate the object’s checksum
+    # value. For more information, see [Checking object integrity][1] in the
+    # *Amazon S3 User Guide*.
+    #
+    #
+    #
+    # [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+    # @return [String]
+    def checksum_type
+      data[:checksum_type]
+    end
+
     # @!endgroup
 
     # @return [Client]
@@ -239,6 +253,7 @@ module Aws::S3
     #   multipart_upload.abort({
     #     request_payer: "requester", # accepts requester
     #     expected_bucket_owner: "AccountId",
+    #     if_match_initiated_time: Time.now,
     #   })
     # @param [Hash] options ({})
     # @option options [String] :request_payer
@@ -261,6 +276,17 @@ module Aws::S3
     #   The account ID of the expected bucket owner. If the account ID that
     #   you provide does not match the actual owner of the bucket, the request
     #   fails with the HTTP status code `403 Forbidden` (access denied).
+    # @option options [Time,DateTime,Date,Integer,String] :if_match_initiated_time
+    #   If present, this header aborts an in progress multipart upload only if
+    #   it was initiated on the provided timestamp. If the initiated timestamp
+    #   of the multipart upload does not match the provided value, the
+    #   operation returns a `412 Precondition Failed` error. If the initiated
+    #   timestamp matches or if the multipart upload doesn’t exist, the
+    #   operation returns a `204 Success (No Content)` response.
+    #
+    #   <note markdown="1"> This functionality is only supported for directory buckets.
+    #
+    #    </note>
     # @return [Types::AbortMultipartUploadOutput]
     def abort(options = {})
       options = options.merge(
@@ -283,6 +309,7 @@ module Aws::S3
     #           etag: "ETag",
     #           checksum_crc32: "ChecksumCRC32",
     #           checksum_crc32c: "ChecksumCRC32C",
+    #           checksum_crc64nvme: "ChecksumCRC64NVME",
     #           checksum_sha1: "ChecksumSHA1",
     #           checksum_sha256: "ChecksumSHA256",
     #           part_number: 1,
@@ -291,10 +318,15 @@ module Aws::S3
     #     },
     #     checksum_crc32: "ChecksumCRC32",
     #     checksum_crc32c: "ChecksumCRC32C",
+    #     checksum_crc64nvme: "ChecksumCRC64NVME",
     #     checksum_sha1: "ChecksumSHA1",
     #     checksum_sha256: "ChecksumSHA256",
+    #     checksum_type: "COMPOSITE", # accepts COMPOSITE, FULL_OBJECT
+    #     mpu_object_size: 1,
     #     request_payer: "requester", # accepts requester
     #     expected_bucket_owner: "AccountId",
+    #     if_match: "IfMatch",
+    #     if_none_match: "IfNoneMatch",
     #     sse_customer_algorithm: "SSECustomerAlgorithm",
     #     sse_customer_key: "SSECustomerKey",
     #     sse_customer_key_md5: "SSECustomerKeyMD5",
@@ -305,9 +337,9 @@ module Aws::S3
     # @option options [String] :checksum_crc32
     #   This header can be used as a data integrity check to verify that the
     #   data received is the same data that was originally sent. This header
-    #   specifies the base64-encoded, 32-bit CRC32 checksum of the object. For
-    #   more information, see [Checking object integrity][1] in the *Amazon S3
-    #   User Guide*.
+    #   specifies the Base64 encoded, 32-bit `CRC32` checksum of the object.
+    #   For more information, see [Checking object integrity][1] in the
+    #   *Amazon S3 User Guide*.
     #
     #
     #
@@ -315,17 +347,28 @@ module Aws::S3
     # @option options [String] :checksum_crc32c
     #   This header can be used as a data integrity check to verify that the
     #   data received is the same data that was originally sent. This header
-    #   specifies the base64-encoded, 32-bit CRC32C checksum of the object.
+    #   specifies the Base64 encoded, 32-bit `CRC32C` checksum of the object.
     #   For more information, see [Checking object integrity][1] in the
     #   *Amazon S3 User Guide*.
     #
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+    # @option options [String] :checksum_crc64nvme
+    #   This header can be used as a data integrity check to verify that the
+    #   data received is the same data that was originally sent. This header
+    #   specifies the Base64 encoded, 64-bit `CRC64NVME` checksum of the
+    #   object. The `CRC64NVME` checksum is always a full object checksum. For
+    #   more information, see [Checking object integrity in the Amazon S3 User
+    #   Guide][1].
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
     # @option options [String] :checksum_sha1
     #   This header can be used as a data integrity check to verify that the
     #   data received is the same data that was originally sent. This header
-    #   specifies the base64-encoded, 160-bit SHA-1 digest of the object. For
+    #   specifies the Base64 encoded, 160-bit `SHA1` digest of the object. For
     #   more information, see [Checking object integrity][1] in the *Amazon S3
     #   User Guide*.
     #
@@ -335,13 +378,28 @@ module Aws::S3
     # @option options [String] :checksum_sha256
     #   This header can be used as a data integrity check to verify that the
     #   data received is the same data that was originally sent. This header
-    #   specifies the base64-encoded, 256-bit SHA-256 digest of the object.
+    #   specifies the Base64 encoded, 256-bit `SHA256` digest of the object.
     #   For more information, see [Checking object integrity][1] in the
     #   *Amazon S3 User Guide*.
     #
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+    # @option options [String] :checksum_type
+    #   This header specifies the checksum type of the object, which
+    #   determines how part-level checksums are combined to create an
+    #   object-level checksum for multipart objects. You can use this header
+    #   as a data integrity check to verify that the checksum type that is
+    #   received is the same checksum that was specified. If the checksum type
+    #   doesn’t match the checksum type that was specified for the object
+    #   during the `CreateMultipartUpload` request, it’ll result in a
+    #   `BadDigest` error. For more information, see Checking object integrity
+    #   in the Amazon S3 User Guide.
+    # @option options [Integer] :mpu_object_size
+    #   The expected total object size of the multipart upload request. If
+    #   there’s a mismatch between the specified object size value and the
+    #   actual object size value, it results in an `HTTP 400 InvalidRequest`
+    #   error.
     # @option options [String] :request_payer
     #   Confirms that the requester knows that they will be charged for the
     #   request. Bucket owners need not specify this parameter in their
@@ -362,6 +420,45 @@ module Aws::S3
     #   The account ID of the expected bucket owner. If the account ID that
     #   you provide does not match the actual owner of the bucket, the request
     #   fails with the HTTP status code `403 Forbidden` (access denied).
+    # @option options [String] :if_match
+    #   Uploads the object only if the ETag (entity tag) value provided during
+    #   the WRITE operation matches the ETag of the object in S3. If the ETag
+    #   values do not match, the operation returns a `412 Precondition Failed`
+    #   error.
+    #
+    #   If a conflicting operation occurs during the upload S3 returns a `409
+    #   ConditionalRequestConflict` response. On a 409 failure you should
+    #   fetch the object's ETag, re-initiate the multipart upload with
+    #   `CreateMultipartUpload`, and re-upload each part.
+    #
+    #   Expects the ETag value as a string.
+    #
+    #   For more information about conditional requests, see [RFC 7232][1], or
+    #   [Conditional requests][2] in the *Amazon S3 User Guide*.
+    #
+    #
+    #
+    #   [1]: https://tools.ietf.org/html/rfc7232
+    #   [2]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-requests.html
+    # @option options [String] :if_none_match
+    #   Uploads the object only if the object key name does not already exist
+    #   in the bucket specified. Otherwise, Amazon S3 returns a `412
+    #   Precondition Failed` error.
+    #
+    #   If a conflicting operation occurs during the upload S3 returns a `409
+    #   ConditionalRequestConflict` response. On a 409 failure you should
+    #   re-initiate the multipart upload with `CreateMultipartUpload` and
+    #   re-upload each part.
+    #
+    #   Expects the '*' (asterisk) character.
+    #
+    #   For more information about conditional requests, see [RFC 7232][1], or
+    #   [Conditional requests][2] in the *Amazon S3 User Guide*.
+    #
+    #
+    #
+    #   [1]: https://tools.ietf.org/html/rfc7232
+    #   [2]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-requests.html
     # @option options [String] :sse_customer_algorithm
     #   The server-side encryption (SSE) algorithm used to encrypt the object.
     #   This parameter is required only when the object was created using a
@@ -589,3 +686,6 @@ module Aws::S3
     class Collection < Aws::Resources::Collection; end
   end
 end
+
+# Load customizations if they exist
+require 'aws-sdk-s3/customizations/multipart_upload'