aws-sdk-s3 1.176.1 → 1.180.0

data/lib/aws-sdk-s3/file_downloader.rb

@@ -28,18 +28,10 @@ module Aws
         @chunk_size = options[:chunk_size]
         @params = {
           bucket: options[:bucket],
-          key: options[:key],
+          key: options[:key]
         }
         @params[:version_id] = options[:version_id] if options[:version_id]
-
-        # checksum_mode only supports the value "ENABLED"
-        # falsey values (false/nil) or "DISABLED" should be considered
-        # disabled and the api parameter should be unset.
-        if (checksum_mode = options.fetch(:checksum_mode, 'ENABLED'))
-          @params[:checksum_mode] = checksum_mode unless checksum_mode.upcase == 'DISABLED'
-        end
         @on_checksum_validated = options[:on_checksum_validated]
-
         @progress_callback = options[:progress_callback]
 
         validate!
@@ -67,11 +59,6 @@ module Aws
       private
 
       def validate!
-        if @on_checksum_validated && @params[:checksum_mode] != 'ENABLED'
-          raise ArgumentError, "You must set checksum_mode: 'ENABLED' " +
-            "when providing a on_checksum_validated callback"
-        end
-
         if @on_checksum_validated && !@on_checksum_validated.respond_to?(:call)
           raise ArgumentError, 'on_checksum_validated must be callable'
         end
@@ -164,9 +151,7 @@ module Aws
 
       def download_in_threads(pending, total_size)
         threads = []
-        if @progress_callback
-          progress = MultipartProgress.new(pending, total_size, @progress_callback)
-        end
+        progress = MultipartProgress.new(pending, total_size, @progress_callback) if @progress_callback
         @thread_count.times do
           thread = Thread.new do
             begin
@@ -208,9 +193,7 @@ module Aws
 
         return resp unless @on_checksum_validated
 
-        if resp.checksum_validated
-          @on_checksum_validated.call(resp.checksum_validated, resp)
-        end
+        @on_checksum_validated.call(resp.checksum_validated, resp) if resp.checksum_validated
 
         resp
       end
@@ -251,7 +234,7 @@ module Aws
       end
 
       # @api private
-      class  MultipartProgress
+      class MultipartProgress
         def initialize(parts, total_size, progress_callback)
           @bytes_received = Array.new(parts.size, 0)
           @part_sizes = parts.map(&:size)

data/lib/aws-sdk-s3/multipart_file_uploader.rb

@@ -16,7 +16,6 @@ module Aws
 
       THREAD_COUNT = 10
 
-      # @api private
       CREATE_OPTIONS = Set.new(
         Client.api.operation(:create_multipart_upload).input.shape.member_names
       )
@@ -25,11 +24,16 @@ module Aws
         Client.api.operation(:complete_multipart_upload).input.shape.member_names
       )
 
-      # @api private
       UPLOAD_PART_OPTIONS = Set.new(
         Client.api.operation(:upload_part).input.shape.member_names
       )
 
+      CHECKSUM_KEYS = Set.new(
+        Client.api.operation(:upload_part).input.shape.members.map do |n, s|
+          n if s.location == 'header' && s.location_name.start_with?('x-amz-checksum-')
+        end.compact
+      )
+
       # @option options [Client] :client
       # @option options [Integer] :thread_count (THREAD_COUNT)
       def initialize(options = {})
@@ -121,15 +125,27 @@ module Aws
         parts
       end
 
+      def checksum_key?(key)
+        CHECKSUM_KEYS.include?(key)
+      end
+
+      def has_checksum_key?(keys)
+        keys.any? { |key| checksum_key?(key) }
+      end
+
       def create_opts(options)
-        CREATE_OPTIONS.inject({}) do |hash, key|
+        opts = { checksum_algorithm: Aws::Plugins::ChecksumAlgorithm::DEFAULT_CHECKSUM }
+        opts[:checksum_type] = 'FULL_OBJECT' if has_checksum_key?(options.keys)
+        CREATE_OPTIONS.inject(opts) do |hash, key|
           hash[key] = options[key] if options.key?(key)
           hash
         end
       end
 
       def complete_opts(options)
-        COMPLETE_OPTIONS.inject({}) do |hash, key|
+        opts = {}
+        opts[:checksum_type] = 'FULL_OBJECT' if has_checksum_key?(options.keys)
+        COMPLETE_OPTIONS.inject(opts) do |hash, key|
           hash[key] = options[key] if options.key?(key)
           hash
         end
@@ -137,7 +153,10 @@ module Aws
 
       def upload_part_opts(options)
         UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
-          hash[key] = options[key] if options.key?(key)
+          if options.key?(key)
+            # don't pass through checksum calculations
+            hash[key] = options[key] unless checksum_key?(key)
+          end
           hash
         end
       end
@@ -159,14 +178,13 @@ module Aws
                 end
                 resp = @client.upload_part(part)
                 part[:body].close
-                completed_part = {etag: resp.etag, part_number: part[:part_number]}
-
-                # get the requested checksum from the response
-                if part[:checksum_algorithm]
-                  k = "checksum_#{part[:checksum_algorithm].downcase}".to_sym
-                  completed_part[k] = resp[k]
-                end
-
+                completed_part = {
+                  etag: resp.etag,
+                  part_number: part[:part_number]
+                }
+                algorithm = resp.context.params[:checksum_algorithm]
+                k = "checksum_#{algorithm.downcase}".to_sym
+                completed_part[k] = resp.send(k)
                 completed.push(completed_part)
               end
               nil

data/lib/aws-sdk-s3/multipart_upload.rb

@@ -103,6 +103,18 @@ module Aws::S3
       data[:checksum_algorithm]
     end
 
+    # The checksum type that is used to calculate the object’s checksum
+    # value. For more information, see [Checking object integrity][1] in the
+    # *Amazon S3 User Guide*.
+    #
+    #
+    #
+    # [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+    # @return [String]
+    def checksum_type
+      data[:checksum_type]
+    end
+
     # @!endgroup
 
     # @return [Client]
@@ -295,6 +307,7 @@ module Aws::S3
     #           etag: "ETag",
     #           checksum_crc32: "ChecksumCRC32",
     #           checksum_crc32c: "ChecksumCRC32C",
+    #           checksum_crc64nvme: "ChecksumCRC64NVME",
     #           checksum_sha1: "ChecksumSHA1",
     #           checksum_sha256: "ChecksumSHA256",
     #           part_number: 1,
@@ -303,8 +316,11 @@ module Aws::S3
     #     },
     #     checksum_crc32: "ChecksumCRC32",
     #     checksum_crc32c: "ChecksumCRC32C",
+    #     checksum_crc64nvme: "ChecksumCRC64NVME",
     #     checksum_sha1: "ChecksumSHA1",
     #     checksum_sha256: "ChecksumSHA256",
+    #     checksum_type: "COMPOSITE", # accepts COMPOSITE, FULL_OBJECT
+    #     mpu_object_size: 1,
     #     request_payer: "requester", # accepts requester
     #     expected_bucket_owner: "AccountId",
     #     if_match: "IfMatch",
@@ -319,7 +335,7 @@ module Aws::S3
     # @option options [String] :checksum_crc32
     #   This header can be used as a data integrity check to verify that the
     #   data received is the same data that was originally sent. This header
-    #   specifies the base64-encoded, 32-bit CRC-32 checksum of the object.
+    #   specifies the Base64 encoded, 32-bit `CRC-32` checksum of the object.
     #   For more information, see [Checking object integrity][1] in the
     #   *Amazon S3 User Guide*.
     #
@@ -329,19 +345,30 @@ module Aws::S3
     # @option options [String] :checksum_crc32c
     #   This header can be used as a data integrity check to verify that the
     #   data received is the same data that was originally sent. This header
-    #   specifies the base64-encoded, 32-bit CRC-32C checksum of the object.
+    #   specifies the Base64 encoded, 32-bit `CRC-32C` checksum of the object.
     #   For more information, see [Checking object integrity][1] in the
     #   *Amazon S3 User Guide*.
     #
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+    # @option options [String] :checksum_crc64nvme
+    #   This header can be used as a data integrity check to verify that the
+    #   data received is the same data that was originally sent. This header
+    #   specifies the Base64 encoded, 64-bit `CRC-64NVME` checksum of the
+    #   object. The `CRC-64NVME` checksum is always a full object checksum.
+    #   For more information, see [Checking object integrity in the Amazon S3
+    #   User Guide][1].
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
     # @option options [String] :checksum_sha1
     #   This header can be used as a data integrity check to verify that the
     #   data received is the same data that was originally sent. This header
-    #   specifies the base64-encoded, 160-bit SHA-1 digest of the object. For
-    #   more information, see [Checking object integrity][1] in the *Amazon S3
-    #   User Guide*.
+    #   specifies the Base64 encoded, 160-bit `SHA-1` digest of the object.
+    #   For more information, see [Checking object integrity][1] in the
+    #   *Amazon S3 User Guide*.
     #
     #
     #
@@ -349,13 +376,28 @@ module Aws::S3
     # @option options [String] :checksum_sha256
     #   This header can be used as a data integrity check to verify that the
     #   data received is the same data that was originally sent. This header
-    #   specifies the base64-encoded, 256-bit SHA-256 digest of the object.
+    #   specifies the Base64 encoded, 256-bit `SHA-256` digest of the object.
     #   For more information, see [Checking object integrity][1] in the
     #   *Amazon S3 User Guide*.
     #
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+    # @option options [String] :checksum_type
+    #   This header specifies the checksum type of the object, which
+    #   determines how part-level checksums are combined to create an
+    #   object-level checksum for multipart objects. You can use this header
+    #   as a data integrity check to verify that the checksum type that is
+    #   received is the same checksum that was specified. If the checksum type
+    #   doesn’t match the checksum type that was specified for the object
+    #   during the `CreateMultipartUpload` request, it’ll result in a
+    #   `BadDigest` error. For more information, see Checking object integrity
+    #   in the Amazon S3 User Guide.
+    # @option options [Integer] :mpu_object_size
+    #   The expected total object size of the multipart upload request. If
+    #   there’s a mismatch between the specified object size value and the
+    #   actual object size value, it results in an `HTTP 400 InvalidRequest`
+    #   error.
     # @option options [String] :request_payer
     #   Confirms that the requester knows that they will be charged for the
     #   request. Bucket owners need not specify this parameter in their

data/lib/aws-sdk-s3/multipart_upload_part.rb

@@ -76,11 +76,10 @@ module Aws::S3
       data[:size]
     end
 
-    # This header can be used as a data integrity check to verify that the
-    # data received is the same data that was originally sent. This header
-    # specifies the base64-encoded, 32-bit CRC-32 checksum of the object.
-    # For more information, see [Checking object integrity][1] in the
-    # *Amazon S3 User Guide*.
+    # The Base64 encoded, 32-bit `CRC-32` checksum of the part. This
+    # checksum is present if the object was uploaded with the `CRC-32`
+    # checksum algorithm. For more information, see [Checking object
+    # integrity][1] in the *Amazon S3 User Guide*.
     #
     #
     #
@@ -90,45 +89,51 @@ module Aws::S3
       data[:checksum_crc32]
     end
 
-    # The base64-encoded, 32-bit CRC-32C checksum of the object. This will
-    # only be present if it was uploaded with the object. When you use an
-    # API operation on an object that was uploaded using multipart uploads,
-    # this value may not be a direct checksum value of the full object.
-    # Instead, it's a calculation based on the checksum values of each
-    # individual part. For more information about how checksums are
-    # calculated with multipart uploads, see [ Checking object integrity][1]
-    # in the *Amazon S3 User Guide*.
+    # The Base64 encoded, 32-bit `CRC-32C` checksum of the part. This
+    # checksum is present if the object was uploaded with the `CRC-32C`
+    # checksum algorithm. For more information, see [Checking object
+    # integrity][1] in the *Amazon S3 User Guide*.
     #
     #
     #
-    # [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html#large-object-checksums
+    # [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
     # @return [String]
     def checksum_crc32c
       data[:checksum_crc32c]
     end
 
-    # The base64-encoded, 160-bit SHA-1 digest of the object. This will only
-    # be present if it was uploaded with the object. When you use the API
-    # operation on an object that was uploaded using multipart uploads, this
-    # value may not be a direct checksum value of the full object. Instead,
-    # it's a calculation based on the checksum values of each individual
-    # part. For more information about how checksums are calculated with
-    # multipart uploads, see [ Checking object integrity][1] in the *Amazon
-    # S3 User Guide*.
+    # The Base64 encoded, 64-bit `CRC-64NVME` checksum of the part. This
+    # checksum is present if the multipart upload request was created with
+    # the `CRC-64NVME` checksum algorithm, or if the object was uploaded
+    # without a checksum (and Amazon S3 added the default checksum,
+    # `CRC-64NVME`, to the uploaded object). For more information, see
+    # [Checking object integrity][1] in the *Amazon S3 User Guide*.
     #
     #
     #
-    # [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html#large-object-checksums
+    # [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+    # @return [String]
+    def checksum_crc64nvme
+      data[:checksum_crc64nvme]
+    end
+
+    # The Base64 encoded, 160-bit `SHA-1` checksum of the part. This
+    # checksum is present if the object was uploaded with the `SHA-1`
+    # checksum algorithm. For more information, see [Checking object
+    # integrity][1] in the *Amazon S3 User Guide*.
+    #
+    #
+    #
+    # [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
     # @return [String]
     def checksum_sha1
       data[:checksum_sha1]
     end
 
-    # This header can be used as a data integrity check to verify that the
-    # data received is the same data that was originally sent. This header
-    # specifies the base64-encoded, 256-bit SHA-256 digest of the object.
-    # For more information, see [Checking object integrity][1] in the
-    # *Amazon S3 User Guide*.
+    # The Base64 encoded, 256-bit `SHA-256` checksum of the part. This
+    # checksum is present if the object was uploaded with the `SHA-256`
+    # checksum algorithm. For more information, see [Checking object
+    # integrity][1] in the *Amazon S3 User Guide*.
     #
     #
     #
@@ -512,9 +517,10 @@ module Aws::S3
     #     body: source_file,
     #     content_length: 1,
     #     content_md5: "ContentMD5",
-    #     checksum_algorithm: "CRC32", # accepts CRC32, CRC32C, SHA1, SHA256
+    #     checksum_algorithm: "CRC32", # accepts CRC32, CRC32C, SHA1, SHA256, CRC64NVME
     #     checksum_crc32: "ChecksumCRC32",
     #     checksum_crc32c: "ChecksumCRC32C",
+    #     checksum_crc64nvme: "ChecksumCRC64NVME",
     #     checksum_sha1: "ChecksumSHA1",
     #     checksum_sha256: "ChecksumSHA256",
     #     sse_customer_algorithm: "SSECustomerAlgorithm",
@@ -530,7 +536,7 @@ module Aws::S3
     #   Size of the body in bytes. This parameter is useful when the size of
     #   the body cannot be determined automatically.
     # @option options [String] :content_md5
-    #   The base64-encoded 128-bit MD5 digest of the part data. This parameter
+    #   The Base64 encoded 128-bit MD5 digest of the part data. This parameter
     #   is auto-populated when using the command from the CLI. This parameter
     #   is required if object lock parameters are specified.
     #
@@ -558,7 +564,7 @@ module Aws::S3
     # @option options [String] :checksum_crc32
     #   This header can be used as a data integrity check to verify that the
     #   data received is the same data that was originally sent. This header
-    #   specifies the base64-encoded, 32-bit CRC-32 checksum of the object.
+    #   specifies the Base64 encoded, 32-bit `CRC-32` checksum of the object.
     #   For more information, see [Checking object integrity][1] in the
     #   *Amazon S3 User Guide*.
     #
@@ -568,19 +574,29 @@ module Aws::S3
     # @option options [String] :checksum_crc32c
     #   This header can be used as a data integrity check to verify that the
     #   data received is the same data that was originally sent. This header
-    #   specifies the base64-encoded, 32-bit CRC-32C checksum of the object.
+    #   specifies the Base64 encoded, 32-bit `CRC-32C` checksum of the object.
     #   For more information, see [Checking object integrity][1] in the
     #   *Amazon S3 User Guide*.
     #
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+    # @option options [String] :checksum_crc64nvme
+    #   This header can be used as a data integrity check to verify that the
+    #   data received is the same data that was originally sent. This header
+    #   specifies the Base64 encoded, 64-bit `CRC-64NVME` checksum of the
+    #   part. For more information, see [Checking object integrity][1] in the
+    #   *Amazon S3 User Guide*.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
     # @option options [String] :checksum_sha1
     #   This header can be used as a data integrity check to verify that the
     #   data received is the same data that was originally sent. This header
-    #   specifies the base64-encoded, 160-bit SHA-1 digest of the object. For
-    #   more information, see [Checking object integrity][1] in the *Amazon S3
-    #   User Guide*.
+    #   specifies the Base64 encoded, 160-bit `SHA-1` digest of the object.
+    #   For more information, see [Checking object integrity][1] in the
+    #   *Amazon S3 User Guide*.
     #
     #
     #
@@ -588,7 +604,7 @@ module Aws::S3
     # @option options [String] :checksum_sha256
     #   This header can be used as a data integrity check to verify that the
     #   data received is the same data that was originally sent. This header
-    #   specifies the base64-encoded, 256-bit SHA-256 digest of the object.
+    #   specifies the Base64 encoded, 256-bit `SHA-256` digest of the object.
     #   For more information, see [Checking object integrity][1] in the
     #   *Amazon S3 User Guide*.
     #