aws-sdk-s3 1.71.1 → 1.76.0

data/lib/aws-sdk-s3/encryptionV2/materials.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 
 module Aws

data/lib/aws-sdk-s3/encryptionV2/utils.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'openssl'
 
 module Aws
@@ -6,24 +8,9 @@ module Aws
       # @api private
       module Utils
 
-        UNSAFE_MSG = "unsafe encryption, data is longer than key length"
-
         class << self
 
-          def encrypt(key, data)
-            case key
-            when OpenSSL::PKey::RSA # asymmetric encryption
-              warn(UNSAFE_MSG) if key.public_key.n.num_bits < cipher_size(data)
-              key.public_encrypt(data)
-            when String # symmetric encryption
-              warn(UNSAFE_MSG) if cipher_size(key) < cipher_size(data)
-              cipher = aes_encryption_cipher(:ECB, key)
-              cipher.update(data) + cipher.final
-            end
-          end
-
           def encrypt_aes_gcm(key, data, auth_data)
-            warn(UNSAFE_MSG) if cipher_size(key) < cipher_size(data)
             cipher = aes_encryption_cipher(:GCM, key)
             cipher.iv = (iv = cipher.random_iv)
             cipher.auth_data = auth_data

data/lib/aws-sdk-s3/encryption_v2.rb

@@ -14,7 +14,10 @@ require 'aws-sdk-s3/encryptionV2/default_key_provider'
 
 module Aws
   module S3
-    module EncryptionV2; end
+    module EncryptionV2
+      AES_GCM_TAG_LEN_BYTES = 16
+      EC_USER_AGENT = 'S3CryptoV2'
+    end
   end
 end
 

data/lib/aws-sdk-s3/file_uploader.rb

@@ -29,6 +29,9 @@ module Aws
       # @param [String, Pathname, File, Tempfile] source The file to upload.
       # @option options [required, String] :bucket The bucket to upload to.
       # @option options [required, String] :key The key for the object.
+      # @option options [Proc] :progress_callback
+      #   A Proc that will be called when each chunk of the upload is sent.
+      #   It will be invoked with [bytes_read], [total_sizes]
       # @return [void]
       def upload(source, options = {})
         if File.size(source) >= multipart_threshold
@@ -49,11 +52,19 @@ module Aws
       end
 
       def put_object(source, options)
+        if (callback = options.delete(:progress_callback))
+          options[:on_chunk_sent] = single_part_progress(callback)
+        end
         open_file(source) do |file|
           @client.put_object(options.merge(body: file))
         end
       end
 
+      def single_part_progress(progress_callback)
+        proc do |_chunk, bytes_read, total_size|
+          progress_callback.call([bytes_read], [total_size])
+        end
+      end
     end
   end
 end

data/lib/aws-sdk-s3/multipart_file_uploader.rb

@@ -39,6 +39,9 @@ module Aws
       # @param [String, Pathname, File, Tempfile] source The file to upload.
       # @option options [required, String] :bucket The bucket to upload to.
       # @option options [required, String] :key The key for the object.
+      # @option options [Proc] :progress_callback
+      #   A Proc that will be called when each chunk of the upload is sent.
+      #   It will be invoked with [bytes_read], [total_sizes]
       # @return [void]
       def upload(source, options = {})
         if File.size(source) < MIN_PART_SIZE
@@ -68,7 +71,7 @@ module Aws
       def upload_parts(upload_id, source, options)
         pending = PartList.new(compute_parts(upload_id, source, options))
         completed = PartList.new
-        errors = upload_in_threads(pending, completed)
+        errors = upload_in_threads(pending, completed, options)
         if errors.empty?
           completed.to_a.sort_by { |part| part[:part_number] }
         else
@@ -127,12 +130,21 @@ module Aws
         end
       end
 
-      def upload_in_threads(pending, completed)
+      def upload_in_threads(pending, completed, options)
         threads = []
+        if (callback = options[:progress_callback])
+          progress = MultipartProgress.new(pending, callback)
+        end
         @thread_count.times do
           thread = Thread.new do
             begin
               while part = pending.shift
+                if progress
+                  part[:on_chunk_sent] =
+                    proc do |_chunk, bytes, _total|
+                      progress.call(part[:part_number], bytes)
+                    end
+                end
                 resp = @client.upload_part(part)
                 part[:body].close
                 completed.push(etag: resp.etag, part_number: part[:part_number])
@@ -182,11 +194,34 @@ module Aws
           @mutex.synchronize { @parts.clear }
         end
 
+        def size
+          @mutex.synchronize { @parts.size }
+        end
+
+        def part_sizes
+          @mutex.synchronize { @parts.map { |p| p[:body].size } }
+        end
+
         def to_a
           @mutex.synchronize { @parts.dup }
         end
 
       end
+
+      # @api private
+      class MultipartProgress
+        def initialize(parts, progress_callback)
+          @bytes_sent = Array.new(parts.size, 0)
+          @total_sizes = parts.part_sizes
+          @progress_callback = progress_callback
+        end
+
+        def call(part_number, bytes_read)
+          # part numbers start at 1
+          @bytes_sent[part_number - 1] = bytes_read
+          @progress_callback.call(@bytes_sent, @total_sizes)
+        end
+      end
     end
   end
 end

data/lib/aws-sdk-s3/multipart_upload_part.rb

@@ -250,8 +250,8 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm` header. This must
-    #   be the same encryption key specified in the initiate multipart upload
+    #   `x-amz-server-side-encryption-customer-algorithm` header. This must be
+    #   the same encryption key specified in the initiate multipart upload
     #   request.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
@@ -302,7 +302,7 @@ module Aws::S3
     #     request_payer: "requester", # accepts requester
     #   })
     # @param [Hash] options ({})
-    # @option options [String, IO] :body
+    # @option options [String, StringIO, File] :body
     #   Object data.
     # @option options [Integer] :content_length
     #   Size of the body in bytes. This parameter is useful when the size of
@@ -319,8 +319,8 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm header`. This must
-    #   be the same encryption key specified in the initiate multipart upload
+    #   `x-amz-server-side-encryption-customer-algorithm header`. This must be
+    #   the same encryption key specified in the initiate multipart upload
     #   request.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to

data/lib/aws-sdk-s3/object.rb

@@ -67,8 +67,8 @@ module Aws::S3
 
     # If the object is an archived object (an object whose storage class is
     # GLACIER), the response includes this header if either the archive
-    # restoration is in progress (see RestoreObject or an archive copy is
-    # already restored.
+    # restoration is in progress (see [RestoreObject][1] or an archive copy
+    # is already restored.
     #
     # If an archive copy is already restored, the header value indicates
     # when Amazon S3 is scheduled to delete the object copy. For example:
@@ -80,11 +80,12 @@ module Aws::S3
     # `ongoing-request="true"`.
     #
     # For more information about archiving objects, see [Transitioning
-    # Objects: General Considerations][1].
+    # Objects: General Considerations][2].
     #
     #
     #
-    # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html#lifecycle-transition-general-considerations
+    # [1]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html
+    # [2]: https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html#lifecycle-transition-general-considerations
     # @return [String]
     def restore
       data[:restore]
@@ -609,7 +610,7 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm` header.
+    #   `x-amz-server-side-encryption-customer-algorithm` header.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
     #   RFC 1321. Amazon S3 uses this header for a message integrity check to
@@ -779,7 +780,7 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm` header.
+    #   `x-amz-server-side-encryption-customer-algorithm` header.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
     #   RFC 1321. Amazon S3 uses this header for a message integrity check to
@@ -885,7 +886,7 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm` header.
+    #   `x-amz-server-side-encryption-customer-algorithm` header.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
     #   RFC 1321. Amazon S3 uses this header for a message integrity check to
@@ -983,7 +984,7 @@ module Aws::S3
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL
-    # @option options [String, IO] :body
+    # @option options [String, StringIO, File] :body
     #   Object data.
     # @option options [String] :cache_control
     #   Can be used to specify caching behavior along the request/reply chain.
@@ -1098,7 +1099,7 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm` header.
+    #   `x-amz-server-side-encryption-customer-algorithm` header.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
     #   RFC 1321. Amazon S3 uses this header for a message integrity check to

data/lib/aws-sdk-s3/object_summary.rb

@@ -48,8 +48,24 @@ module Aws::S3
       data[:last_modified]
     end
 
-    # The entity tag is an MD5 hash of the object. ETag reflects only
-    # changes to the contents of an object, not its metadata.
+    # The entity tag is a hash of the object. The ETag reflects changes only
+    # to the contents of an object, not its metadata. The ETag may or may
+    # not be an MD5 digest of the object data. Whether or not it is depends
+    # on how the object was created and how it is encrypted as described
+    # below:
+    #
+    # * Objects created by the PUT Object, POST Object, or Copy operation,
+    #   or through the AWS Management Console, and are encrypted by SSE-S3
+    #   or plaintext, have ETags that are an MD5 digest of their object
+    #   data.
+    #
+    # * Objects created by the PUT Object, POST Object, or Copy operation,
+    #   or through the AWS Management Console, and are encrypted by SSE-C or
+    #   SSE-KMS, have ETags that are not an MD5 digest of their object data.
+    #
+    # * If an object is created by either the Multipart Upload or Part Copy
+    #   operation, the ETag is not an MD5 digest, regardless of the method
+    #   of encryption.
     # @return [String]
     def etag
       data[:etag]
@@ -359,7 +375,7 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm` header.
+    #   `x-amz-server-side-encryption-customer-algorithm` header.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
     #   RFC 1321. Amazon S3 uses this header for a message integrity check to
@@ -529,7 +545,7 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm` header.
+    #   `x-amz-server-side-encryption-customer-algorithm` header.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
     #   RFC 1321. Amazon S3 uses this header for a message integrity check to
@@ -635,7 +651,7 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm` header.
+    #   `x-amz-server-side-encryption-customer-algorithm` header.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
     #   RFC 1321. Amazon S3 uses this header for a message integrity check to
@@ -733,7 +749,7 @@ module Aws::S3
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL
-    # @option options [String, IO] :body
+    # @option options [String, StringIO, File] :body
     #   Object data.
     # @option options [String] :cache_control
     #   Can be used to specify caching behavior along the request/reply chain.
@@ -848,7 +864,7 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm` header.
+    #   `x-amz-server-side-encryption-customer-algorithm` header.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
     #   RFC 1321. Amazon S3 uses this header for a message integrity check to

data/lib/aws-sdk-s3/object_version.rb

@@ -331,7 +331,7 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm` header.
+    #   `x-amz-server-side-encryption-customer-algorithm` header.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
     #   RFC 1321. Amazon S3 uses this header for a message integrity check to
@@ -406,7 +406,7 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm` header.
+    #   `x-amz-server-side-encryption-customer-algorithm` header.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
     #   RFC 1321. Amazon S3 uses this header for a message integrity check to

data/lib/aws-sdk-s3/plugins/accelerate.rb

@@ -3,36 +3,46 @@
 module Aws
   module S3
     module Plugins
-
       # Provides support for using `Aws::S3::Client` with Amazon S3 Transfer
       # Acceleration.
       #
       # Go here for more information about transfer acceleration:
       # [http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html](http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html)
       class Accelerate < Seahorse::Client::Plugin
-
-        option(:use_accelerate_endpoint,
+        option(
+          :use_accelerate_endpoint,
           default: false,
           doc_type: 'Boolean',
           docstring: <<-DOCS)
 When set to `true`, accelerated bucket endpoints will be used
 for all object operations. You must first enable accelerate for
-each bucket.  [Go here for more information](http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html).
+each bucket. [Go here for more information](http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html).
           DOCS
 
         def add_handlers(handlers, config)
           operations = config.api.operation_names - [
-            :create_bucket, :list_buckets, :delete_bucket,
+            :create_bucket, :list_buckets, :delete_bucket
           ]
-          handlers.add(OptionHandler, step: :initialize, operations: operations)
-          handlers.add(AccelerateHandler, step: :build, priority: 0, operations: operations)
+          # Need 2 handlers so that the context can be set for other plugins
+          # and to remove :use_accelerate_endpoint from the params.
+          handlers.add(
+            OptionHandler, step: :initialize, operations: operations
+          )
+          handlers.add(
+            AccelerateHandler, step: :build, priority: 0, operations: operations
+          )
         end
 
         # @api private
         class OptionHandler < Seahorse::Client::Handler
           def call(context)
-            accelerate = context.params.delete(:use_accelerate_endpoint)
-            accelerate = context.config.use_accelerate_endpoint if accelerate.nil?
+            # Support client configuration and per-operation configuration
+            if context.params.is_a?(Hash)
+              accelerate = context.params.delete(:use_accelerate_endpoint)
+            end
+            if accelerate.nil?
+              accelerate = context.config.use_accelerate_endpoint
+            end
             context[:use_accelerate_endpoint] = accelerate
             @handler.call(context)
           end
@@ -40,39 +50,24 @@ each bucket.  [Go here for more information](http://docs.aws.amazon.com/AmazonS3
 
         # @api private
         class AccelerateHandler < Seahorse::Client::Handler
-
           def call(context)
             if context[:use_accelerate_endpoint]
-              if context[:use_dualstack_endpoint]
-                use_combined_accelerate_dualstack_endpoint(context)
-              else
-                use_accelerate_endpoint(context)
-              end
+              dualstack = !!context[:use_dualstack_endpoint]
+              use_accelerate_endpoint(context, dualstack)
             end
             @handler.call(context)
           end
 
           private
 
-          def use_accelerate_endpoint(context)
+          def use_accelerate_endpoint(context, dualstack)
             bucket_name = context.params[:bucket]
             validate_bucket_name!(bucket_name)
             endpoint = URI.parse(context.http_request.endpoint.to_s)
             endpoint.scheme = 'https'
             endpoint.port = 443
-            endpoint.host = "#{bucket_name}.s3-accelerate.amazonaws.com"
-            context.http_request.endpoint = endpoint.to_s
-            # s3 accelerate endpoint doesn't work with 'expect' header
-            context.http_request.headers.delete('expect')
-          end
-
-          def use_combined_accelerate_dualstack_endpoint(context)
-            bucket_name = context.params[:bucket]
-            validate_bucket_name!(bucket_name)
-            endpoint = URI.parse(context.http_request.endpoint.to_s)
-            endpoint.scheme = 'https'
-            endpoint.port = 443
-            endpoint.host = "#{bucket_name}.s3-accelerate.dualstack.amazonaws.com"
+            endpoint.host = "#{bucket_name}.s3-accelerate"\
+                            "#{'.dualstack' if dualstack}.amazonaws.com"
             context.http_request.endpoint = endpoint.to_s
             # s3 accelerate endpoint doesn't work with 'expect' header
             context.http_request.headers.delete('expect')
@@ -80,17 +75,11 @@ each bucket.  [Go here for more information](http://docs.aws.amazon.com/AmazonS3
 
           def validate_bucket_name!(bucket_name)
             unless BucketDns.dns_compatible?(bucket_name, _ssl = true)
-              msg = 'unable to use `accelerate: true` on buckets with '\
-                    'non-DNS compatible names'
-              raise ArgumentError, msg
-            end
-            if bucket_name.include?('.')
-              msg = 'unable to use `accelerate: true` on buckets with dots'\
-                    "in their name: #{bucket_name.inspect}"
-              raise ArgumentError, msg
+              raise ArgumentError,
+                    'Unable to use `use_accelerate_endpoint: true` on buckets '\
+                    'with non-DNS compatible names.'
             end
           end
-
         end
       end
     end