aws-sdk-s3 1.73.0 → 1.78.0

data/lib/aws-sdk-s3/file_uploader.rb

@@ -29,6 +29,9 @@ module Aws
       # @param [String, Pathname, File, Tempfile] source The file to upload.
       # @option options [required, String] :bucket The bucket to upload to.
       # @option options [required, String] :key The key for the object.
+      # @option options [Proc] :progress_callback
+      #   A Proc that will be called when each chunk of the upload is sent.
+      #   It will be invoked with [bytes_read], [total_sizes]
       # @return [void]
       def upload(source, options = {})
         if File.size(source) >= multipart_threshold
@@ -49,11 +52,19 @@ module Aws
       end
 
       def put_object(source, options)
+        if (callback = options.delete(:progress_callback))
+          options[:on_chunk_sent] = single_part_progress(callback)
+        end
         open_file(source) do |file|
           @client.put_object(options.merge(body: file))
         end
       end
 
+      def single_part_progress(progress_callback)
+        proc do |_chunk, bytes_read, total_size|
+          progress_callback.call([bytes_read], [total_size])
+        end
+      end
     end
   end
 end

data/lib/aws-sdk-s3/multipart_file_uploader.rb

@@ -39,6 +39,9 @@ module Aws
       # @param [String, Pathname, File, Tempfile] source The file to upload.
       # @option options [required, String] :bucket The bucket to upload to.
       # @option options [required, String] :key The key for the object.
+      # @option options [Proc] :progress_callback
+      #   A Proc that will be called when each chunk of the upload is sent.
+      #   It will be invoked with [bytes_read], [total_sizes]
       # @return [void]
       def upload(source, options = {})
         if File.size(source) < MIN_PART_SIZE
@@ -68,7 +71,7 @@ module Aws
       def upload_parts(upload_id, source, options)
         pending = PartList.new(compute_parts(upload_id, source, options))
         completed = PartList.new
-        errors = upload_in_threads(pending, completed)
+        errors = upload_in_threads(pending, completed, options)
         if errors.empty?
           completed.to_a.sort_by { |part| part[:part_number] }
         else
@@ -127,12 +130,21 @@ module Aws
         end
       end
 
-      def upload_in_threads(pending, completed)
+      def upload_in_threads(pending, completed, options)
         threads = []
+        if (callback = options[:progress_callback])
+          progress = MultipartProgress.new(pending, callback)
+        end
         @thread_count.times do
           thread = Thread.new do
             begin
               while part = pending.shift
+                if progress
+                  part[:on_chunk_sent] =
+                    proc do |_chunk, bytes, _total|
+                      progress.call(part[:part_number], bytes)
+                    end
+                end
                 resp = @client.upload_part(part)
                 part[:body].close
                 completed.push(etag: resp.etag, part_number: part[:part_number])
@@ -182,11 +194,34 @@ module Aws
           @mutex.synchronize { @parts.clear }
         end
 
+        def size
+          @mutex.synchronize { @parts.size }
+        end
+
+        def part_sizes
+          @mutex.synchronize { @parts.map { |p| p[:body].size } }
+        end
+
         def to_a
           @mutex.synchronize { @parts.dup }
         end
 
       end
+
+      # @api private
+      class MultipartProgress
+        def initialize(parts, progress_callback)
+          @bytes_sent = Array.new(parts.size, 0)
+          @total_sizes = parts.part_sizes
+          @progress_callback = progress_callback
+        end
+
+        def call(part_number, bytes_read)
+          # part numbers start at 1
+          @bytes_sent[part_number - 1] = bytes_read
+          @progress_callback.call(@bytes_sent, @total_sizes)
+        end
+      end
     end
   end
 end

data/lib/aws-sdk-s3/multipart_upload_part.rb

@@ -302,7 +302,7 @@ module Aws::S3
     #     request_payer: "requester", # accepts requester
     #   })
     # @param [Hash] options ({})
-    # @option options [String, IO] :body
+    # @option options [String, StringIO, File] :body
     #   Object data.
     # @option options [Integer] :content_length
     #   Size of the body in bytes. This parameter is useful when the size of

data/lib/aws-sdk-s3/object.rb

@@ -983,7 +983,7 @@ module Aws::S3
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL
-    # @option options [String, IO] :body
+    # @option options [String, StringIO, File] :body
     #   Object data.
     # @option options [String] :cache_control
     #   Can be used to specify caching behavior along the request/reply chain.

data/lib/aws-sdk-s3/object_summary.rb

@@ -48,8 +48,24 @@ module Aws::S3
       data[:last_modified]
     end
 
-    # The entity tag is an MD5 hash of the object. ETag reflects only
-    # changes to the contents of an object, not its metadata.
+    # The entity tag is a hash of the object. The ETag reflects changes only
+    # to the contents of an object, not its metadata. The ETag may or may
+    # not be an MD5 digest of the object data. Whether or not it is depends
+    # on how the object was created and how it is encrypted as described
+    # below:
+    #
+    # * Objects created by the PUT Object, POST Object, or Copy operation,
+    #   or through the AWS Management Console, and are encrypted by SSE-S3
+    #   or plaintext, have ETags that are an MD5 digest of their object
+    #   data.
+    #
+    # * Objects created by the PUT Object, POST Object, or Copy operation,
+    #   or through the AWS Management Console, and are encrypted by SSE-C or
+    #   SSE-KMS, have ETags that are not an MD5 digest of their object data.
+    #
+    # * If an object is created by either the Multipart Upload or Part Copy
+    #   operation, the ETag is not an MD5 digest, regardless of the method
+    #   of encryption.
     # @return [String]
     def etag
       data[:etag]
@@ -733,7 +749,7 @@ module Aws::S3
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL
-    # @option options [String, IO] :body
+    # @option options [String, StringIO, File] :body
     #   Object data.
     # @option options [String] :cache_control
     #   Can be used to specify caching behavior along the request/reply chain.

data/lib/aws-sdk-s3/plugins/accelerate.rb

@@ -3,36 +3,46 @@
 module Aws
   module S3
     module Plugins
-
       # Provides support for using `Aws::S3::Client` with Amazon S3 Transfer
       # Acceleration.
       #
       # Go here for more information about transfer acceleration:
       # [http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html](http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html)
       class Accelerate < Seahorse::Client::Plugin
-
-        option(:use_accelerate_endpoint,
+        option(
+          :use_accelerate_endpoint,
           default: false,
           doc_type: 'Boolean',
           docstring: <<-DOCS)
 When set to `true`, accelerated bucket endpoints will be used
 for all object operations. You must first enable accelerate for
-each bucket.  [Go here for more information](http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html).
+each bucket. [Go here for more information](http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html).
           DOCS
 
         def add_handlers(handlers, config)
           operations = config.api.operation_names - [
-            :create_bucket, :list_buckets, :delete_bucket,
+            :create_bucket, :list_buckets, :delete_bucket
           ]
-          handlers.add(OptionHandler, step: :initialize, operations: operations)
-          handlers.add(AccelerateHandler, step: :build, priority: 0, operations: operations)
+          # Need 2 handlers so that the context can be set for other plugins
+          # and to remove :use_accelerate_endpoint from the params.
+          handlers.add(
+            OptionHandler, step: :initialize, operations: operations
+          )
+          handlers.add(
+            AccelerateHandler, step: :build, priority: 0, operations: operations
+          )
         end
 
         # @api private
         class OptionHandler < Seahorse::Client::Handler
           def call(context)
-            accelerate = context.params.delete(:use_accelerate_endpoint)
-            accelerate = context.config.use_accelerate_endpoint if accelerate.nil?
+            # Support client configuration and per-operation configuration
+            if context.params.is_a?(Hash)
+              accelerate = context.params.delete(:use_accelerate_endpoint)
+            end
+            if accelerate.nil?
+              accelerate = context.config.use_accelerate_endpoint
+            end
             context[:use_accelerate_endpoint] = accelerate
             @handler.call(context)
           end
@@ -40,39 +50,24 @@ each bucket.  [Go here for more information](http://docs.aws.amazon.com/AmazonS3
 
         # @api private
         class AccelerateHandler < Seahorse::Client::Handler
-
           def call(context)
             if context[:use_accelerate_endpoint]
-              if context[:use_dualstack_endpoint]
-                use_combined_accelerate_dualstack_endpoint(context)
-              else
-                use_accelerate_endpoint(context)
-              end
+              dualstack = !!context[:use_dualstack_endpoint]
+              use_accelerate_endpoint(context, dualstack)
             end
             @handler.call(context)
           end
 
           private
 
-          def use_accelerate_endpoint(context)
+          def use_accelerate_endpoint(context, dualstack)
             bucket_name = context.params[:bucket]
             validate_bucket_name!(bucket_name)
             endpoint = URI.parse(context.http_request.endpoint.to_s)
             endpoint.scheme = 'https'
             endpoint.port = 443
-            endpoint.host = "#{bucket_name}.s3-accelerate.amazonaws.com"
-            context.http_request.endpoint = endpoint.to_s
-            # s3 accelerate endpoint doesn't work with 'expect' header
-            context.http_request.headers.delete('expect')
-          end
-
-          def use_combined_accelerate_dualstack_endpoint(context)
-            bucket_name = context.params[:bucket]
-            validate_bucket_name!(bucket_name)
-            endpoint = URI.parse(context.http_request.endpoint.to_s)
-            endpoint.scheme = 'https'
-            endpoint.port = 443
-            endpoint.host = "#{bucket_name}.s3-accelerate.dualstack.amazonaws.com"
+            endpoint.host = "#{bucket_name}.s3-accelerate"\
+                            "#{'.dualstack' if dualstack}.amazonaws.com"
             context.http_request.endpoint = endpoint.to_s
             # s3 accelerate endpoint doesn't work with 'expect' header
             context.http_request.headers.delete('expect')
@@ -80,17 +75,11 @@ each bucket.  [Go here for more information](http://docs.aws.amazon.com/AmazonS3
 
           def validate_bucket_name!(bucket_name)
             unless BucketDns.dns_compatible?(bucket_name, _ssl = true)
-              msg = 'unable to use `accelerate: true` on buckets with '\
-                    'non-DNS compatible names'
-              raise ArgumentError, msg
-            end
-            if bucket_name.include?('.')
-              msg = 'unable to use `accelerate: true` on buckets with dots'\
-                    "in their name: #{bucket_name.inspect}"
-              raise ArgumentError, msg
+              raise ArgumentError,
+                    'Unable to use `use_accelerate_endpoint: true` on buckets '\
+                    'with non-DNS compatible names.'
             end
           end
-
         end
       end
     end

data/lib/aws-sdk-s3/plugins/dualstack.rb

@@ -22,7 +22,9 @@ for all operations.
         # @api private
         class OptionHandler < Seahorse::Client::Handler
           def call(context)
-            dualstack = context.params.delete(:use_dualstack_endpoint)
+            if context.params.is_a?(Hash)
+              dualstack = context.params.delete(:use_dualstack_endpoint)
+            end
             dualstack = context.config.use_dualstack_endpoint if dualstack.nil?
             context[:use_dualstack_endpoint] = dualstack
             @handler.call(context)

data/lib/aws-sdk-s3/plugins/sse_cpk.rb

@@ -20,7 +20,7 @@ This should only be disabled for local testing.
         class Handler < Seahorse::Client::Handler
 
           def call(context)
-            compute_key_md5(context)
+            compute_key_md5(context) if context.params.is_a?(Hash)
             @handler.call(context)
           end
 

data/lib/aws-sdk-s3/presigned_post.rb

@@ -185,35 +185,58 @@ module Aws
       #   the post will expire. Defaults to one hour from creation of the
       #   presigned post. May not exceed one week from creation time.
       # @option options [String] :key See {PresignedPost#key}.
-      # @option options [String] :key_starts_with See {PresignedPost#key_starts_with}.
+      # @option options [String] :key_starts_with
+      #   See {PresignedPost#key_starts_with}.
       # @option options [String] :acl See {PresignedPost#acl}.
-      # @option options [String] :acl_starts_with See {PresignedPost#acl_starts_with}.
-      # @option options [String] :cache_control See {PresignedPost#cache_control}.
-      # @option options [String] :cache_control_starts_with See {PresignedPost#cache_control_starts_with}.
+      # @option options [String] :acl_starts_with
+      #   See {PresignedPost#acl_starts_with}.
+      # @option options [String] :cache_control
+      #   See {PresignedPost#cache_control}.
+      # @option options [String] :cache_control_starts_with
+      #   See {PresignedPost#cache_control_starts_with}.
       # @option options [String] :content_type See {PresignedPost#content_type}.
-      # @option options [String] :content_type_starts_with See {PresignedPost#content_type_starts_with}.
-      # @option options [String] :content_disposition See {PresignedPost#content_disposition}.
-      # @option options [String] :content_disposition_starts_with See {PresignedPost#content_disposition_starts_with}.
-      # @option options [String] :content_encoding See {PresignedPost#content_encoding}.
-      # @option options [String] :content_encoding_starts_with See {PresignedPost#content_encoding_starts_with}.
+      # @option options [String] :content_type_starts_with
+      #   See {PresignedPost#content_type_starts_with}.
+      # @option options [String] :content_disposition
+      #   See {PresignedPost#content_disposition}.
+      # @option options [String] :content_disposition_starts_with
+      #   See {PresignedPost#content_disposition_starts_with}.
+      # @option options [String] :content_encoding
+      #   See {PresignedPost#content_encoding}.
+      # @option options [String] :content_encoding_starts_with
+      #   See {PresignedPost#content_encoding_starts_with}.
       # @option options [String] :expires See {PresignedPost#expires}.
-      # @option options [String] :expires_starts_with See {PresignedPost#expires_starts_with}.
-      # @option options [Range<Integer>] :content_length_range See {PresignedPost#content_length_range}.
-      # @option options [String] :success_action_redirect See {PresignedPost#success_action_redirect}.
-      # @option options [String] :success_action_redirect_starts_with See {PresignedPost#success_action_redirect_starts_with}.
-      # @option options [String] :success_action_status See {PresignedPost#success_action_status}.
-      # @option options [String] :storage_class See {PresignedPost#storage_class}.
-      # @option options [String] :website_redirect_location See {PresignedPost#website_redirect_location}.
-      # @option options [Hash<String,String>] :metadata See {PresignedPost#metadata}.
-      # @option options [Hash<String,String>] :metadata_starts_with See {PresignedPost#metadata_starts_with}.
-      # @option options [String] :server_side_encryption See {PresignedPost#server_side_encryption}.
-      # @option options [String] :server_side_encryption_aws_kms_key_id See {PresignedPost#server_side_encryption_aws_kms_key_id}.
-      # @option options [String] :server_side_encryption_customer_algorithm See {PresignedPost#server_side_encryption_customer_algorithm}.
-      # @option options [String] :server_side_encryption_customer_key See {PresignedPost#server_side_encryption_customer_key}.
+      # @option options [String] :expires_starts_with
+      #   See {PresignedPost#expires_starts_with}.
+      # @option options [Range<Integer>] :content_length_range
+      #   See {PresignedPost#content_length_range}.
+      # @option options [String] :success_action_redirect
+      #   See {PresignedPost#success_action_redirect}.
+      # @option options [String] :success_action_redirect_starts_with
+      #   See {PresignedPost#success_action_redirect_starts_with}.
+      # @option options [String] :success_action_status
+      #   See {PresignedPost#success_action_status}.
+      # @option options [String] :storage_class
+      #   See {PresignedPost#storage_class}.
+      # @option options [String] :website_redirect_location
+      #   See {PresignedPost#website_redirect_location}.
+      # @option options [Hash<String,String>] :metadata
+      #   See {PresignedPost#metadata}.
+      # @option options [Hash<String,String>] :metadata_starts_with
+      #   See {PresignedPost#metadata_starts_with}.
+      # @option options [String] :server_side_encryption
+      #   See {PresignedPost#server_side_encryption}.
+      # @option options [String] :server_side_encryption_aws_kms_key_id
+      #   See {PresignedPost#server_side_encryption_aws_kms_key_id}.
+      # @option options [String] :server_side_encryption_customer_algorithm
+      #   See {PresignedPost#server_side_encryption_customer_algorithm}.
+      # @option options [String] :server_side_encryption_customer_key
+      #   See {PresignedPost#server_side_encryption_customer_key}.
       def initialize(credentials, bucket_region, bucket_name, options = {})
         @credentials = credentials.credentials
         @bucket_region = bucket_region
         @bucket_name = bucket_name
+        @accelerate = !!options.delete(:use_accelerate_endpoint)
         @url = options.delete(:url) || bucket_url
         @fields = {}
         @key_set = false
@@ -272,7 +295,7 @@ module Aws
 
       # @!group Fields
 
-      # The key to use for the uploaded object. Use can use `${filename}`
+      # The key to use for the uploaded object. You can use `${filename}`
       # as a variable in the key. This will be replaced with the name
       # of the file as provided by the user.
       #
@@ -507,7 +530,10 @@ module Aws
       #   (KMS) master encryption key to use for the object.
       #   @param [String] value
       #   @return [self]
-      define_field(:server_side_encryption_aws_kms_key_id, 'x-amz-server-side-encryption-aws-kms-key-id')
+      define_field(
+        :server_side_encryption_aws_kms_key_id,
+        'x-amz-server-side-encryption-aws-kms-key-id'
+      )
 
       # @!endgroup
 
@@ -520,7 +546,10 @@ module Aws
       #   @param [String] value
       #   @see #server_side_encryption_customer_key
       #   @return [self]
-      define_field(:server_side_encryption_customer_algorithm, 'x-amz-server-side-encryption-customer-algorithm')
+      define_field(
+        :server_side_encryption_customer_algorithm,
+        'x-amz-server-side-encryption-customer-algorithm'
+      )
 
       # Specifies the customer-provided encryption key for Amazon S3 to use
       # in encrypting data. This value is used to store the object and then
@@ -582,10 +611,14 @@ module Aws
       def bucket_url
         url = Aws::Partitions::EndpointProvider.resolve(@bucket_region, 's3')
         url = URI.parse(url)
-        if Plugins::BucketDns.dns_compatible?(@bucket_name, true)
-          url.host = @bucket_name + '.' + url.host
+        if Plugins::BucketDns.dns_compatible?(@bucket_name, _ssl = true)
+          if @accelerate
+            url.host = "#{@bucket_name}.s3-accelerate.amazonaws.com"
+          else
+            url.host = "#{@bucket_name}.#{url.host}"
+          end
         else
-          url.path = '/' + @bucket_name
+          url.path = "/#{@bucket_name}"
         end
         if @bucket_region == 'us-east-1'
           # keep legacy behavior by default

data/lib/aws-sdk-s3/presigner.rb

@@ -41,7 +41,7 @@ module Aws
       #  signer = Aws::S3::Presigner.new
       #  url = signer.presigned_url(:get_object, bucket: "bucket", key: "key")
       #
-      # @param [Symbol] :method Symbolized method name of the operation you want
+      # @param [Symbol] method Symbolized method name of the operation you want
       #   to presign.
       #
       # @option params [Integer] :expires_in (900) The number of seconds
@@ -89,7 +89,7 @@ module Aws
       #    :get_object, bucket: "bucket", key: "key"
       #  )
       #
-      # @param [Symbol] :method Symbolized method name of the operation you want
+      # @param [Symbol] method Symbolized method name of the operation you want
       #   to presign.
       #
       # @option params [Integer] :expires_in (900) The number of seconds