aws-sdk-s3 1.232.1 → 1.233.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b94c399221f0a258a78a6339e0887c7cfed857495e036fc96dc1c71552eeb02a
-  data.tar.gz: b000a464b400582cdc17a7295d7601a718e51b91169c1a1af03e8712584d6cdd
+  metadata.gz: 2a94f968574a4140131a94ed10f6e238db29a2cb3ceac5eaab1bccdda34efa4f
+  data.tar.gz: 63bbf8c2512e03438f6e22c634bb9070cc52e0b06096d7a6945dc55100303631
 SHA512:
-  metadata.gz: ebd8b1083b8e6421dafeeed2ac4ad91d7810fdedf5d88d56c7a3e75d81636cd57ddc7e20cb97824db9acf1858a3e4f8a63155b48610fa39eebafce059d9576ee
-  data.tar.gz: f0f726ac4e9239ea0cae007668b7f8720c6ec2a314c8fe72b5db0220be5cc1508d2269634f759ab8522da78e151814d9b2d0753b1493cc5afd0a7be9b098944c
+  metadata.gz: 46c2a374e3f4bbd446d32b3403f08e2b648f37dc5f1cfb1d1c2224a211b43e6450ebdaa9cc3bbf9c6fa3e2a5b0884671127baf13163391fb11647a339f34ae5f
+  data.tar.gz: 564b1d9d422faa9180efc8e86f31803c517335608ff98eb05fa328094cc5de1a23cd22f13afeb09aa3e84aeb5f0e0f3b8a2e9a2f69437403efcbc3aa7bb2ebb2

data/CHANGELOG.md

@@ -1,6 +1,30 @@
 Unreleased Changes
 ------------------
 
+1.233.1 (2026-10-01)
+------------------
+
+* Issue - Fix unbounded memory growth in `upload_stream` on `TransferManager` and `Aws::S3::Object` when the data source outpaces the upload (#3407).
+
+* Issue - Prevent `upload_file`, `upload_stream` and `download_file` on `TransferManager` and `Aws::S3::Object` from hanging when a worker thread is terminated by a non-`StandardError`.
+
+1.233.0 (2026-09-30)
+------------------
+
+* Feature - Amazon S3 adds a new optional S3 Inventory field, IntelligentTieringReferenceDate, reporting the reference date S3 Intelligent-Tiering uses to evaluate an object's tier-transition eligibility. The value is populated for objects in the Intelligent-Tiering storage class and left blank for others.
+
+1.232.3 (2026-09-28)
+------------------
+
+* Issue - Remove `:thread_count` from transfer options even when a custom `:executor` is configured on `TransferManager`, fixing an `ArgumentError` raised on single-part `upload_file` calls.
+
+* Issue - Fix `:thread_count` being ignored by the deprecated `Object#download_file`, which always used the default thread count.
+
+1.232.2 (2026-09-25)
+------------------
+
+* Issue - Ensure the internally-created executor is shutdown on error in `TransferManager` and `Aws::S3::Object` resource methods, preventing leaked worker threads on multipart transfer failures (#3419).
+
 1.232.1 (2026-09-16)
 ------------------
 

data/VERSION

@@ -1 +1 @@
-1.232.1
+1.233.1

data/lib/aws-sdk-s3/client.rb

@@ -7723,7 +7723,7 @@ module Aws::S3
     #   resp.inventory_configuration.id #=> String
     #   resp.inventory_configuration.included_object_versions #=> String, one of "All", "Current"
     #   resp.inventory_configuration.optional_fields #=> Array
-    #   resp.inventory_configuration.optional_fields[0] #=> String, one of "Size", "LastModifiedDate", "StorageClass", "ETag", "IsMultipartUploaded", "ReplicationStatus", "EncryptionStatus", "ObjectLockRetainUntilDate", "ObjectLockMode", "ObjectLockLegalHoldStatus", "ObjectLockEventHoldStatus", "ObjectLockEventHoldDuration", "IntelligentTieringAccessTier", "BucketKeyStatus", "ChecksumAlgorithm", "ObjectAccessControlList", "ObjectOwner", "LifecycleExpirationDate"
+    #   resp.inventory_configuration.optional_fields[0] #=> String, one of "Size", "LastModifiedDate", "StorageClass", "ETag", "IsMultipartUploaded", "ReplicationStatus", "EncryptionStatus", "ObjectLockRetainUntilDate", "ObjectLockMode", "ObjectLockLegalHoldStatus", "ObjectLockEventHoldStatus", "ObjectLockEventHoldDuration", "IntelligentTieringAccessTier", "BucketKeyStatus", "ChecksumAlgorithm", "ObjectAccessControlList", "ObjectOwner", "LifecycleExpirationDate", "IntelligentTieringReferenceDate"
     #   resp.inventory_configuration.schedule.frequency #=> String, one of "Daily", "Weekly"
     #
     # @see http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/GetBucketInventoryConfiguration AWS API Documentation
@@ -12760,7 +12760,7 @@ module Aws::S3
     #   resp.inventory_configuration_list[0].id #=> String
     #   resp.inventory_configuration_list[0].included_object_versions #=> String, one of "All", "Current"
     #   resp.inventory_configuration_list[0].optional_fields #=> Array
-    #   resp.inventory_configuration_list[0].optional_fields[0] #=> String, one of "Size", "LastModifiedDate", "StorageClass", "ETag", "IsMultipartUploaded", "ReplicationStatus", "EncryptionStatus", "ObjectLockRetainUntilDate", "ObjectLockMode", "ObjectLockLegalHoldStatus", "ObjectLockEventHoldStatus", "ObjectLockEventHoldDuration", "IntelligentTieringAccessTier", "BucketKeyStatus", "ChecksumAlgorithm", "ObjectAccessControlList", "ObjectOwner", "LifecycleExpirationDate"
+    #   resp.inventory_configuration_list[0].optional_fields[0] #=> String, one of "Size", "LastModifiedDate", "StorageClass", "ETag", "IsMultipartUploaded", "ReplicationStatus", "EncryptionStatus", "ObjectLockRetainUntilDate", "ObjectLockMode", "ObjectLockLegalHoldStatus", "ObjectLockEventHoldStatus", "ObjectLockEventHoldDuration", "IntelligentTieringAccessTier", "BucketKeyStatus", "ChecksumAlgorithm", "ObjectAccessControlList", "ObjectOwner", "LifecycleExpirationDate", "IntelligentTieringReferenceDate"
     #   resp.inventory_configuration_list[0].schedule.frequency #=> String, one of "Daily", "Weekly"
     #   resp.is_truncated #=> Boolean
     #   resp.next_continuation_token #=> String
@@ -16319,7 +16319,7 @@ module Aws::S3
     #       },
     #       id: "InventoryId", # required
     #       included_object_versions: "All", # required, accepts All, Current
-    #       optional_fields: ["Size"], # accepts Size, LastModifiedDate, StorageClass, ETag, IsMultipartUploaded, ReplicationStatus, EncryptionStatus, ObjectLockRetainUntilDate, ObjectLockMode, ObjectLockLegalHoldStatus, ObjectLockEventHoldStatus, ObjectLockEventHoldDuration, IntelligentTieringAccessTier, BucketKeyStatus, ChecksumAlgorithm, ObjectAccessControlList, ObjectOwner, LifecycleExpirationDate
+    #       optional_fields: ["Size"], # accepts Size, LastModifiedDate, StorageClass, ETag, IsMultipartUploaded, ReplicationStatus, EncryptionStatus, ObjectLockRetainUntilDate, ObjectLockMode, ObjectLockLegalHoldStatus, ObjectLockEventHoldStatus, ObjectLockEventHoldDuration, IntelligentTieringAccessTier, BucketKeyStatus, ChecksumAlgorithm, ObjectAccessControlList, ObjectOwner, LifecycleExpirationDate, IntelligentTieringReferenceDate
     #       schedule: { # required
     #         frequency: "Daily", # required, accepts Daily, Weekly
     #       },
@@ -24209,7 +24209,7 @@ module Aws::S3
         tracer: tracer
       )
       context[:gem_name] = 'aws-sdk-s3'
-      context[:gem_version] = '1.232.1'
+      context[:gem_version] = '1.233.1'
       Seahorse::Client::Request.new(handlers, context)
     end
 

data/lib/aws-sdk-s3/customizations/object.rb

@@ -380,7 +380,9 @@ module Aws
       #   and {Client#upload_part} can be provided.
       #
       # @option options [Integer] :thread_count (10) The number of parallel multipart uploads.
-      #   An additional thread is used internally for task coordination.
+      #   An additional thread is used internally for task coordination. This also bounds
+      #   how many parts are buffered ahead of the upload, limiting memory usage to roughly
+      #   `2 * :thread_count * :part_size`.
       #
       # @option options [Boolean] :tempfile (false) Normally read data is stored
       #   in memory when building the parts in order to complete the underlying
@@ -405,18 +407,22 @@ module Aws
       # @see Client#upload_part
       def upload_stream(options = {}, &block)
         upload_opts = options.merge(bucket: bucket_name, key: key)
-        executor = DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
-        uploader = MultipartStreamUploader.new(
-          client: client,
-          executor: executor,
-          tempfile: upload_opts.delete(:tempfile),
-          part_size: upload_opts.delete(:part_size)
-        )
-        Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
-          uploader.upload(upload_opts, &block)
+        thread_count = upload_opts.delete(:thread_count) || DefaultExecutor::DEFAULT_MAX_THREADS
+        executor = DefaultExecutor.new(max_threads: thread_count, max_queue: thread_count)
+        begin
+          uploader = MultipartStreamUploader.new(
+            client: client,
+            executor: executor,
+            tempfile: upload_opts.delete(:tempfile),
+            part_size: upload_opts.delete(:part_size)
+          )
+          Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
+            uploader.upload(upload_opts, &block)
+          end
+          true
+        ensure
+          executor.shutdown
         end
-        executor.shutdown
-        true
       end
       deprecated(:upload_stream, use: 'Aws::S3::TransferManager#upload_stream', version: 'next major version')
 
@@ -480,17 +486,20 @@ module Aws
       def upload_file(source, options = {})
         upload_opts = options.merge(bucket: bucket_name, key: key)
         executor = DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
-        uploader = FileUploader.new(
-          client: client,
-          executor: executor,
-          multipart_threshold: upload_opts.delete(:multipart_threshold)
-        )
-        response = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
-          uploader.upload(source, upload_opts)
+        begin
+          uploader = FileUploader.new(
+            client: client,
+            executor: executor,
+            multipart_threshold: upload_opts.delete(:multipart_threshold)
+          )
+          response = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
+            uploader.upload(source, upload_opts)
+          end
+          yield response if block_given?
+          true
+        ensure
+          executor.shutdown
         end
-        yield response if block_given?
-        executor.shutdown
-        true
       end
       deprecated(:upload_file, use: 'Aws::S3::TransferManager#upload_file', version: 'next major version')
 
@@ -561,13 +570,16 @@ module Aws
       # @see Client#head_object
       def download_file(destination, options = {})
         download_opts = options.merge(bucket: bucket_name, key: key)
-        executor = DefaultExecutor.new(max_threads: download_opts.delete([:thread_count]))
-        downloader = FileDownloader.new(client: client, executor: executor)
-        Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
-          downloader.download(destination, download_opts)
+        executor = DefaultExecutor.new(max_threads: download_opts.delete(:thread_count))
+        begin
+          downloader = FileDownloader.new(client: client, executor: executor)
+          Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
+            downloader.download(destination, download_opts)
+          end
+          true
+        ensure
+          executor.shutdown
         end
-        executor.shutdown
-        true
       end
       deprecated(:download_file, use: 'Aws::S3::TransferManager#download_file', version: 'next major version')
 

data/lib/aws-sdk-s3/default_executor.rb

@@ -4,6 +4,13 @@ module Aws
   module S3
     # @api private
     class DefaultExecutor
+      # Raised when a task is posted to an executor that is shutting down or has been shut down.
+      class RejectedExecutionError < RuntimeError
+        def initialize(msg = 'Executor has been shutdown and is no longer accepting tasks')
+          super
+        end
+      end
+
       DEFAULT_MAX_THREADS = 10
       RUNNING = :running
       SHUTTING_DOWN = :shutting_down
@@ -11,10 +18,12 @@ module Aws
 
       def initialize(options = {})
         @max_threads = options[:max_threads] || DEFAULT_MAX_THREADS
+        @max_queue = options[:max_queue] || 0
         @state = RUNNING
-        @queue = Queue.new
+        @queue = @max_queue.zero? ? Queue.new : SizedQueue.new(@max_queue)
         @pool = []
         @mutex = Mutex.new
+        @fatal_error = nil
       end
 
       # Submits a task for execution.
@@ -23,12 +32,15 @@ module Aws
       # @return [Boolean] Returns true if the task was submitted successfully
       def post(*args, &block)
         @mutex.synchronize do
-          raise 'Executor has been shutdown and is no longer accepting tasks' unless @state == RUNNING
+          raise RejectedExecutionError unless @state == RUNNING
 
-          @queue << [args, block]
           ensure_worker_available
         end
+        # Outside the mutex so a caller blocked on a full queue can't hold up #shutdown or #kill.
+        @queue.push([args, block])
         true
+      rescue ClosedQueueError
+        raise RejectedExecutionError
       end
 
       # Immediately terminates all worker threads and clears pending tasks.
@@ -38,6 +50,7 @@ module Aws
       def kill
         @mutex.synchronize do
           @state = SHUTDOWN
+          @queue.close
           @pool.each(&:kill)
           @pool.clear
           @queue.clear
@@ -52,35 +65,58 @@ module Aws
       #   If nil, waits indefinitely. If timeout expires, remaining threads are killed.
       # @return [Boolean] true when shutdown is complete
       def shutdown(timeout = nil)
+        return true unless begin_shutdown
+
+        deadline = timeout && (Time.now + timeout)
+        join_workers(deadline)
+        kill_remaining_workers if timeout
+
+        finalize_shutdown
+        raise @fatal_error if @fatal_error
+
+        true
+      end
+
+      private
+
+      def begin_shutdown
         @mutex.synchronize do
-          return true if @state == SHUTDOWN
+          return false if @state == SHUTDOWN
 
           @state = SHUTTING_DOWN
-          @pool.size.times { @queue << :shutdown }
+          # Close rather than push sentinels, which could block on a full queue.
+          @queue.close
         end
+        true
+      end
 
-        if timeout
-          deadline = Time.now + timeout
-          @pool.each do |thread|
-            remaining = deadline - Time.now
-            break if remaining <= 0
+      def join_workers(deadline)
+        until (threads = @mutex.synchronize { @pool.select(&:alive?) }).empty?
+          threads.each do |thread|
+            remaining = deadline - Time.now if deadline
+            break if remaining && remaining <= 0
 
-            thread.join([remaining, 0].max)
+            begin
+              thread.join(remaining && [remaining, 0].max)
+            rescue Exception # rubocop:disable Lint/RescueException
+              nil # recorded by #replace_worker and raised by #shutdown
+            end
           end
-          @pool.select(&:alive?).each(&:kill)
-        else
-          @pool.each(&:join)
+          break if deadline && Time.now >= deadline
         end
+      end
+
+      def kill_remaining_workers
+        @mutex.synchronize { @pool.select(&:alive?).each(&:kill) }
+      end
 
+      def finalize_shutdown
         @mutex.synchronize do
           @pool.clear
           @state = SHUTDOWN
         end
-        true
       end
 
-      private
-
       def ensure_worker_available
         return unless @state == RUNNING
 
@@ -91,11 +127,21 @@ module Aws
       def spawn_worker
         Thread.new do
           while (job = @queue.shift)
-            break if job == :shutdown
-
             args, block = job
             block.call(*args)
           end
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          # Replace this worker so queued tasks still run.
+          replace_worker(e)
+          raise
+        end
+      end
+
+      def replace_worker(error)
+        @mutex.synchronize do
+          @fatal_error ||= error
+          @pool.delete(Thread.current)
+          @pool << spawn_worker unless @state == SHUTDOWN
         end
       end
     end

data/lib/aws-sdk-s3/multipart_file_uploader.rb

@@ -140,7 +140,7 @@ module Aws
       end
 
       def upload_with_executor(pending, completed, options)
-        upload_attempts = 0
+        queued_parts = 0
         completion_queue = Queue.new
         abort_upload = false
         errors = []
@@ -149,25 +149,33 @@ module Aws
         while (part = pending.shift)
           break if abort_upload
 
-          upload_attempts += 1
-          @executor.post(part) do |p|
-            Thread.current[:net_http_override_body_stream_chunk] = @http_chunk_size if @http_chunk_size
-            update_progress(progress, p)
-            resp = @client.upload_part(p)
-            completed_part = { etag: resp.etag, part_number: p[:part_number] }
-            apply_part_checksum(resp, completed_part)
-            completed.push(completed_part)
+          begin
+            @executor.post(part) do |p|
+              Thread.current[:net_http_override_body_stream_chunk] = @http_chunk_size if @http_chunk_size
+              update_progress(progress, p)
+              resp = @client.upload_part(p)
+              completed_part = { etag: resp.etag, part_number: p[:part_number] }
+              apply_part_checksum(resp, completed_part)
+              completed.push(completed_part)
+            # Any error, or the upload completes without this part.
+            rescue Exception => e # rubocop:disable Lint/RescueException
+              abort_upload = true
+              errors << e
+            ensure
+              p[:body].close
+              Thread.current[:net_http_override_body_stream_chunk] = nil if @http_chunk_size
+              completion_queue << :done
+            end
+            queued_parts += 1
           rescue StandardError => e
+            # Rejected by the executor; abort rather than orphan the upload.
             abort_upload = true
             errors << e
-          ensure
-            p[:body].close
-            Thread.current[:net_http_override_body_stream_chunk] = nil if @http_chunk_size
-            completion_queue << :done
+            break
           end
         end
 
-        upload_attempts.times { completion_queue.pop }
+        queued_parts.times { completion_queue.pop }
         errors
       end
 

data/lib/aws-sdk-s3/multipart_stream_uploader.rb

@@ -113,18 +113,22 @@ module Aws
       def read_to_part_body(read_pipe)
         return if read_pipe.closed?
 
-        temp_io = @tempfile ? Tempfile.new('aws-sdk-s3-upload_stream') : StringIO.new(String.new)
-        temp_io.binmode
-        bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size)
-        temp_io.rewind
-        if bytes_copied.zero?
-          if temp_io.is_a?(Tempfile)
+        if @tempfile
+          temp_io = Tempfile.new('aws-sdk-s3-upload_stream')
+          temp_io.binmode
+          bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size)
+          temp_io.rewind
+          if bytes_copied.zero?
             temp_io.close
             temp_io.unlink
+            nil
+          else
+            temp_io
           end
-          nil
         else
-          temp_io
+          # A single sized read; copy_stream into a StringIO grows by doubling and fragments the heap.
+          data = read_pipe.read(@part_size)
+          data.nil? ? nil : StringIO.new(data)
         end
       end
 
@@ -139,20 +143,31 @@ module Aws
           end
           break unless part_body || current_part_num == 1
 
-          queued_parts += 1
-          @executor.post(part_body, current_part_num, options) do |body, num, opts|
-            part = opts.merge(body: body, part_number: num)
-            resp = @client.upload_part(part)
-            completed_part = create_completed_part(resp, part)
-            completed.push(completed_part)
+          begin
+            @executor.post(part_body, current_part_num, options) do |body, num, opts|
+              part = opts.merge(body: body, part_number: num)
+              resp = @client.upload_part(part)
+              completed_part = create_completed_part(resp, part)
+              completed.push(completed_part)
+            # Any error, or the upload completes without this part.
+            rescue Exception => e # rubocop:disable Lint/RescueException
+              mutex.synchronize do
+                errors.push(e)
+                read_pipe.close_read unless read_pipe.closed?
+              end
+            ensure
+              clear_body(body)
+              completion_queue << :done
+            end
+            queued_parts += 1
           rescue StandardError => e
+            # Rejected by the executor. Closing the pipe stops the producer so the upload can abort.
             mutex.synchronize do
               errors.push(e)
               read_pipe.close_read unless read_pipe.closed?
             end
-          ensure
-            clear_body(body)
-            completion_queue << :done
+            clear_body(part_body)
+            break
           end
         end
         queued_parts.times { completion_queue.pop }

data/lib/aws-sdk-s3/transfer_manager.rb

@@ -160,11 +160,14 @@ module Aws
       #   * `:errors` - Array of errors for failed downloads (only present when failures occur)
       def download_directory(destination, bucket:, **options)
         Aws::Plugins::UserAgent.metric('S3_TRANSFER', 'S3_TRANSFER_DOWNLOAD_DIRECTORY') do
-          executor = @executor || DefaultExecutor.new(max_threads: options.delete(:thread_count))
-          downloader = DirectoryDownloader.new(client: @client, executor: executor, logger: @logger)
-          result = downloader.download(destination, bucket: bucket, **options)
-          executor.shutdown unless @executor
-          result
+          thread_count = options.delete(:thread_count)
+          executor = @executor || DefaultExecutor.new(max_threads: thread_count)
+          begin
+            downloader = DirectoryDownloader.new(client: @client, executor: executor, logger: @logger)
+            downloader.download(destination, bucket: bucket, **options)
+          ensure
+            executor.shutdown unless @executor
+          end
         end
       end
 
@@ -245,11 +248,15 @@ module Aws
       # @see Client#head_object
       def download_file(destination, bucket:, key:, **options)
         download_opts = options.merge(bucket: bucket, key: key)
-        executor = @executor || DefaultExecutor.new(max_threads: download_opts.delete(:thread_count))
-        downloader = FileDownloader.new(client: @client, executor: executor)
-        downloader.download(destination, download_opts)
-        executor.shutdown unless @executor
-        true
+        thread_count = download_opts.delete(:thread_count)
+        executor = @executor || DefaultExecutor.new(max_threads: thread_count)
+        begin
+          downloader = FileDownloader.new(client: @client, executor: executor)
+          downloader.download(destination, download_opts)
+          true
+        ensure
+          executor.shutdown unless @executor
+        end
       end
 
       # Uploads all files under the given directory to the provided S3 bucket.
@@ -360,11 +367,14 @@ module Aws
       #   * `:errors` - Array of error objects for failed uploads (only present when failures occur)
       def upload_directory(source, bucket:, **options)
         Aws::Plugins::UserAgent.metric('S3_TRANSFER', 'S3_TRANSFER_UPLOAD_DIRECTORY') do
-          executor = @executor || DefaultExecutor.new(max_threads: options.delete(:thread_count))
-          uploader = DirectoryUploader.new(client: @client, executor: executor, logger: @logger)
-          result = uploader.upload(source, bucket, **options.merge(http_chunk_size: resolve_http_chunk_size(options)))
-          executor.shutdown unless @executor
-          result
+          thread_count = options.delete(:thread_count)
+          executor = @executor || DefaultExecutor.new(max_threads: thread_count)
+          begin
+            uploader = DirectoryUploader.new(client: @client, executor: executor, logger: @logger)
+            uploader.upload(source, bucket, **options.merge(http_chunk_size: resolve_http_chunk_size(options)))
+          ensure
+            executor.shutdown unless @executor
+          end
         end
       end
 
@@ -443,18 +453,21 @@ module Aws
       def upload_file(source, bucket:, key:, **options)
         upload_opts = options.merge(bucket: bucket, key: key)
         http_chunk_size = resolve_http_chunk_size(upload_opts)
-
-        executor = @executor || DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
-        uploader = FileUploader.new(
-          multipart_threshold: upload_opts.delete(:multipart_threshold),
-          http_chunk_size: http_chunk_size,
-          client: @client,
-          executor: executor
-        )
-        response = uploader.upload(source, upload_opts)
-        yield response if block_given?
-        executor.shutdown unless @executor
-        true
+        thread_count = upload_opts.delete(:thread_count)
+        executor = @executor || DefaultExecutor.new(max_threads: thread_count)
+        begin
+          uploader = FileUploader.new(
+            multipart_threshold: upload_opts.delete(:multipart_threshold),
+            http_chunk_size: http_chunk_size,
+            client: @client,
+            executor: executor
+          )
+          response = uploader.upload(source, upload_opts)
+          yield response if block_given?
+          true
+        ensure
+          executor.shutdown unless @executor
+        end
       end
 
       # Uploads a stream in a streaming fashion to S3.
@@ -491,6 +504,9 @@ module Aws
       # @option options [Integer] :thread_count (10)
       #   The number of parallel multipart uploads. Only used when no custom executor is provided (creates
       #   {DefaultExecutor} with the given thread count). An additional thread is used internally for task coordination.
+      #   This also bounds how many parts are buffered ahead of the upload, limiting memory usage to roughly
+      #   `2 * :thread_count * :part_size`. When a custom `:executor` is provided, it is responsible for applying
+      #   its own backpressure.
       #
       # @option options [Boolean] :tempfile (false)
       #   Normally read data is stored in memory when building the parts in order to complete the underlying
@@ -511,16 +527,20 @@ module Aws
       # @see Client#upload_part
       def upload_stream(bucket:, key:, **options, &block)
         upload_opts = options.merge(bucket: bucket, key: key)
-        executor = @executor || DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
-        uploader = MultipartStreamUploader.new(
-          client: @client,
-          executor: executor,
-          tempfile: upload_opts.delete(:tempfile),
-          part_size: upload_opts.delete(:part_size)
-        )
-        uploader.upload(upload_opts, &block)
-        executor.shutdown unless @executor
-        true
+        thread_count = upload_opts.delete(:thread_count) || DefaultExecutor::DEFAULT_MAX_THREADS
+        executor = @executor || DefaultExecutor.new(max_threads: thread_count, max_queue: thread_count)
+        begin
+          uploader = MultipartStreamUploader.new(
+            client: @client,
+            executor: executor,
+            tempfile: upload_opts.delete(:tempfile),
+            part_size: upload_opts.delete(:part_size)
+          )
+          uploader.upload(upload_opts, &block)
+          true
+        ensure
+          executor.shutdown unless @executor
+        end
       end
 
       private

data/lib/aws-sdk-s3.rb

@@ -75,7 +75,7 @@ module Aws::S3
   autoload :ObjectVersion, 'aws-sdk-s3/object_version'
   autoload :EventStreams, 'aws-sdk-s3/event_streams'
 
-  GEM_VERSION = '1.232.1'
+  GEM_VERSION = '1.233.1'
 
 end
 

data/sig/client.rbs

@@ -1671,7 +1671,7 @@ module Aws
                                                   }?,
                                                   id: ::String,
                                                   included_object_versions: ("All" | "Current"),
-                                                  optional_fields: Array[("Size" | "LastModifiedDate" | "StorageClass" | "ETag" | "IsMultipartUploaded" | "ReplicationStatus" | "EncryptionStatus" | "ObjectLockRetainUntilDate" | "ObjectLockMode" | "ObjectLockLegalHoldStatus" | "ObjectLockEventHoldStatus" | "ObjectLockEventHoldDuration" | "IntelligentTieringAccessTier" | "BucketKeyStatus" | "ChecksumAlgorithm" | "ObjectAccessControlList" | "ObjectOwner" | "LifecycleExpirationDate")]?,
+                                                  optional_fields: Array[("Size" | "LastModifiedDate" | "StorageClass" | "ETag" | "IsMultipartUploaded" | "ReplicationStatus" | "EncryptionStatus" | "ObjectLockRetainUntilDate" | "ObjectLockMode" | "ObjectLockLegalHoldStatus" | "ObjectLockEventHoldStatus" | "ObjectLockEventHoldDuration" | "IntelligentTieringAccessTier" | "BucketKeyStatus" | "ChecksumAlgorithm" | "ObjectAccessControlList" | "ObjectOwner" | "LifecycleExpirationDate" | "IntelligentTieringReferenceDate")]?,
                                                   schedule: {
                                                     frequency: ("Daily" | "Weekly")
                                                   }

data/sig/types.rbs

@@ -1515,7 +1515,7 @@ module Aws::S3
       attr_accessor filter: Types::InventoryFilter
       attr_accessor id: ::String
       attr_accessor included_object_versions: ("All" | "Current")
-      attr_accessor optional_fields: ::Array[("Size" | "LastModifiedDate" | "StorageClass" | "ETag" | "IsMultipartUploaded" | "ReplicationStatus" | "EncryptionStatus" | "ObjectLockRetainUntilDate" | "ObjectLockMode" | "ObjectLockLegalHoldStatus" | "ObjectLockEventHoldStatus" | "ObjectLockEventHoldDuration" | "IntelligentTieringAccessTier" | "BucketKeyStatus" | "ChecksumAlgorithm" | "ObjectAccessControlList" | "ObjectOwner" | "LifecycleExpirationDate")]
+      attr_accessor optional_fields: ::Array[("Size" | "LastModifiedDate" | "StorageClass" | "ETag" | "IsMultipartUploaded" | "ReplicationStatus" | "EncryptionStatus" | "ObjectLockRetainUntilDate" | "ObjectLockMode" | "ObjectLockLegalHoldStatus" | "ObjectLockEventHoldStatus" | "ObjectLockEventHoldDuration" | "IntelligentTieringAccessTier" | "BucketKeyStatus" | "ChecksumAlgorithm" | "ObjectAccessControlList" | "ObjectOwner" | "LifecycleExpirationDate" | "IntelligentTieringReferenceDate")]
       attr_accessor schedule: Types::InventorySchedule
       SENSITIVE: []
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws-sdk-s3
 version: !ruby/object:Gem::Version
-  version: 1.232.1
+  version: 1.233.1
 platform: ruby
 authors:
 - Amazon Web Services