aws-sdk-s3 1.213.0 → 1.214.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 438515448f62837952b523ea5cb8f41592cb39e2afa557b30c33c521a307db0c
-  data.tar.gz: fcb1cc7e9cc73429393692857809f2dc156bec4e6b93eba0ccf7824e4ede0d35
+  metadata.gz: 07e46262bf85414dd7871d9d15e79bb1d49a233e8afcc28d33303c83cbd93c89
+  data.tar.gz: 379e303c06523e12f6669689b540ba3338d4a4bb2f98705bdceef069bd51e445
 SHA512:
-  metadata.gz: 7d2ae919bc1d2e451538e35c2e4a61ed087266cf4b3205eb64aba047a6a75039e2c431cd74844a71176eec93ec3b9fbe387effd734c3342c7aef76a4571c949d
-  data.tar.gz: cf67bd5b39c6cd5264a86f71103324c6f19bca24b7ff355b1ab81a70a4d9f2d41c195043e96fcf56f6fcba563f7161b7e17ecd163fd53a941b096bfb3f8b5410
+  metadata.gz: 7748aaaaa2db0d367ea16ef7732a095c2ba45a143fc16f1dd0e4542fed575976227660e188ec05c88c4dfc539468a3b79b0ac52c149b05202026e6a4c8e2bb89
+  data.tar.gz: 49ec7554aca1a1f8a4acb23091265b594b8dfbd51a07bccab4ed8cdc49817adb645b5a6fa5132532356e4010079252aa2850b9012bc46956d7c12a555f255853

data/CHANGELOG.md

@@ -1,6 +1,11 @@
 Unreleased Changes
 ------------------
 
+1.214.0 (2026-03-04)
+------------------
+
+* Feature - Added `#upload_directory` and `#download_directory` to `Aws::S3::TransferManager` for bulk directory transfers.
+
 1.213.0 (2026-01-28)
 ------------------
 

data/VERSION

@@ -1 +1 @@
-1.213.0
+1.214.0

data/lib/aws-sdk-s3/client.rb

@@ -22553,7 +22553,7 @@ module Aws::S3
         tracer: tracer
       )
       context[:gem_name] = 'aws-sdk-s3'
-      context[:gem_version] = '1.213.0'
+      context[:gem_version] = '1.214.0'
       Seahorse::Client::Request.new(handlers, context)
     end
 

data/lib/aws-sdk-s3/customizations.rb

@@ -7,15 +7,22 @@ module Aws
     autoload :Encryption, 'aws-sdk-s3/encryption'
     autoload :EncryptionV2, 'aws-sdk-s3/encryption_v2'
     autoload :EncryptionV3, 'aws-sdk-s3/encryption_v3'
-    autoload :FilePart, 'aws-sdk-s3/file_part'
+    autoload :LegacySigner, 'aws-sdk-s3/legacy_signer'
+
+    # transfer manager + multipart upload/download utilities
     autoload :DefaultExecutor, 'aws-sdk-s3/default_executor'
+    autoload :FilePart, 'aws-sdk-s3/file_part'
     autoload :FileUploader, 'aws-sdk-s3/file_uploader'
     autoload :FileDownloader, 'aws-sdk-s3/file_downloader'
-    autoload :LegacySigner, 'aws-sdk-s3/legacy_signer'
     autoload :MultipartDownloadError, 'aws-sdk-s3/multipart_download_error'
     autoload :MultipartFileUploader, 'aws-sdk-s3/multipart_file_uploader'
     autoload :MultipartStreamUploader, 'aws-sdk-s3/multipart_stream_uploader'
     autoload :MultipartUploadError, 'aws-sdk-s3/multipart_upload_error'
+    autoload :DirectoryProgress, 'aws-sdk-s3/directory_progress'
+    autoload :DirectoryDownloadError, 'aws-sdk-s3/directory_download_error'
+    autoload :DirectoryDownloader, 'aws-sdk-s3/directory_downloader'
+    autoload :DirectoryUploadError, 'aws-sdk-s3/directory_upload_error'
+    autoload :DirectoryUploader, 'aws-sdk-s3/directory_uploader'
     autoload :ObjectCopier, 'aws-sdk-s3/object_copier'
     autoload :ObjectMultipartCopier, 'aws-sdk-s3/object_multipart_copier'
     autoload :PresignedPost, 'aws-sdk-s3/presigned_post'

data/lib/aws-sdk-s3/directory_download_error.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    # Raised when DirectoryDownloader fails to download objects from S3 bucket
+    class DirectoryDownloadError < StandardError
+      def initialize(message, errors = [])
+        @errors = errors
+        super(message)
+      end
+
+      # @return [Array<StandardError>] The list of errors encountered when downloading objects
+      attr_reader :errors
+    end
+  end
+end

data/lib/aws-sdk-s3/directory_downloader.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    # @api private
+    # This is a one-shot class that downloads objects from a bucket to a local directory.
+    # This works as follows:
+    # * ObjectProducer runs in a background thread, calling `list_objects_v2` and
+    #   pushing entries into a SizedQueue (max: 100).
+    # * An internal executor pulls from that queue and posts work. Each task uses
+    #   FileDownloader to download objects then signals completion via `completion_queue`.
+    #
+    # We track how many tasks we posted, then pop that many times from `completion_queue`
+    # to wait for everything to finish.
+    #
+    # Errors are collected in a mutex-protected array. On failure (unless ignore_failure is set),
+    # we call abort which closes the queue - the producer catches ClosedQueueError and exits cleanly.
+    class DirectoryDownloader
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+        @executor = options[:executor] || DefaultExecutor.new
+        @logger = options[:logger]
+        @producer = nil
+        @mutex = Mutex.new
+      end
+
+      attr_reader :client, :executor
+
+      def abort
+        @producer&.close
+      end
+
+      def download(destination, bucket:, **options)
+        if File.exist?(destination)
+          raise ArgumentError, 'invalid destination, expected a directory' unless File.directory?(destination)
+        else
+          FileUtils.mkdir_p(destination)
+        end
+
+        download_opts = build_download_opts(destination, options)
+        @producer = ObjectProducer.new(build_producer_opts(destination, bucket, options))
+        downloader = FileDownloader.new(client: @client, executor: @executor)
+        downloads, errors = process_download_queue(downloader, download_opts)
+        build_result(downloads, errors)
+      end
+
+      private
+
+      def build_download_opts(destination, opts)
+        {
+          destination: destination,
+          ignore_failure: opts[:ignore_failure] || false
+        }
+      end
+
+      def build_producer_opts(destination, bucket, opts)
+        {
+          client: @client,
+          directory_downloader: self,
+          destination: destination,
+          bucket: bucket,
+          s3_prefix: opts[:s3_prefix],
+          filter_callback: opts[:filter_callback],
+          request_callback: opts[:request_callback]
+        }
+      end
+
+      def build_result(download_count, errors)
+        if @producer&.closed?
+          msg = "directory download failed: #{errors.map(&:message).join('; ')}"
+          raise DirectoryDownloadError.new(msg, errors)
+        else
+          {
+            completed_downloads: [download_count - errors.count, 0].max,
+            failed_downloads: errors.count,
+            errors: errors.any? ? errors : nil
+          }.compact
+        end
+      end
+
+      def download_object(entry, downloader, errors, opts)
+        raise entry.error if entry.error
+
+        FileUtils.mkdir_p(File.dirname(entry.path)) unless Dir.exist?(File.dirname(entry.path))
+        downloader.download(entry.path, entry.params)
+        @logger&.debug("Downloaded #{entry.params[:key]} from #{entry.params[:bucket]} to #{entry.path}")
+      rescue StandardError => e
+        @logger&.warn("Failed to download #{entry.params[:key]} from #{entry.params[:bucket]}: #{e.message}")
+        @mutex.synchronize { errors << e }
+        abort unless opts[:ignore_failure]
+      end
+
+      def process_download_queue(downloader, opts)
+        queue_executor = DefaultExecutor.new(max_threads: 2)
+        completion_queue = Queue.new
+        posted_count = 0
+        errors = []
+        begin
+          @producer.each do |object|
+            queue_executor.post(object) do |o|
+              download_object(o, downloader, errors, opts)
+            ensure
+              completion_queue << :done
+            end
+            posted_count += 1
+          end
+        rescue ClosedQueueError
+          # abort already requested
+        rescue StandardError => e
+          @mutex.synchronize { errors << e }
+          abort
+        end
+        posted_count.times { completion_queue.pop }
+        [posted_count, errors]
+      ensure
+        queue_executor&.shutdown
+      end
+
+      # @api private
+      class ObjectProducer
+        include Enumerable
+
+        DEFAULT_QUEUE_SIZE = 100
+        DONE_MARKER = :done
+
+        def initialize(opts = {})
+          @directory_downloader = opts[:directory_downloader]
+          @destination_dir = opts[:destination]
+          @bucket = opts[:bucket]
+          @client = opts[:client]
+          @s3_prefix = opts[:s3_prefix]
+          @filter_callback = opts[:filter_callback]
+          @request_callback = opts[:request_callback]
+          @object_queue = SizedQueue.new(DEFAULT_QUEUE_SIZE)
+        end
+
+        def closed?
+          @object_queue.closed?
+        end
+
+        def close
+          @object_queue.close
+          @object_queue.clear
+        end
+
+        def each
+          producer_thread = Thread.new do
+            stream_objects
+            @object_queue << DONE_MARKER
+          rescue ClosedQueueError
+            # abort requested
+          rescue StandardError => e
+            close
+            raise e
+          end
+
+          while (object = @object_queue.shift) && object != DONE_MARKER
+            yield object
+          end
+        ensure
+          producer_thread.value
+        end
+
+        private
+
+        def apply_request_callback(key, params)
+          callback_params = @request_callback.call(key, params.dup)
+          return params unless callback_params.is_a?(Hash) && callback_params.any?
+
+          params.merge(callback_params)
+        end
+
+        def build_object_entry(key)
+          params = { bucket: @bucket, key: key }
+          params = apply_request_callback(key, params) if @request_callback
+          error = validate_key(key)
+          return DownloadEntry.new(path: '', params: params, error: error) if error
+
+          full_path = normalize_path(File.join(@destination_dir, key))
+          DownloadEntry.new(path: full_path, params: params, error: error)
+        end
+
+        def include_object?(obj)
+          return true unless @filter_callback
+
+          @filter_callback.call(obj)
+        end
+
+        def directory_marker?(obj)
+          obj.key.end_with?('/') && obj.size.zero?
+        end
+
+        def normalize_path(path)
+          return path if File::SEPARATOR == '/'
+
+          path.tr('/', File::SEPARATOR)
+        end
+
+        def stream_objects(continuation_token: nil)
+          resp = @client.list_objects_v2(bucket: @bucket, prefix: @s3_prefix, continuation_token: continuation_token)
+          resp.contents&.each do |o|
+            next if directory_marker?(o)
+            next unless include_object?(o)
+
+            @object_queue << build_object_entry(o.key)
+          end
+          stream_objects(continuation_token: resp.next_continuation_token) if resp.next_continuation_token
+        end
+
+        def validate_key(key)
+          segments = key.split('/')
+          return unless segments.any? { |s| %w[. ..].include?(s) }
+
+          DirectoryDownloadError.new("invalid key '#{key}': contains '.' or '..' path segments")
+        end
+
+        # @api private
+        class DownloadEntry
+          def initialize(opts = {})
+            @path = opts[:path]
+            @params = opts[:params]
+            @error = opts[:error]
+          end
+
+          attr_reader :path, :params, :error
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/directory_upload_error.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    # Raised when DirectoryUploader fails to upload files to S3 bucket
+    class DirectoryUploadError < StandardError
+      def initialize(message, errors = [])
+        @errors = errors
+        super(message)
+      end
+
+      # @return [Array<StandardError>] The list of errors encountered when uploading files
+      attr_reader :errors
+    end
+  end
+end

data/lib/aws-sdk-s3/directory_uploader.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Aws
+  module S3
+    # @api private
+    # This is a one-shot class that uploads files from a local directory to a bucket.
+    # This works as follows:
+    # * FileProducer runs in a background thread, scanning the directory and
+    #   pushing entries into a SizedQueue (max: 100).
+    # * An internal executor pulls from that queue and posts work. Each task uses
+    #   FileUploader to upload files then signals completion via `completion_queue`.
+    #
+    # We track how many tasks we posted, then pop that many times from `completion_queue`
+    # to wait for everything to finish.
+    #
+    # Errors are collected in a mutex-protected array. On failure (unless ignore_failure is set),
+    # we call abort which closes the queue - the producer catches ClosedQueueError and exits cleanly.
+    class DirectoryUploader
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+        @executor = options[:executor] || DefaultExecutor.new
+        @logger = options[:logger]
+        @producer = nil
+        @mutex = Mutex.new
+      end
+
+      attr_reader :client, :executor
+
+      def abort
+        @producer&.close
+      end
+
+      def upload(source_directory, bucket, **opts)
+        raise ArgumentError, 'Invalid directory' unless Dir.exist?(source_directory)
+
+        uploader = FileUploader.new(
+          multipart_threshold: opts.delete(:multipart_threshold),
+          http_chunk_size: opts.delete(:http_chunk_size),
+          client: @client,
+          executor: @executor
+        )
+        upload_opts = build_upload_opts(opts)
+        @producer = FileProducer.new(build_producer_opts(source_directory, bucket, opts))
+        uploads, errors = process_upload_queue(uploader, upload_opts)
+        build_result(uploads, errors)
+      end
+
+      private
+
+      def build_upload_opts(opts)
+        { ignore_failure: opts[:ignore_failure] || false }
+      end
+
+      def build_producer_opts(source_directory, bucket, opts)
+        {
+          directory_uploader: self,
+          source_dir: source_directory,
+          bucket: bucket,
+          s3_prefix: opts[:s3_prefix],
+          recursive: opts[:recursive] || false,
+          follow_symlinks: opts[:follow_symlinks] || false,
+          filter_callback: opts[:filter_callback],
+          request_callback: opts[:request_callback]
+        }
+      end
+
+      def build_result(upload_count, errors)
+        if @producer&.closed?
+          msg = "directory upload failed: #{errors.map(&:message).join('; ')}"
+          raise DirectoryUploadError.new(msg, errors)
+        else
+          {
+            completed_uploads: [upload_count - errors.count, 0].max,
+            failed_uploads: errors.count,
+            errors: errors.any? ? errors : nil
+          }.compact
+        end
+      end
+
+      def process_upload_queue(uploader, opts)
+        queue_executor = DefaultExecutor.new(max_threads: 2)
+        completion_queue = Queue.new
+        posted_count = 0
+        errors = []
+        begin
+          @producer.each do |file|
+            queue_executor.post(file) do |f|
+              upload_file(f, uploader, errors, opts)
+            ensure
+              completion_queue << :done
+            end
+            posted_count += 1
+          end
+        rescue ClosedQueueError
+          # abort already requested
+        rescue StandardError => e
+          @mutex.synchronize { errors << e }
+          abort
+        end
+        posted_count.times { completion_queue.pop }
+        [posted_count, errors]
+      ensure
+        queue_executor&.shutdown
+      end
+
+      def upload_file(entry, uploader, errors, opts)
+        uploader.upload(entry.path, entry.params)
+        @logger&.debug("Uploaded #{entry.path} to #{entry.params[:bucket]} as #{entry.params[:key]}")
+      rescue StandardError => e
+        @logger&.warn("Failed to upload #{entry.path} to #{entry.params[:bucket]}: #{e.message}")
+        @mutex.synchronize { errors << e }
+        abort unless opts[:ignore_failure]
+      end
+
+      # @api private
+      class FileProducer
+        include Enumerable
+
+        DEFAULT_QUEUE_SIZE = 100
+        DONE_MARKER = :done
+
+        def initialize(opts = {})
+          @directory_uploader = opts[:directory_uploader]
+          @source_dir = opts[:source_dir]
+          @bucket = opts[:bucket]
+          @s3_prefix = opts[:s3_prefix]
+          @recursive = opts[:recursive]
+          @follow_symlinks = opts[:follow_symlinks]
+          @filter_callback = opts[:filter_callback]
+          @request_callback = opts[:request_callback]
+          @file_queue = SizedQueue.new(DEFAULT_QUEUE_SIZE)
+        end
+
+        def closed?
+          @file_queue.closed?
+        end
+
+        def close
+          @file_queue.close
+          @file_queue.clear
+        end
+
+        def each
+          producer_thread = Thread.new do
+            if @recursive
+              find_recursively
+            else
+              find_directly
+            end
+            @file_queue << DONE_MARKER
+          rescue ClosedQueueError
+            # abort requested
+          rescue StandardError => e
+            # encountered a traversal error, we must abort immediately
+            close
+            raise DirectoryUploadError, "Directory traversal failed for '#{@source_dir}': #{e.message}"
+          end
+
+          while (file = @file_queue.shift) && file != DONE_MARKER
+            yield file
+          end
+        ensure
+          producer_thread.value
+        end
+
+        private
+
+        def apply_request_callback(file_path, params)
+          callback_params = @request_callback.call(file_path, params.dup)
+          return params unless callback_params.is_a?(Hash) && callback_params.any?
+
+          params.merge(callback_params)
+        end
+
+        def build_upload_entry(file_path, key)
+          params = { bucket: @bucket, key: @s3_prefix ? File.join(@s3_prefix, key) : key }
+          params = apply_request_callback(file_path, params) if @request_callback
+          UploadEntry.new(path: file_path, params: params)
+        end
+
+        def find_directly
+          Dir.each_child(@source_dir) do |entry|
+            entry_path = File.join(@source_dir, entry)
+            stat = nil
+
+            if @follow_symlinks
+              stat = File.stat(entry_path)
+              next if stat.directory?
+            else
+              stat = File.lstat(entry_path)
+              next if stat.symlink? || stat.directory?
+            end
+
+            next unless stat.file?
+            next unless include_file?(entry_path, entry)
+
+            @file_queue << build_upload_entry(entry_path, entry)
+          end
+        end
+
+        def find_recursively
+          if @follow_symlinks
+            ancestors = Set.new
+            ancestors << File.stat(@source_dir).ino
+            scan_directory(@source_dir, ancestors: ancestors)
+          else
+            scan_directory(@source_dir)
+          end
+        end
+
+        def include_file?(file_path, file_name)
+          return true unless @filter_callback
+
+          @filter_callback.call(file_path, file_name)
+        end
+
+        def scan_directory(dir_path, key_prefix: '', ancestors: nil)
+          Dir.each_child(dir_path) do |entry|
+            full_path = File.join(dir_path, entry)
+            next unless include_file?(full_path, entry)
+
+            stat = get_file_stat(full_path)
+            next unless stat
+
+            if stat.directory?
+              handle_directory(full_path, entry, key_prefix, ancestors)
+            elsif stat.file? # skip non-file types
+              key = key_prefix.empty? ? entry : File.join(key_prefix, entry)
+              @file_queue << build_upload_entry(full_path, key)
+            end
+          end
+        end
+
+        def get_file_stat(full_path)
+          return File.stat(full_path) if @follow_symlinks
+
+          lstat = File.lstat(full_path)
+          return if lstat.symlink?
+
+          lstat
+        end
+
+        def handle_directory(dir_path, dir_name, key_prefix, ancestors)
+          ino = nil
+          if @follow_symlinks && ancestors
+            ino = File.stat(dir_path).ino
+            return if ancestors.include?(ino) # cycle detected - skip
+
+            ancestors.add(ino)
+          end
+          new_prefix = key_prefix.empty? ? dir_name : File.join(key_prefix, dir_name)
+          scan_directory(dir_path, key_prefix: new_prefix, ancestors: ancestors)
+          ancestors.delete(ino) if @follow_symlinks && ancestors
+        end
+
+        # @api private
+        class UploadEntry
+          def initialize(opts = {})
+            @path = opts[:path]
+            @params = opts[:params]
+          end
+
+          attr_reader :path, :params
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/transfer_manager.rb

@@ -7,7 +7,9 @@ module Aws
     #
     # * upload a file with multipart upload
     # * upload a stream with multipart upload
-    # * download a S3 object with multipart download
+    # * upload all files in a directory to an S3 bucket recursively or non-recursively
+    # * download an S3 object with multipart download
+    # * download all objects in an S3 bucket with same prefix to a local directory
     # * track transfer progress by using progress listener
     #
     # ## Executor Management
@@ -49,19 +51,21 @@ module Aws
     #     executor.shutdown # You must shutdown custom executors
     #
     class TransferManager
-
       # @param [Hash] options
       # @option options [S3::Client] :client (S3::Client.new)
       #   The S3 client to use for {TransferManager} operations. If not provided, a new default client
       #   will be created automatically.
-      # @option options [Object] :executor
+      # @option options [Object] :executor (nil)
       #   The executor to use for multipart operations. Must implement the same interface as {DefaultExecutor}.
       #   If not provided, a new {DefaultExecutor} will be created automatically for each operation and
       #   shutdown after completion. When provided a custom executor, it will be reused across operations, and
       #   you are responsible for shutting it down when finished.
+      # @option options [Logger] :logger (nil)
+      #   The Logger instance for logging transfer operations. If not set, logging is disabled.
       def initialize(options = {})
         @client = options[:client] || Client.new
         @executor = options[:executor]
+        @logger = options[:logger]
       end
 
       # @return [S3::Client]
@@ -70,6 +74,98 @@ module Aws
       # @return [Object]
       attr_reader :executor
 
+      # @return [Logger]
+      attr_reader :logger
+
+      # Downloads objects in a S3 bucket to a local directory.
+      #
+      # The downloaded directory structure will match the provided S3 virtual bucket. For example,
+      # assume that you have the following keys in your bucket:
+      #
+      # * sample.jpg
+      # * photos/2022/January/sample.jpg
+      # * photos/2022/February/sample1.jpg
+      # * photos/2022/February/sample2.jpg
+      # * photos/2022/February/sample3.jpg
+      #
+      # Given a request to download bucket to a destination with path of `/test`, the downloaded
+      # directory would look like this:
+      #
+      # ```
+      # |- test
+      #   |- sample.jpg
+      #   |- photos
+      #      |- 2022
+      #          |- January
+      #             |- sample.jpg
+      #          |- February
+      #             |- sample1.jpg
+      #             |- sample2.jpg
+      #             |- sample3.jpg
+      # ```
+      #
+      # Directory markers (zero-byte objects ending with `/`) are skipped during download.
+      # Existing files with same name as downloaded objects will be overwritten.
+      #
+      # Object keys containing path traversal sequences (`..` or `.`) will raise an error.
+      #
+      # @example Downloading buckets to a local directory
+      #     tm = TransferManager.new
+      #     tm.download_directory('/local/path', bucket: 'my-bucket')
+      #     # => {completed_downloads: 7, failed_downloads: 0, errors: 0}
+      #
+      # @param [String] destination
+      #  The location directory path to download objects to. Created if it doesn't exist.
+      #  If files with the same names already exist in the destination, they will be overwritten.
+      #
+      # @param [String] bucket
+      #   The name of the bucket to download from.
+      #
+      # @param [Hash] options
+      #
+      # @option options [String] :s3_prefix (nil)
+      #   Limit the download to objects that begin with the specified prefix. The prefix is
+      #   passed directly to the S3 ListObjectsV2 API for filtering. To match only objects
+      #   within a specific "folder", include a trailing `/` (e.g., `"photos/"` instead of
+      #   `"photos"`). The full object key is preserved in the local file path.
+      #
+      # @option options [Boolean] :ignore_failure (false)
+      #   How to handle individual file download failures:
+      #   * `false` (default) - Cancel all ongoing requests, terminate the ongoing downloads and raise an exception
+      #   * `true` - Continue downloading remaining objects, report failures in result.
+      #
+      # @option options [Proc] :filter_callback (nil)
+      #   A Proc to filter which objects to download. Called with `(object)` for each object.
+      #   Return `true` to download the object, `false` to skip it.
+      #
+      # @option options [Proc] :request_callback (nil)
+      #   A Proc to modify download parameters for each object. Called with `(key, params)`.
+      #   Must return the modified parameters.
+      #
+      # @option options [Integer] :thread_count (10)
+      #   The number of threads to use for multipart downloads of individual large files.
+      #   Only used when no custom executor is provided to the {TransferManager}.
+      #
+      # @note On case-insensitive filesystems (e.g., Windows, macOS default), S3 object keys that
+      #   differ only by case (e.g., "File.txt" and "file.txt") may overwrite each other when
+      #   downloaded. This condition is not automatically detected. Use the `:filter_callback`
+      #   option to handle such conflicts if needed.
+      #
+      # @raise [DirectoryDownloadError] Raised when download fails with `ignore_failure: false` (default)
+      #
+      # @return [Hash] Returns a hash with download statistics:
+      #
+      #   * `:completed_downloads` - Number of objects successfully downloaded
+      #   * `:failed_downloads` - Number of objects that failed to download
+      #   * `:errors` - Array of errors for failed downloads (only present when failures occur)
+      def download_directory(destination, bucket:, **options)
+        executor = @executor || DefaultExecutor.new(max_threads: options.delete(:thread_count))
+        downloader = DirectoryDownloader.new(client: @client, executor: executor, logger: @logger)
+        result = downloader.download(destination, bucket: bucket, **options)
+        executor.shutdown unless @executor
+        result
+      end
+
       # Downloads a file in S3 to a path on disk.
       #
       #     # small files (< 5MB) are downloaded in a single API call
@@ -120,8 +216,9 @@ module Aws
       #
       # @option options [Integer] :chunk_size required in `"get_range"` mode.
       #
-      # @option options [Integer] :thread_count (10) Customize threads used in the multipart download.
-      #   Only used when no custom executor is provided (creates {DefaultExecutor} with given thread count).
+      # @option options [Integer] :thread_count (10)
+      #   The number of threads to use for multipart downloads.
+      #   Only used when no custom executor is provided to the {TransferManager}.
       #
       # @option options [String] :checksum_mode ("ENABLED")
       #   This option is deprecated. Use `:response_checksum_validation` on your S3 client instead.
@@ -153,6 +250,120 @@ module Aws
         true
       end
 
+      # Uploads all files under the given directory to the provided S3 bucket.
+      # The key name transformation depends on the optional prefix.
+      #
+      # By default, all subdirectories will be uploaded non-recursively and symbolic links are not
+      # followed automatically. Assume you have a local directory `/test` with the following structure:
+      #
+      # ```
+      # |- test
+      #   |- sample.jpg
+      #   |- photos
+      #      |- 2022
+      #          |- January
+      #             |- sample.jpg
+      #          |- February
+      #             |- sample1.jpg
+      #             |- sample2.jpg
+      #             |- sample3.jpg
+      # ```
+      #
+      # Give a request to upload directory `/test` to an S3 bucket on default setting, the target bucket will have the
+      # following S3 objects:
+      #
+      # * sample.jpg
+      #
+      # If `:recursive` set to `true`, the target bucket will have the following S3 buckets:
+      #
+      # * sample.jpg
+      # * photos/2022/January/sample.jpg
+      # * photos/2022/February/sample1.jpg
+      # * photos/2022/February/sample2.jpg
+      # * photos/2022/February/sample3.jpg
+      #
+      # Only regular files are uploaded; special files (sockets, pipes, devices) are skipped.
+      # Symlink cycles are detected and skipped when following symlinks.
+      # Empty directories are not represented in S3. Existing S3 objects with the same key are
+      # overwritten.
+      #
+      # @example Uploading a directory
+      #     tm = TransferManager.new
+      #     tm.upload_directory('/path/to/directory', bucket: 'bucket')
+      #     # => {completed_uploads: 7, failed_uploads: 0}
+      #
+      # @example Using filter callback to upload only text files
+      #     tm = TransferManager.new
+      #     filter = proc do |file_path, file_name|
+      #       File.extname(file_name) == '.txt'  # Only upload .txt files
+      #     end
+      #     tm.upload_directory('/path/to/directory', bucket: 'bucket', filter_callback: filter)
+      #
+      # @param [String, Pathname, File, Tempfile] source
+      #  The source directory to upload.
+      #
+      # @param [String] bucket
+      #   The name of the bucket to upload objects to.
+      #
+      # @param [Hash] options
+      #
+      # @option options [String] :s3_prefix (nil)
+      #   The S3 key prefix to use for each object. If not provided, files will be uploaded to the root of the bucket.
+      #
+      # @option options [Boolean] :recursive (false)
+      #   Whether to upload directories recursively:
+      #
+      #   * `false` (default) - only files in the top-level directory are uploaded, subdirectories are ignored.
+      #   * `true` - all files and subdirectories are uploaded recursively.
+      #
+      # @option options [Boolean] :follow_symlinks (false)
+      #   Whether to follow symbolic links when traversing the file tree:
+      #
+      #   * `false` (default) - symbolic links are ignored and not uploaded.
+      #   * `true` - symbolic links are followed and their target files/directories are uploaded. Symlink cycles
+      #     are detected and skipped.
+      #
+      # @option options [Boolean] :ignore_failure (false)
+      #   How to handle individual file upload failures:
+      #
+      #   * `false` (default) - Cancel all ongoing requests, terminate the directory upload, and raise an exception
+      #   * `true` - Ignore the failure and continue the transfer for other files
+      #
+      # @option options [Proc] :filter_callback (nil)
+      #   A Proc to filter which files to upload. Called with `(file_path, file_name)` for each file.
+      #   Return `true` to upload the file, `false` to skip it.
+      #
+      # @option options [Proc] :request_callback (nil)
+      #   A Proc to modify upload parameters for each file. Called with `(file_path, params)`.
+      #   Must return the modified parameters.
+      #
+      # @option options [Integer] :http_chunk_size (16384) Size in bytes for each chunk when streaming request bodies
+      #   over HTTP. Controls the buffer size used when sending data to S3. Larger values may improve throughput by
+      #   reducing the number of network writes, but use more memory. Custom values must be at least 16KB.
+      #   Only Ruby MRI is supported.
+      #
+      # @option options [Integer] :thread_count (10)
+      #   The number of threads to use for multipart uploads of individual large files.
+      #   Only used when no custom executor is provided to the {TransferManager}.
+      #
+      # @raise [DirectoryUploadError] Raised when:
+      #
+      #   * Upload failure with `ignore_failure: false` (default)
+      #   * Directory traversal failure (permission denied, broken symlink, etc.)
+      #
+      # @return [Hash] Returns a hash with upload statistics:
+      #
+      #   * `:completed_uploads` - Number of files successfully uploaded
+      #   * `:failed_uploads` - Number of files that failed to upload
+      #   * `:errors` - Array of error objects for failed uploads (only present when failures occur)
+      def upload_directory(source, bucket:, **options)
+        executor = @executor || DefaultExecutor.new(max_threads: options.delete(:thread_count))
+        uploader = DirectoryUploader.new(client: @client, executor: executor, logger: @logger)
+        result = uploader.upload(source, bucket, **options.merge(http_chunk_size: resolve_http_chunk_size(options)))
+        executor.shutdown unless @executor
+        result
+      end
+
       # Uploads a file from disk to S3.
       #
       #     # a small file are uploaded with PutObject API
@@ -202,8 +413,9 @@ module Aws
       #   Files larger han or equal to `:multipart_threshold` are uploaded using the S3 multipart upload APIs.
       #   Default threshold is `100MB`.
       #
-      # @option options [Integer] :thread_count (10) Customize threads used in the multipart upload.
-      #   Only used when no custom executor is provided (creates {DefaultExecutor} with the given thread count).
+      # @option options [Integer] :thread_count (10)
+      #   The number of threads to use for multipart uploads.
+      #   Only used when no custom executor is provided to the {TransferManager}.
       #
       # @option option [Integer] :http_chunk_size (16384) Size in bytes for each chunk when streaming request bodies
       #   over HTTP. Controls the buffer size used when sending data to S3. Larger values may improve throughput by
@@ -226,17 +438,7 @@ module Aws
       # @see Client#upload_part
       def upload_file(source, bucket:, key:, **options)
         upload_opts = options.merge(bucket: bucket, key: key)
-        http_chunk_size =
-          if defined?(JRUBY_VERSION)
-            nil
-          else
-            chunk = upload_opts.delete(:http_chunk_size)
-            if chunk && chunk < Aws::Plugins::ChecksumAlgorithm::DEFAULT_TRAILER_CHUNK_SIZE
-              raise ArgumentError, ':http_chunk_size must be at least 16384 bytes (16KB)'
-            end
-
-            chunk
-          end
+        http_chunk_size = resolve_http_chunk_size(upload_opts)
 
         executor = @executor || DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
         uploader = FileUploader.new(
@@ -316,6 +518,19 @@ module Aws
         executor.shutdown unless @executor
         true
       end
+
+      private
+
+      def resolve_http_chunk_size(opts)
+        return if defined?(JRUBY_VERSION)
+
+        chunk = opts.delete(:http_chunk_size)
+        if chunk && chunk < Aws::Plugins::ChecksumAlgorithm::DEFAULT_TRAILER_CHUNK_SIZE
+          raise ArgumentError, ':http_chunk_size must be at least 16384 bytes (16KB)'
+        end
+
+        chunk
+      end
     end
   end
 end

data/lib/aws-sdk-s3.rb

@@ -75,7 +75,7 @@ module Aws::S3
   autoload :ObjectVersion, 'aws-sdk-s3/object_version'
   autoload :EventStreams, 'aws-sdk-s3/event_streams'
 
-  GEM_VERSION = '1.213.0'
+  GEM_VERSION = '1.214.0'
 
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws-sdk-s3
 version: !ruby/object:Gem::Version
-  version: 1.213.0
+  version: 1.214.0
 platform: ruby
 authors:
 - Amazon Web Services
@@ -96,6 +96,10 @@ files:
 - lib/aws-sdk-s3/customizations/types/list_object_versions_output.rb
 - lib/aws-sdk-s3/customizations/types/permanent_redirect.rb
 - lib/aws-sdk-s3/default_executor.rb
+- lib/aws-sdk-s3/directory_download_error.rb
+- lib/aws-sdk-s3/directory_downloader.rb
+- lib/aws-sdk-s3/directory_upload_error.rb
+- lib/aws-sdk-s3/directory_uploader.rb
 - lib/aws-sdk-s3/encryption.rb
 - lib/aws-sdk-s3/encryption/client.rb
 - lib/aws-sdk-s3/encryption/decrypt_handler.rb