aws-sdk-s3 1.213.0 → 1.216.0

data/lib/aws-sdk-s3/directory_downloader.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    # @api private
+    # This is a one-shot class that downloads objects from a bucket to a local directory.
+    # This works as follows:
+    # * ObjectProducer runs in a background thread, calling `list_objects_v2` and
+    #   pushing entries into a SizedQueue (max: 100).
+    # * An internal executor pulls from that queue and posts work. Each task uses
+    #   FileDownloader to download objects then signals completion via `completion_queue`.
+    #
+    # We track how many tasks we posted, then pop that many times from `completion_queue`
+    # to wait for everything to finish.
+    #
+    # Errors are collected in a mutex-protected array. On failure (unless ignore_failure is set),
+    # we call abort which closes the queue - the producer catches ClosedQueueError and exits cleanly.
+    class DirectoryDownloader
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+        @executor = options[:executor] || DefaultExecutor.new
+        @logger = options[:logger]
+        @producer = nil
+        @mutex = Mutex.new
+      end
+
+      attr_reader :client, :executor
+
+      def abort
+        @producer&.close
+      end
+
+      def download(destination, bucket:, **options)
+        if File.exist?(destination)
+          raise ArgumentError, 'invalid destination, expected a directory' unless File.directory?(destination)
+        else
+          FileUtils.mkdir_p(destination)
+        end
+
+        download_opts = build_download_opts(destination, options)
+        @producer = ObjectProducer.new(build_producer_opts(destination, bucket, options))
+        downloader = FileDownloader.new(client: @client, executor: @executor)
+        downloads, errors = process_download_queue(downloader, download_opts)
+        build_result(downloads, errors)
+      end
+
+      private
+
+      def build_download_opts(destination, opts)
+        {
+          destination: destination,
+          ignore_failure: opts[:ignore_failure] || false
+        }
+      end
+
+      def build_producer_opts(destination, bucket, opts)
+        {
+          client: @client,
+          directory_downloader: self,
+          destination: destination,
+          bucket: bucket,
+          s3_prefix: opts[:s3_prefix],
+          filter_callback: opts[:filter_callback],
+          request_callback: opts[:request_callback]
+        }
+      end
+
+      def build_result(download_count, errors)
+        if @producer&.closed?
+          msg = "directory download failed: #{errors.map(&:message).join('; ')}"
+          raise DirectoryDownloadError.new(msg, errors)
+        else
+          {
+            completed_downloads: [download_count - errors.count, 0].max,
+            failed_downloads: errors.count,
+            errors: errors.any? ? errors : nil
+          }.compact
+        end
+      end
+
+      def download_object(entry, downloader, errors, opts)
+        raise entry.error if entry.error
+
+        FileUtils.mkdir_p(File.dirname(entry.path)) unless Dir.exist?(File.dirname(entry.path))
+        downloader.download(entry.path, entry.params)
+        @logger&.debug("Downloaded #{entry.params[:key]} from #{entry.params[:bucket]} to #{entry.path}")
+      rescue StandardError => e
+        @logger&.warn("Failed to download #{entry.params[:key]} from #{entry.params[:bucket]}: #{e.message}")
+        @mutex.synchronize { errors << e }
+        abort unless opts[:ignore_failure]
+      end
+
+      def process_download_queue(downloader, opts)
+        queue_executor = DefaultExecutor.new(max_threads: 2)
+        completion_queue = Queue.new
+        posted_count = 0
+        errors = []
+        begin
+          @producer.each do |object|
+            queue_executor.post(object) do |o|
+              download_object(o, downloader, errors, opts)
+            ensure
+              completion_queue << :done
+            end
+            posted_count += 1
+          end
+        rescue ClosedQueueError
+          # abort already requested
+        rescue StandardError => e
+          @mutex.synchronize { errors << e }
+          abort
+        end
+        posted_count.times { completion_queue.pop }
+        [posted_count, errors]
+      ensure
+        queue_executor&.shutdown
+      end
+
+      # @api private
+      class ObjectProducer
+        include Enumerable
+
+        DEFAULT_QUEUE_SIZE = 100
+        DONE_MARKER = :done
+
+        def initialize(opts = {})
+          @directory_downloader = opts[:directory_downloader]
+          @destination_dir = opts[:destination]
+          @bucket = opts[:bucket]
+          @client = opts[:client]
+          @s3_prefix = opts[:s3_prefix]
+          @filter_callback = opts[:filter_callback]
+          @request_callback = opts[:request_callback]
+          @object_queue = SizedQueue.new(DEFAULT_QUEUE_SIZE)
+        end
+
+        def closed?
+          @object_queue.closed?
+        end
+
+        def close
+          @object_queue.close
+          @object_queue.clear
+        end
+
+        def each
+          producer_thread = Thread.new do
+            stream_objects
+            @object_queue << DONE_MARKER
+          rescue ClosedQueueError
+            # abort requested
+          rescue StandardError => e
+            close
+            raise e
+          end
+
+          while (object = @object_queue.shift) && object != DONE_MARKER
+            yield object
+          end
+        ensure
+          producer_thread.value
+        end
+
+        private
+
+        def apply_request_callback(key, params)
+          callback_params = @request_callback.call(key, params.dup)
+          return params unless callback_params.is_a?(Hash) && callback_params.any?
+
+          params.merge(callback_params)
+        end
+
+        def build_object_entry(key)
+          params = { bucket: @bucket, key: key }
+          params = apply_request_callback(key, params) if @request_callback
+          error = validate_key(key)
+          return DownloadEntry.new(path: '', params: params, error: error) if error
+
+          full_path = normalize_path(File.join(@destination_dir, key))
+          DownloadEntry.new(path: full_path, params: params, error: error)
+        end
+
+        def include_object?(obj)
+          return true unless @filter_callback
+
+          @filter_callback.call(obj)
+        end
+
+        def directory_marker?(obj)
+          obj.key.end_with?('/') && obj.size.zero?
+        end
+
+        def normalize_path(path)
+          return path if File::SEPARATOR == '/'
+
+          path.tr('/', File::SEPARATOR)
+        end
+
+        def stream_objects(continuation_token: nil)
+          resp = @client.list_objects_v2(bucket: @bucket, prefix: @s3_prefix, continuation_token: continuation_token)
+          resp.contents&.each do |o|
+            next if directory_marker?(o)
+            next unless include_object?(o)
+
+            @object_queue << build_object_entry(o.key)
+          end
+          stream_objects(continuation_token: resp.next_continuation_token) if resp.next_continuation_token
+        end
+
+        def validate_key(key)
+          segments = key.split('/')
+          return unless segments.any? { |s| %w[. ..].include?(s) }
+
+          DirectoryDownloadError.new("invalid key '#{key}': contains '.' or '..' path segments")
+        end
+
+        # @api private
+        class DownloadEntry
+          def initialize(opts = {})
+            @path = opts[:path]
+            @params = opts[:params]
+            @error = opts[:error]
+          end
+
+          attr_reader :path, :params, :error
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/directory_upload_error.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    # Raised when DirectoryUploader fails to upload files to S3 bucket
+    class DirectoryUploadError < StandardError
+      def initialize(message, errors = [])
+        @errors = errors
+        super(message)
+      end
+
+      # @return [Array<StandardError>] The list of errors encountered when uploading files
+      attr_reader :errors
+    end
+  end
+end

data/lib/aws-sdk-s3/directory_uploader.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Aws
+  module S3
+    # @api private
+    # This is a one-shot class that uploads files from a local directory to a bucket.
+    # This works as follows:
+    # * FileProducer runs in a background thread, scanning the directory and
+    #   pushing entries into a SizedQueue (max: 100).
+    # * An internal executor pulls from that queue and posts work. Each task uses
+    #   FileUploader to upload files then signals completion via `completion_queue`.
+    #
+    # We track how many tasks we posted, then pop that many times from `completion_queue`
+    # to wait for everything to finish.
+    #
+    # Errors are collected in a mutex-protected array. On failure (unless ignore_failure is set),
+    # we call abort which closes the queue - the producer catches ClosedQueueError and exits cleanly.
+    class DirectoryUploader
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+        @executor = options[:executor] || DefaultExecutor.new
+        @logger = options[:logger]
+        @producer = nil
+        @mutex = Mutex.new
+      end
+
+      attr_reader :client, :executor
+
+      def abort
+        @producer&.close
+      end
+
+      def upload(source_directory, bucket, **opts)
+        raise ArgumentError, 'Invalid directory' unless Dir.exist?(source_directory)
+
+        uploader = FileUploader.new(
+          multipart_threshold: opts.delete(:multipart_threshold),
+          http_chunk_size: opts.delete(:http_chunk_size),
+          client: @client,
+          executor: @executor
+        )
+        upload_opts = build_upload_opts(opts)
+        @producer = FileProducer.new(build_producer_opts(source_directory, bucket, opts))
+        uploads, errors = process_upload_queue(uploader, upload_opts)
+        build_result(uploads, errors)
+      end
+
+      private
+
+      def build_upload_opts(opts)
+        { ignore_failure: opts[:ignore_failure] || false }
+      end
+
+      def build_producer_opts(source_directory, bucket, opts)
+        {
+          directory_uploader: self,
+          source_dir: source_directory,
+          bucket: bucket,
+          s3_prefix: opts[:s3_prefix],
+          recursive: opts[:recursive] || false,
+          follow_symlinks: opts[:follow_symlinks] || false,
+          filter_callback: opts[:filter_callback],
+          request_callback: opts[:request_callback]
+        }
+      end
+
+      def build_result(upload_count, errors)
+        if @producer&.closed?
+          msg = "directory upload failed: #{errors.map(&:message).join('; ')}"
+          raise DirectoryUploadError.new(msg, errors)
+        else
+          {
+            completed_uploads: [upload_count - errors.count, 0].max,
+            failed_uploads: errors.count,
+            errors: errors.any? ? errors : nil
+          }.compact
+        end
+      end
+
+      def process_upload_queue(uploader, opts)
+        queue_executor = DefaultExecutor.new(max_threads: 2)
+        completion_queue = Queue.new
+        posted_count = 0
+        errors = []
+        begin
+          @producer.each do |file|
+            queue_executor.post(file) do |f|
+              upload_file(f, uploader, errors, opts)
+            ensure
+              completion_queue << :done
+            end
+            posted_count += 1
+          end
+        rescue ClosedQueueError
+          # abort already requested
+        rescue StandardError => e
+          @mutex.synchronize { errors << e }
+          abort
+        end
+        posted_count.times { completion_queue.pop }
+        [posted_count, errors]
+      ensure
+        queue_executor&.shutdown
+      end
+
+      def upload_file(entry, uploader, errors, opts)
+        uploader.upload(entry.path, entry.params)
+        @logger&.debug("Uploaded #{entry.path} to #{entry.params[:bucket]} as #{entry.params[:key]}")
+      rescue StandardError => e
+        @logger&.warn("Failed to upload #{entry.path} to #{entry.params[:bucket]}: #{e.message}")
+        @mutex.synchronize { errors << e }
+        abort unless opts[:ignore_failure]
+      end
+
+      # @api private
+      class FileProducer
+        include Enumerable
+
+        DEFAULT_QUEUE_SIZE = 100
+        DONE_MARKER = :done
+
+        def initialize(opts = {})
+          @directory_uploader = opts[:directory_uploader]
+          @source_dir = opts[:source_dir]
+          @bucket = opts[:bucket]
+          @s3_prefix = opts[:s3_prefix]
+          @recursive = opts[:recursive]
+          @follow_symlinks = opts[:follow_symlinks]
+          @filter_callback = opts[:filter_callback]
+          @request_callback = opts[:request_callback]
+          @file_queue = SizedQueue.new(DEFAULT_QUEUE_SIZE)
+        end
+
+        def closed?
+          @file_queue.closed?
+        end
+
+        def close
+          @file_queue.close
+          @file_queue.clear
+        end
+
+        def each
+          producer_thread = Thread.new do
+            if @recursive
+              find_recursively
+            else
+              find_directly
+            end
+            @file_queue << DONE_MARKER
+          rescue ClosedQueueError
+            # abort requested
+          rescue StandardError => e
+            # encountered a traversal error, we must abort immediately
+            close
+            raise DirectoryUploadError, "Directory traversal failed for '#{@source_dir}': #{e.message}"
+          end
+
+          while (file = @file_queue.shift) && file != DONE_MARKER
+            yield file
+          end
+        ensure
+          producer_thread.value
+        end
+
+        private
+
+        def apply_request_callback(file_path, params)
+          callback_params = @request_callback.call(file_path, params.dup)
+          return params unless callback_params.is_a?(Hash) && callback_params.any?
+
+          params.merge(callback_params)
+        end
+
+        def build_upload_entry(file_path, key)
+          params = { bucket: @bucket, key: @s3_prefix ? File.join(@s3_prefix, key) : key }
+          params = apply_request_callback(file_path, params) if @request_callback
+          UploadEntry.new(path: file_path, params: params)
+        end
+
+        def find_directly
+          Dir.each_child(@source_dir) do |entry|
+            entry_path = File.join(@source_dir, entry)
+            stat = nil
+
+            if @follow_symlinks
+              stat = File.stat(entry_path)
+              next if stat.directory?
+            else
+              stat = File.lstat(entry_path)
+              next if stat.symlink? || stat.directory?
+            end
+
+            next unless stat.file?
+            next unless include_file?(entry_path, entry)
+
+            @file_queue << build_upload_entry(entry_path, entry)
+          end
+        end
+
+        def find_recursively
+          if @follow_symlinks
+            ancestors = Set.new
+            ancestors << File.stat(@source_dir).ino
+            scan_directory(@source_dir, ancestors: ancestors)
+          else
+            scan_directory(@source_dir)
+          end
+        end
+
+        def include_file?(file_path, file_name)
+          return true unless @filter_callback
+
+          @filter_callback.call(file_path, file_name)
+        end
+
+        def scan_directory(dir_path, key_prefix: '', ancestors: nil)
+          Dir.each_child(dir_path) do |entry|
+            full_path = File.join(dir_path, entry)
+            next unless include_file?(full_path, entry)
+
+            stat = get_file_stat(full_path)
+            next unless stat
+
+            if stat.directory?
+              handle_directory(full_path, entry, key_prefix, ancestors)
+            elsif stat.file? # skip non-file types
+              key = key_prefix.empty? ? entry : File.join(key_prefix, entry)
+              @file_queue << build_upload_entry(full_path, key)
+            end
+          end
+        end
+
+        def get_file_stat(full_path)
+          return File.stat(full_path) if @follow_symlinks
+
+          lstat = File.lstat(full_path)
+          return if lstat.symlink?
+
+          lstat
+        end
+
+        def handle_directory(dir_path, dir_name, key_prefix, ancestors)
+          ino = nil
+          if @follow_symlinks && ancestors
+            ino = File.stat(dir_path).ino
+            return if ancestors.include?(ino) # cycle detected - skip
+
+            ancestors.add(ino)
+          end
+          new_prefix = key_prefix.empty? ? dir_name : File.join(key_prefix, dir_name)
+          scan_directory(dir_path, key_prefix: new_prefix, ancestors: ancestors)
+          ancestors.delete(ino) if @follow_symlinks && ancestors
+        end
+
+        # @api private
+        class UploadEntry
+          def initialize(opts = {})
+            @path = opts[:path]
+            @params = opts[:params]
+          end
+
+          attr_reader :path, :params
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/resource.rb

@@ -64,6 +64,7 @@ module Aws::S3
     #     grant_write_acp: "GrantWriteACP",
     #     object_lock_enabled_for_bucket: false,
     #     object_ownership: "BucketOwnerPreferred", # accepts BucketOwnerPreferred, ObjectWriter, BucketOwnerEnforced
+    #     bucket_namespace: "account-regional", # accepts account-regional, global
     #   })
     # @param [Hash] options ({})
     # @option options [String] :acl
@@ -171,6 +172,33 @@ module Aws::S3
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/about-object-ownership.html
+    # @option options [String] :bucket_namespace
+    #   Specifies the namespace where you want to create your general purpose
+    #   bucket. When you create a general purpose bucket, you can choose to
+    #   create a bucket in the shared global namespace or you can choose to
+    #   create a bucket in your account regional namespace. Your account
+    #   regional namespace is a subdivision of the global namespace that only
+    #   your account can create buckets in. For more information on bucket
+    #   namespaces, see [Namespaces for general purpose buckets][1].
+    #
+    #   General purpose buckets in your account regional namespace must follow
+    #   a specific naming convention. These buckets consist of a bucket name
+    #   prefix that you create, and a suffix that contains your 12-digit
+    #   Amazon Web Services Account ID, the Amazon Web Services Region code,
+    #   and ends with `-an`. Bucket names must follow the format
+    #   `bucket-name-prefix-accountId-region-an` (for example,
+    #   `amzn-s3-demo-bucket-111122223333-us-west-2-an`). For information
+    #   about bucket naming restrictions, see [Account regional namespace
+    #   naming rules][2] in the *Amazon S3 User Guide*.
+    #
+    #   <note markdown="1"> This functionality is not supported for directory buckets.
+    #
+    #    </note>
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/gpbucketnamespaces.html
+    #   [2]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html#account-regional-naming-rules
     # @return [Bucket]
     def create_bucket(options = {})
       Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do