aws-sdk-s3 1.75.0

data/lib/aws-sdk-s3/multipart_file_uploader.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'set'
+
+module Aws
+  module S3
+    # @api private
+    class MultipartFileUploader
+
+      MIN_PART_SIZE = 5 * 1024 * 1024 # 5MB
+
+      FILE_TOO_SMALL = "unable to multipart upload files smaller than 5MB"
+
+      MAX_PARTS = 10_000
+
+      THREAD_COUNT = 10
+
+      # @api private
+      CREATE_OPTIONS = Set.new(
+        Client.api.operation(:create_multipart_upload).input.shape.member_names
+      )
+
+      # @api private
+      UPLOAD_PART_OPTIONS = Set.new(
+        Client.api.operation(:upload_part).input.shape.member_names
+      )
+
+      # @option options [Client] :client
+      # @option options [Integer] :thread_count (THREAD_COUNT)
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+        @thread_count = options[:thread_count] || THREAD_COUNT
+      end
+
+      # @return [Client]
+      attr_reader :client
+
+      # @param [String, Pathname, File, Tempfile] source The file to upload.
+      # @option options [required, String] :bucket The bucket to upload to.
+      # @option options [required, String] :key The key for the object.
+      # @option options [Proc] :progress_callback
+      #   A Proc that will be called when each chunk of the upload is sent.
+      #   It will be invoked with [bytes_read], [total_sizes]
+      # @return [void]
+      def upload(source, options = {})
+        if File.size(source) < MIN_PART_SIZE
+          raise ArgumentError, FILE_TOO_SMALL
+        else
+          upload_id = initiate_upload(options)
+          parts = upload_parts(upload_id, source, options)
+          complete_upload(upload_id, parts, options)
+        end
+      end
+
+      private
+
+      def initiate_upload(options)
+        @client.create_multipart_upload(create_opts(options)).upload_id
+      end
+
+      def complete_upload(upload_id, parts, options)
+        @client.complete_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id,
+          multipart_upload: { parts: parts }
+        )
+      end
+
+      def upload_parts(upload_id, source, options)
+        pending = PartList.new(compute_parts(upload_id, source, options))
+        completed = PartList.new
+        errors = upload_in_threads(pending, completed, options)
+        if errors.empty?
+          completed.to_a.sort_by { |part| part[:part_number] }
+        else
+          abort_upload(upload_id, options, errors)
+        end
+      end
+
+      def abort_upload(upload_id, options, errors)
+        @client.abort_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id
+        )
+        msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
+        raise MultipartUploadError.new(msg, errors)
+      rescue MultipartUploadError => error
+        raise error
+      rescue => error
+        msg = "failed to abort multipart upload: #{error.message}"
+        raise MultipartUploadError.new(msg, errors + [error])
+      end
+
+      def compute_parts(upload_id, source, options)
+        size = File.size(source)
+        default_part_size = compute_default_part_size(size)
+        offset = 0
+        part_number = 1
+        parts = []
+        while offset < size
+          parts << upload_part_opts(options).merge(
+            upload_id: upload_id,
+            part_number: part_number,
+            body: FilePart.new(
+              source: source,
+              offset: offset,
+              size: part_size(size, default_part_size, offset)
+            )
+          )
+          part_number += 1
+          offset += default_part_size
+        end
+        parts
+      end
+
+      def create_opts(options)
+        CREATE_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def upload_part_opts(options)
+        UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def upload_in_threads(pending, completed, options)
+        threads = []
+        if (callback = options[:progress_callback])
+          progress = MultipartProgress.new(pending, callback)
+        end
+        @thread_count.times do
+          thread = Thread.new do
+            begin
+              while part = pending.shift
+                if progress
+                  part[:on_chunk_sent] =
+                    proc do |_chunk, bytes, _total|
+                      progress.call(part[:part_number], bytes)
+                    end
+                end
+                resp = @client.upload_part(part)
+                part[:body].close
+                completed.push(etag: resp.etag, part_number: part[:part_number])
+              end
+              nil
+            rescue => error
+              # keep other threads from uploading other parts
+              pending.clear!
+              error
+            end
+          end
+          thread.abort_on_exception = true
+          threads << thread
+        end
+        threads.map(&:value).compact
+      end
+
+      def compute_default_part_size(source_size)
+        [(source_size.to_f / MAX_PARTS).ceil, MIN_PART_SIZE].max.to_i
+      end
+
+      def part_size(total_size, part_size, offset)
+        if offset + part_size > total_size
+          total_size - offset
+        else
+          part_size
+        end
+      end
+
+      # @api private
+      class PartList
+
+        def initialize(parts = [])
+          @parts = parts
+          @mutex = Mutex.new
+        end
+
+        def push(part)
+          @mutex.synchronize { @parts.push(part) }
+        end
+
+        def shift
+          @mutex.synchronize { @parts.shift }
+        end
+
+        def clear!
+          @mutex.synchronize { @parts.clear }
+        end
+
+        def size
+          @mutex.synchronize { @parts.size }
+        end
+
+        def part_sizes
+          @mutex.synchronize { @parts.map { |p| p[:body].size } }
+        end
+
+        def to_a
+          @mutex.synchronize { @parts.dup }
+        end
+
+      end
+
+      # @api private
+      class MultipartProgress
+        def initialize(parts, progress_callback)
+          @bytes_sent = Array.new(parts.size, 0)
+          @total_sizes = parts.part_sizes
+          @progress_callback = progress_callback
+        end
+
+        def call(part_number, bytes_read)
+          # part numbers start at 1
+          @bytes_sent[part_number - 1] = bytes_read
+          @progress_callback.call(@bytes_sent, @total_sizes)
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/multipart_stream_uploader.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require 'thread'
+require 'set'
+require 'tempfile'
+require 'stringio'
+
+module Aws
+  module S3
+    # @api private
+    class MultipartStreamUploader
+      # api private
+      PART_SIZE = 5 * 1024 * 1024 # 5MB
+
+      # api private
+      THREAD_COUNT = 10
+
+      # api private
+      TEMPFILE_PREIX = 'aws-sdk-s3-upload_stream'.freeze
+
+      # @api private
+      CREATE_OPTIONS =
+        Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names)
+
+      # @api private
+      UPLOAD_PART_OPTIONS =
+        Set.new(Client.api.operation(:upload_part).input.shape.member_names)
+
+      # @option options [Client] :client
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+        @tempfile = options[:tempfile]
+        @part_size = options[:part_size] || PART_SIZE
+        @thread_count = options[:thread_count] || THREAD_COUNT
+      end
+
+      # @return [Client]
+      attr_reader :client
+
+      # @option options [required,String] :bucket
+      # @option options [required,String] :key
+      # @return [void]
+      def upload(options = {}, &block)
+        upload_id = initiate_upload(options)
+        parts = upload_parts(upload_id, options, &block)
+        complete_upload(upload_id, parts, options)
+      end
+
+      private
+
+      def initiate_upload(options)
+        @client.create_multipart_upload(create_opts(options)).upload_id
+      end
+
+      def complete_upload(upload_id, parts, options)
+        @client.complete_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id,
+          multipart_upload: { parts: parts })
+      end
+
+      def upload_parts(upload_id, options, &block)
+        completed = Queue.new
+        errors = begin
+          IO.pipe do |read_pipe, write_pipe|
+            threads = upload_in_threads(read_pipe, completed, upload_part_opts(options).merge(upload_id: upload_id))
+            begin
+              block.call(write_pipe)
+            ensure
+              # Ensure the pipe is closed to avoid https://github.com/jruby/jruby/issues/6111
+              write_pipe.close
+            end
+            threads.map(&:value).compact
+          end
+        rescue => e
+          [e]
+        end
+
+        if errors.empty?
+          Array.new(completed.size) { completed.pop }.sort_by { |part| part[:part_number] }
+        else
+          abort_upload(upload_id, options, errors)
+        end
+      end
+
+      def abort_upload(upload_id, options, errors)
+        @client.abort_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id
+        )
+        msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
+        raise MultipartUploadError.new(msg, errors)
+      rescue MultipartUploadError => error
+        raise error
+      rescue => error
+        msg = "failed to abort multipart upload: #{error.message}"
+        raise MultipartUploadError.new(msg, errors + [error])
+      end
+
+      def create_opts(options)
+        CREATE_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def upload_part_opts(options)
+        UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def read_to_part_body(read_pipe)
+        return if read_pipe.closed?
+        temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new(String.new)
+        temp_io.binmode
+        bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size)
+        temp_io.rewind
+        if bytes_copied == 0
+          if Tempfile === temp_io
+            temp_io.close
+            temp_io.unlink
+          end
+          nil
+        else
+          temp_io
+        end
+      end
+
+      def upload_in_threads(read_pipe, completed, options)
+        mutex = Mutex.new
+        part_number = 0
+        @thread_count.times.map do
+          thread = Thread.new do
+            begin
+              loop do
+                body, thread_part_number = mutex.synchronize do
+                  [read_to_part_body(read_pipe), part_number += 1]
+                end
+                break unless (body || thread_part_number == 1)
+                begin
+                  part = options.merge(
+                    body: body,
+                    part_number: thread_part_number,
+                  )
+                  resp = @client.upload_part(part)
+                  completed << {etag: resp.etag, part_number: part[:part_number]}
+                ensure
+                  if Tempfile === body
+                    body.close
+                    body.unlink
+                  elsif StringIO === body
+                    body.string.clear
+                  end
+                end
+              end
+              nil
+            rescue => error
+              # keep other threads from uploading other parts
+              mutex.synchronize { read_pipe.close_read unless read_pipe.closed? }
+              error
+            end
+          end
+          thread.abort_on_exception = true
+          thread
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/multipart_upload.rb

@@ -0,0 +1,401 @@
+# frozen_string_literal: true
+
+# WARNING ABOUT GENERATED CODE
+#
+# This file is generated. See the contributing guide for more information:
+# https://github.com/aws/aws-sdk-ruby/blob/master/CONTRIBUTING.md
+#
+# WARNING ABOUT GENERATED CODE
+
+module Aws::S3
+
+  class MultipartUpload
+
+    extend Aws::Deprecations
+
+    # @overload def initialize(bucket_name, object_key, id, options = {})
+    #   @param [String] bucket_name
+    #   @param [String] object_key
+    #   @param [String] id
+    #   @option options [Client] :client
+    # @overload def initialize(options = {})
+    #   @option options [required, String] :bucket_name
+    #   @option options [required, String] :object_key
+    #   @option options [required, String] :id
+    #   @option options [Client] :client
+    def initialize(*args)
+      options = Hash === args.last ? args.pop.dup : {}
+      @bucket_name = extract_bucket_name(args, options)
+      @object_key = extract_object_key(args, options)
+      @id = extract_id(args, options)
+      @data = options.delete(:data)
+      @client = options.delete(:client) || Client.new(options)
+      @waiter_block_warned = false
+    end
+
+    # @!group Read-Only Attributes
+
+    # @return [String]
+    def bucket_name
+      @bucket_name
+    end
+
+    # @return [String]
+    def object_key
+      @object_key
+    end
+
+    # @return [String]
+    def id
+      @id
+    end
+
+    # Upload ID that identifies the multipart upload.
+    # @return [String]
+    def upload_id
+      data[:upload_id]
+    end
+
+    # Key of the object for which the multipart upload was initiated.
+    # @return [String]
+    def key
+      data[:key]
+    end
+
+    # Date and time at which the multipart upload was initiated.
+    # @return [Time]
+    def initiated
+      data[:initiated]
+    end
+
+    # The class of storage used to store the object.
+    # @return [String]
+    def storage_class
+      data[:storage_class]
+    end
+
+    # Specifies the owner of the object that is part of the multipart
+    # upload.
+    # @return [Types::Owner]
+    def owner
+      data[:owner]
+    end
+
+    # Identifies who initiated the multipart upload.
+    # @return [Types::Initiator]
+    def initiator
+      data[:initiator]
+    end
+
+    # @!endgroup
+
+    # @return [Client]
+    def client
+      @client
+    end
+
+    # @raise [NotImplementedError]
+    # @api private
+    def load
+      msg = "#load is not implemented, data only available via enumeration"
+      raise NotImplementedError, msg
+    end
+    alias :reload :load
+
+    # @raise [NotImplementedError] Raises when {#data_loaded?} is `false`.
+    # @return [Types::MultipartUpload]
+    #   Returns the data for this {MultipartUpload}.
+    def data
+      load unless @data
+      @data
+    end
+
+    # @return [Boolean]
+    #   Returns `true` if this resource is loaded.  Accessing attributes or
+    #   {#data} on an unloaded resource will trigger a call to {#load}.
+    def data_loaded?
+      !!@data
+    end
+
+    # @deprecated Use [Aws::S3::Client] #wait_until instead
+    #
+    # Waiter polls an API operation until a resource enters a desired
+    # state.
+    #
+    # @note The waiting operation is performed on a copy. The original resource
+    #   remains unchanged.
+    #
+    # ## Basic Usage
+    #
+    # Waiter will polls until it is successful, it fails by
+    # entering a terminal state, or until a maximum number of attempts
+    # are made.
+    #
+    #     # polls in a loop until condition is true
+    #     resource.wait_until(options) {|resource| condition}
+    #
+    # ## Example
+    #
+    #     instance.wait_until(max_attempts:10, delay:5) do |instance|
+    #       instance.state.name == 'running'
+    #     end
+    #
+    # ## Configuration
+    #
+    # You can configure the maximum number of polling attempts, and the
+    # delay (in seconds) between each polling attempt. The waiting condition is
+    # set by passing a block to {#wait_until}:
+    #
+    #     # poll for ~25 seconds
+    #     resource.wait_until(max_attempts:5,delay:5) {|resource|...}
+    #
+    # ## Callbacks
+    #
+    # You can be notified before each polling attempt and before each
+    # delay. If you throw `:success` or `:failure` from these callbacks,
+    # it will terminate the waiter.
+    #
+    #     started_at = Time.now
+    #     # poll for 1 hour, instead of a number of attempts
+    #     proc = Proc.new do |attempts, response|
+    #       throw :failure if Time.now - started_at > 3600
+    #     end
+    #
+    #       # disable max attempts
+    #     instance.wait_until(before_wait:proc, max_attempts:nil) {...}
+    #
+    # ## Handling Errors
+    #
+    # When a waiter is successful, it returns the Resource. When a waiter
+    # fails, it raises an error.
+    #
+    #     begin
+    #       resource.wait_until(...)
+    #     rescue Aws::Waiters::Errors::WaiterFailed
+    #       # resource did not enter the desired state in time
+    #     end
+    #
+    # @yieldparam [Resource] resource to be used in the waiting condition.
+    #
+    # @raise [Aws::Waiters::Errors::FailureStateError] Raised when the waiter
+    #   terminates because the waiter has entered a state that it will not
+    #   transition out of, preventing success.
+    #
+    #   yet successful.
+    #
+    # @raise [Aws::Waiters::Errors::UnexpectedError] Raised when an error is
+    #   encountered while polling for a resource that is not expected.
+    #
+    # @raise [NotImplementedError] Raised when the resource does not
+    #
+    # @option options [Integer] :max_attempts (10) Maximum number of
+    # attempts
+    # @option options [Integer] :delay (10) Delay between each
+    # attempt in seconds
+    # @option options [Proc] :before_attempt (nil) Callback
+    # invoked before each attempt
+    # @option options [Proc] :before_wait (nil) Callback
+    # invoked before each wait
+    # @return [Resource] if the waiter was successful
+    def wait_until(options = {}, &block)
+      self_copy = self.dup
+      attempts = 0
+      options[:max_attempts] = 10 unless options.key?(:max_attempts)
+      options[:delay] ||= 10
+      options[:poller] = Proc.new do
+        attempts += 1
+        if block.call(self_copy)
+          [:success, self_copy]
+        else
+          self_copy.reload unless attempts == options[:max_attempts]
+          :retry
+        end
+      end
+      Aws::Waiters::Waiter.new(options).wait({})
+    end
+
+    # @!group Actions
+
+    # @example Request syntax with placeholder values
+    #
+    #   multipart_upload.abort({
+    #     request_payer: "requester", # accepts requester
+    #   })
+    # @param [Hash] options ({})
+    # @option options [String] :request_payer
+    #   Confirms that the requester knows that they will be charged for the
+    #   request. Bucket owners need not specify this parameter in their
+    #   requests. For information about downloading objects from requester
+    #   pays buckets, see [Downloading Objects in Requestor Pays Buckets][1]
+    #   in the *Amazon S3 Developer Guide*.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+    # @return [Types::AbortMultipartUploadOutput]
+    def abort(options = {})
+      options = options.merge(
+        bucket: @bucket_name,
+        key: @object_key,
+        upload_id: @id
+      )
+      resp = @client.abort_multipart_upload(options)
+      resp.data
+    end
+
+    # @example Request syntax with placeholder values
+    #
+    #   object = multipart_upload.complete({
+    #     multipart_upload: {
+    #       parts: [
+    #         {
+    #           etag: "ETag",
+    #           part_number: 1,
+    #         },
+    #       ],
+    #     },
+    #     request_payer: "requester", # accepts requester
+    #   })
+    # @param [Hash] options ({})
+    # @option options [Types::CompletedMultipartUpload] :multipart_upload
+    #   The container for the multipart upload request information.
+    # @option options [String] :request_payer
+    #   Confirms that the requester knows that they will be charged for the
+    #   request. Bucket owners need not specify this parameter in their
+    #   requests. For information about downloading objects from requester
+    #   pays buckets, see [Downloading Objects in Requestor Pays Buckets][1]
+    #   in the *Amazon S3 Developer Guide*.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+    # @return [Object]
+    def complete(options = {})
+      options = options.merge(
+        bucket: @bucket_name,
+        key: @object_key,
+        upload_id: @id
+      )
+      @client.complete_multipart_upload(options)
+      Object.new(
+        bucket_name: @bucket_name,
+        key: @object_key,
+        client: @client
+      )
+    end
+
+    # @!group Associations
+
+    # @return [Object]
+    def object
+      Object.new(
+        bucket_name: @bucket_name,
+        key: @object_key,
+        client: @client
+      )
+    end
+
+    # @param [String] part_number
+    # @return [MultipartUploadPart]
+    def part(part_number)
+      MultipartUploadPart.new(
+        bucket_name: @bucket_name,
+        object_key: @object_key,
+        multipart_upload_id: @id,
+        part_number: part_number,
+        client: @client
+      )
+    end
+
+    # @example Request syntax with placeholder values
+    #
+    #   parts = multipart_upload.parts({
+    #     request_payer: "requester", # accepts requester
+    #   })
+    # @param [Hash] options ({})
+    # @option options [String] :request_payer
+    #   Confirms that the requester knows that they will be charged for the
+    #   request. Bucket owners need not specify this parameter in their
+    #   requests. For information about downloading objects from requester
+    #   pays buckets, see [Downloading Objects in Requestor Pays Buckets][1]
+    #   in the *Amazon S3 Developer Guide*.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+    # @return [MultipartUploadPart::Collection]
+    def parts(options = {})
+      batches = Enumerator.new do |y|
+        options = options.merge(
+          bucket: @bucket_name,
+          key: @object_key,
+          upload_id: @id
+        )
+        resp = @client.list_parts(options)
+        resp.each_page do |page|
+          batch = []
+          page.data.parts.each do |p|
+            batch << MultipartUploadPart.new(
+              bucket_name: options[:bucket],
+              object_key: options[:key],
+              multipart_upload_id: options[:upload_id],
+              part_number: p.part_number,
+              data: p,
+              client: @client
+            )
+          end
+          y.yield(batch)
+        end
+      end
+      MultipartUploadPart::Collection.new(batches)
+    end
+
+    # @deprecated
+    # @api private
+    def identifiers
+      {
+        bucket_name: @bucket_name,
+        object_key: @object_key,
+        id: @id
+      }
+    end
+    deprecated(:identifiers)
+
+    private
+
+    def extract_bucket_name(args, options)
+      value = args[0] || options.delete(:bucket_name)
+      case value
+      when String then value
+      when nil then raise ArgumentError, "missing required option :bucket_name"
+      else
+        msg = "expected :bucket_name to be a String, got #{value.class}"
+        raise ArgumentError, msg
+      end
+    end
+
+    def extract_object_key(args, options)
+      value = args[1] || options.delete(:object_key)
+      case value
+      when String then value
+      when nil then raise ArgumentError, "missing required option :object_key"
+      else
+        msg = "expected :object_key to be a String, got #{value.class}"
+        raise ArgumentError, msg
+      end
+    end
+
+    def extract_id(args, options)
+      value = args[2] || options.delete(:id)
+      case value
+      when String then value
+      when nil then raise ArgumentError, "missing required option :id"
+      else
+        msg = "expected :id to be a String, got #{value.class}"
+        raise ArgumentError, msg
+      end
+    end
+
+    class Collection < Aws::Resources::Collection; end
+  end
+end