aws-sdk-s3 1.0.0.rc1

data/lib/aws-sdk-s3/multipart_file_uploader.rb

@@ -0,0 +1,187 @@
+require 'pathname'
+require 'thread'
+require 'set'
+
+module Aws
+  module S3
+    # @api private
+    class MultipartFileUploader
+
+      MIN_PART_SIZE = 5 * 1024 * 1024 # 5MB
+
+      FILE_TOO_SMALL = "unable to multipart upload files smaller than 5MB"
+
+      MAX_PARTS = 10_000
+
+      THREAD_COUNT = 10
+
+      # @api private
+      CREATE_OPTIONS =
+        Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names)
+
+      # @api private
+      UPLOAD_PART_OPTIONS =
+        Set.new(Client.api.operation(:upload_part).input.shape.member_names)
+
+      # @option options [Client] :client
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+        @thread_count = options[:thread_count] || THREAD_COUNT
+      end
+
+      # @return [Client]
+      attr_reader :client
+
+      # @param [String,Pathname,File,Tempfile] source
+      # @option options [required,String] :bucket
+      # @option options [required,String] :key
+      # @return [void]
+      def upload(source, options = {})
+        if File.size(source) < MIN_PART_SIZE
+          raise ArgumentError, FILE_TOO_SMALL
+        else
+          upload_id = initiate_upload(options)
+          parts = upload_parts(upload_id, source, options)
+          complete_upload(upload_id, parts, options)
+        end
+      end
+
+      private
+
+      def initiate_upload(options)
+        @client.create_multipart_upload(create_opts(options)).upload_id
+      end
+
+      def complete_upload(upload_id, parts, options)
+        @client.complete_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id,
+          multipart_upload: { parts: parts })
+      end
+
+      def upload_parts(upload_id, source, options)
+        pending = PartList.new(compute_parts(upload_id, source, options))
+        completed = PartList.new
+        errors = upload_in_threads(pending, completed)
+        if errors.empty?
+          completed.to_a.sort_by { |part| part[:part_number] }
+        else
+          abort_upload(upload_id, options, errors)
+        end
+      end
+
+      def abort_upload(upload_id, options, errors)
+        @client.abort_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id
+        )
+        msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
+        raise MultipartUploadError.new(msg, errors)
+      rescue MultipartUploadError => error
+        raise error
+      rescue => error
+        msg = "failed to abort multipart upload: #{error.message}"
+        raise MultipartUploadError.new(msg, errors + [error])
+      end
+
+      def compute_parts(upload_id, source, options)
+        size = File.size(source)
+        default_part_size = compute_default_part_size(size)
+        offset = 0
+        part_number = 1
+        parts = []
+        while offset < size
+          parts << upload_part_opts(options).merge({
+            upload_id: upload_id,
+            part_number: part_number,
+            body: FilePart.new(
+              source: source,
+              offset: offset,
+              size: part_size(size, default_part_size, offset)
+            )
+          })
+          part_number += 1
+          offset += default_part_size
+        end
+        parts
+      end
+
+      def create_opts(options)
+        CREATE_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def upload_part_opts(options)
+        UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def upload_in_threads(pending, completed)
+        threads = []
+        @thread_count.times do
+          thread = Thread.new do
+            begin
+              while part = pending.shift
+                resp = @client.upload_part(part)
+                part[:body].close
+                completed.push(etag: resp.etag, part_number: part[:part_number])
+              end
+              nil
+            rescue => error
+              # keep other threads from uploading other parts
+              pending.clear!
+              error
+            end
+          end
+          thread.abort_on_exception = true
+          threads << thread
+        end
+        threads.map(&:value).compact
+      end
+
+      def compute_default_part_size(source_size)
+        [(source_size.to_f / MAX_PARTS).ceil, MIN_PART_SIZE].max.to_i
+      end
+
+      def part_size(total_size, part_size, offset)
+        if offset + part_size > total_size
+          total_size - offset
+        else
+          part_size
+        end
+      end
+
+      # @api private
+      class PartList
+
+        def initialize(parts = [])
+          @parts = parts
+          @mutex = Mutex.new
+        end
+
+        def push(part)
+          @mutex.synchronize { @parts.push(part) }
+        end
+
+        def shift
+          @mutex.synchronize { @parts.shift }
+        end
+
+        def clear!
+          @mutex.synchronize { @parts.clear }
+        end
+
+        def to_a
+          @mutex.synchronize { @parts.dup }
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/multipart_upload.rb

@@ -0,0 +1,287 @@
+# WARNING ABOUT GENERATED CODE
+#
+# This file is generated. See the contributing for info on making contributions:
+# https://github.com/aws/aws-sdk-ruby/blob/master/CONTRIBUTING.md
+#
+# WARNING ABOUT GENERATED CODE
+
+module Aws
+  module S3
+    class MultipartUpload
+
+      extend Aws::Deprecations
+
+      # @overload def initialize(bucket_name, object_key, id, options = {})
+      #   @param [String] bucket_name
+      #   @param [String] object_key
+      #   @param [String] id
+      #   @option options [Client] :client
+      # @overload def initialize(options = {})
+      #   @option options [required, String] :bucket_name
+      #   @option options [required, String] :object_key
+      #   @option options [required, String] :id
+      #   @option options [Client] :client
+      def initialize(*args)
+        options = Hash === args.last ? args.pop.dup : {}
+        @bucket_name = extract_bucket_name(args, options)
+        @object_key = extract_object_key(args, options)
+        @id = extract_id(args, options)
+        @data = options.delete(:data)
+        @client = options.delete(:client) || Client.new(options)
+      end
+
+      # @!group Read-Only Attributes
+
+      # @return [String]
+      def bucket_name
+        @bucket_name
+      end
+
+      # @return [String]
+      def object_key
+        @object_key
+      end
+
+      # @return [String]
+      def id
+        @id
+      end
+
+      # Upload ID that identifies the multipart upload.
+      # @return [String]
+      def upload_id
+        data.upload_id
+      end
+
+      # Key of the object for which the multipart upload was initiated.
+      # @return [String]
+      def key
+        data.key
+      end
+
+      # Date and time at which the multipart upload was initiated.
+      # @return [Time]
+      def initiated
+        data.initiated
+      end
+
+      # The class of storage used to store the object.
+      # @return [String]
+      def storage_class
+        data.storage_class
+      end
+
+      # @return [Types::Owner]
+      def owner
+        data.owner
+      end
+
+      # Identifies who initiated the multipart upload.
+      # @return [Types::Initiator]
+      def initiator
+        data.initiator
+      end
+
+      # @!endgroup
+
+      # @return [Client]
+      def client
+        @client
+      end
+
+      # @raise [Errors::ResourceNotLoadable]
+      # @api private
+      def load
+        msg = "#load is not implemented, data only available via enumeration"
+        raise Errors::ResourceNotLoadable, msg
+      end
+      alias :reload :load
+
+      # @raise [Errors::ResourceNotLoadableError] Raises when {#data_loaded?} is `false`.
+      # @return [Types::MultipartUpload]
+      #   Returns the data for this {MultipartUpload}.
+      def data
+        load unless @data
+        @data
+      end
+
+      # @return [Boolean]
+      #   Returns `true` if this resource is loaded.  Accessing attributes or
+      #   {#data} on an unloaded resource will trigger a call to {#load}.
+      def data_loaded?
+        !!@data
+      end
+
+      # @!group Actions
+
+      # @example Request syntax with placeholder values
+      #
+      #   multipart_upload.abort({
+      #     request_payer: "requester", # accepts requester
+      #   })
+      # @param [Hash] options ({})
+      # @option options [String] :request_payer
+      #   Confirms that the requester knows that she or he will be charged for
+      #   the request. Bucket owners need not specify this parameter in their
+      #   requests. Documentation on downloading objects from requester pays
+      #   buckets can be found at
+      #   http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+      # @return [Types::AbortMultipartUploadOutput]
+      def abort(options = {})
+        options = options.merge(
+          bucket: @bucket_name,
+          key: @object_key,
+          upload_id: @id
+        )
+        resp = @client.abort_multipart_upload(options)
+        resp.data
+      end
+
+      # @example Request syntax with placeholder values
+      #
+      #   object = multipart_upload.complete({
+      #     multipart_upload: {
+      #       parts: [
+      #         {
+      #           etag: "ETag",
+      #           part_number: 1,
+      #         },
+      #       ],
+      #     },
+      #     request_payer: "requester", # accepts requester
+      #   })
+      # @param [Hash] options ({})
+      # @option options [Types::CompletedMultipartUpload] :multipart_upload
+      # @option options [String] :request_payer
+      #   Confirms that the requester knows that she or he will be charged for
+      #   the request. Bucket owners need not specify this parameter in their
+      #   requests. Documentation on downloading objects from requester pays
+      #   buckets can be found at
+      #   http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+      # @return [Object]
+      def complete(options = {})
+        options = options.merge(
+          bucket: @bucket_name,
+          key: @object_key,
+          upload_id: @id
+        )
+        resp = @client.complete_multipart_upload(options)
+        Object.new(
+          bucket_name: @bucket_name,
+          key: @object_key,
+          client: @client
+        )
+      end
+
+      # @!group Associations
+
+      # @return [Object]
+      def object
+        Object.new(
+          bucket_name: @bucket_name,
+          key: @object_key,
+          client: @client
+        )
+      end
+
+      # @param [String] part_number
+      # @return [MultipartUploadPart]
+      def part(part_number)
+        MultipartUploadPart.new(
+          bucket_name: @bucket_name,
+          object_key: @object_key,
+          multipart_upload_id: @id,
+          part_number: part_number,
+          client: @client
+        )
+      end
+
+      # @example Request syntax with placeholder values
+      #
+      #   parts = multipart_upload.parts({
+      #     request_payer: "requester", # accepts requester
+      #   })
+      # @param [Hash] options ({})
+      # @option options [String] :request_payer
+      #   Confirms that the requester knows that she or he will be charged for
+      #   the request. Bucket owners need not specify this parameter in their
+      #   requests. Documentation on downloading objects from requester pays
+      #   buckets can be found at
+      #   http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+      # @return [MultipartUploadPart::Collection]
+      def parts(options = {})
+        batches = Enumerator.new do |y|
+          options = options.merge(
+            bucket: @bucket_name,
+            key: @object_key,
+            upload_id: @id
+          )
+          resp = @client.list_parts(options)
+          resp.each_page do |page|
+            batch = []
+            page.data.parts.each do |p|
+              batch << MultipartUploadPart.new(
+                bucket_name: options[:bucket],
+                object_key: options[:key],
+                multipart_upload_id: options[:upload_id],
+                part_number: p.part_number,
+                data: p,
+                client: @client
+              )
+            end
+            y.yield(batch)
+          end
+        end
+        MultipartUploadPart::Collection.new(batches)
+      end
+
+      # @deprecated
+      # @api private
+      def identifiers
+        {
+          bucket_name: @bucket_name,
+          object_key: @object_key,
+          id: @id
+        }
+      end
+      deprecated(:identifiers)
+
+      private
+
+      def extract_bucket_name(args, options)
+        value = args[0] || options.delete(:bucket_name)
+        case value
+        when String then value
+        when nil then raise ArgumentError, "missing required option :bucket_name"
+        else
+          msg = "expected :bucket_name to be a String, got #{value.class}"
+          raise ArgumentError, msg
+        end
+      end
+
+      def extract_object_key(args, options)
+        value = args[1] || options.delete(:object_key)
+        case value
+        when String then value
+        when nil then raise ArgumentError, "missing required option :object_key"
+        else
+          msg = "expected :object_key to be a String, got #{value.class}"
+          raise ArgumentError, msg
+        end
+      end
+
+      def extract_id(args, options)
+        value = args[2] || options.delete(:id)
+        case value
+        when String then value
+        when nil then raise ArgumentError, "missing required option :id"
+        else
+          msg = "expected :id to be a String, got #{value.class}"
+          raise ArgumentError, msg
+        end
+      end
+
+      class Collection < Aws::Resources::Collection; end
+    end
+  end
+end