aws-sdk-s3 1.13.0 → 1.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 76fdd52e31b770c550908db83818c5842b0e704c
-  data.tar.gz: 596f4b30103399a4bfe7b8cf2cad093cfd8ac6e1
+  metadata.gz: ebf985aabc5276f0257be4b38c436b5261e133aa
+  data.tar.gz: eb740e2166050fd5b48daf78d95797c167fc5d5b
 SHA512:
-  metadata.gz: 4a3f32fa2257bb3dd5bdfd94b329a54a047e1fa6b6ccb1425e21b7e86c9664dd1d62165d400a65dbc6c814fa0935c08f2f3e9d7535cf7576094e4b332c745405
-  data.tar.gz: 7e3729ceec9fe3cd819926dd14ba0fdf675af8610773f5a0daf5766fbd5bcfb6242bd4b91e1910dbb04c17732004bb46d8bd1976f22181d4625a95a3d1bf5381
+  metadata.gz: 5de1c034a92b407852bd6b268d42c3ecb8593f0d1c27ea0056e6b83f09263f42a4e8bfdfa3df18f01f0435bf1206a6f779c7d34e3c23d36a3e21510345a0da1e
+  data.tar.gz: 4a6d12d6dd43abb6064fd45b8ce703c98f46ffa51d220cd2b67a7f35e4ac1bcf00c0876874359f623db56b2f6ad95ff355b8669e967830bdd912a23868a7268f

data/lib/aws-sdk-s3.rb

@@ -63,6 +63,6 @@ require_relative 'aws-sdk-s3/event_streams'
 # @service
 module Aws::S3
 
-  GEM_VERSION = '1.13.0'
+  GEM_VERSION = '1.14.0'
 
 end

data/lib/aws-sdk-s3/client.rb

@@ -6371,7 +6371,7 @@ module Aws::S3
         params: params,
         config: config)
       context[:gem_name] = 'aws-sdk-s3'
-      context[:gem_version] = '1.13.0'
+      context[:gem_version] = '1.14.0'
       Seahorse::Client::Request.new(handlers, context)
     end
 

data/lib/aws-sdk-s3/customizations.rb

@@ -6,6 +6,7 @@ require 'aws-sdk-s3/file_uploader'
 require 'aws-sdk-s3/file_downloader'
 require 'aws-sdk-s3/legacy_signer'
 require 'aws-sdk-s3/multipart_file_uploader'
+require 'aws-sdk-s3/multipart_stream_uploader'
 require 'aws-sdk-s3/multipart_upload_error'
 require 'aws-sdk-s3/object_copier'
 require 'aws-sdk-s3/object_multipart_copier'

data/lib/aws-sdk-s3/customizations/object.rb

@@ -214,6 +214,54 @@ module Aws
         url.to_s
       end
 
+      # Uploads a stream in a streaming fashion to the current object in S3.
+      #
+      #     # Passed chunks automatically split into multipart upload parts
+      #     # and the parts are uploaded in parallel. This allows for streaming uploads
+      #     # that never touch the disk.
+      #
+      #  Note that this is known to have issues in JRuby until jruby-9.1.15.0, so avoid using this with older versions of JRuby.
+      #
+      # @example Streaming chunks of data
+      #     obj.upload_file do |write_stream|
+      #       10.times { write_stream << 'hello' }
+      #     end
+      #
+      # @option options [Integer] :thread_count
+      #   The number of parallel multipart uploads
+      #   Default `:thread_count` is `10`.
+      #
+      # @option options [Boolean] :tempfile
+      #   Normally read data is stored in memory when building the parts in order to complete
+      #   the underlying multipart upload. By passing `:tempfile => true` data read will be
+      #   temporarily stored on disk reducing the memory footprint vastly.
+      #   Default `:tempfile` is `false`.
+      #
+      # @option options [Integer] :part_size
+      #   Define how big each part size but the last should be.
+      #   Default `:part_size` is `5 * 1024 * 1024`.
+      #
+      # @raise [MultipartUploadError] If an object is being uploaded in
+      #   parts, and the upload can not be completed, then the upload is
+      #   aborted and this error is raised.  The raised error has a `#errors`
+      #   method that returns the failures that caused the upload to be
+      #   aborted.
+      #
+      # @return [Boolean] Returns `true` when the object is uploaded
+      #   without any errors.
+      #
+      def upload_stream(options = {}, &block)
+        uploading_options = options.dup
+        uploader = MultipartStreamUploader.new(
+          client: client,
+          thread_count: uploading_options.delete(:thread_count),
+          tempfile: uploading_options.delete(:tempfile),
+          part_size: uploading_options.delete(:part_size),
+        )
+        uploader.upload(uploading_options.merge(bucket: bucket_name, key: key), &block)
+        true
+      end
+
       # Uploads a file from disk to the current object in S3.
       #
       #     # small files are uploaded in a single API call

data/lib/aws-sdk-s3/customizations/object_summary.rb

@@ -60,6 +60,13 @@ module Aws
         object.upload_file(source, options)
       end
 
+      # @options (see Object#upload_stream)
+      # @return (see Object#upload_stream)
+      # @see Object#upload_stream
+      def upload_stream(options = {}, &block)
+        object.upload_stream(options, &block)
+      end
+
       # @param (see Object#download_file)
       # @options (see Object#download_file)
       # @return (see Object#download_file)

data/lib/aws-sdk-s3/multipart_stream_uploader.rb

@@ -0,0 +1,160 @@
+require 'thread'
+require 'set'
+require 'tempfile'
+require 'stringio'
+
+module Aws
+  module S3
+    # @api private
+    class MultipartStreamUploader
+      # api private
+      PART_SIZE = 5 * 1024 * 1024 # 5MB
+
+      # api private
+      THREAD_COUNT = 10
+
+      # api private
+      TEMPFILE_PREIX = 'aws-sdk-s3-upload_stream'.freeze
+
+      # @api private
+      CREATE_OPTIONS =
+        Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names)
+
+      # @api private
+      UPLOAD_PART_OPTIONS =
+        Set.new(Client.api.operation(:upload_part).input.shape.member_names)
+
+      # @option options [Client] :client
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+        @tempfile = options[:tempfile]
+        @part_size = options[:part_size] || PART_SIZE
+        @thread_count = options[:thread_count] || THREAD_COUNT
+      end
+
+      # @return [Client]
+      attr_reader :client
+
+      # @option options [required,String] :bucket
+      # @option options [required,String] :key
+      # @return [void]
+      def upload(options = {}, &block)
+        upload_id = initiate_upload(options)
+        parts = upload_parts(upload_id, options, &block)
+        complete_upload(upload_id, parts, options)
+      end
+
+      private
+
+      def initiate_upload(options)
+        @client.create_multipart_upload(create_opts(options)).upload_id
+      end
+
+      def complete_upload(upload_id, parts, options)
+        @client.complete_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id,
+          multipart_upload: { parts: parts })
+      end
+
+      def upload_parts(upload_id, options, &block)
+        completed = Queue.new
+        errors = IO.pipe do |read_pipe, write_pipe|
+          threads = upload_in_threads(read_pipe, completed, upload_part_opts(options).merge(upload_id: upload_id))
+          block.call(write_pipe)
+          write_pipe.close
+          threads.map(&:value).compact
+        end
+        if errors.empty?
+          Array.new(completed.size) { completed.pop }.sort_by { |part| part[:part_number] }
+        else
+          abort_upload(upload_id, options, errors)
+        end
+      end
+
+      def abort_upload(upload_id, options, errors)
+        @client.abort_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id
+        )
+        msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
+        raise MultipartUploadError.new(msg, errors)
+      rescue MultipartUploadError => error
+        raise error
+      rescue => error
+        msg = "failed to abort multipart upload: #{error.message}"
+        raise MultipartUploadError.new(msg, errors + [error])
+      end
+
+      def create_opts(options)
+        CREATE_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def upload_part_opts(options)
+        UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def read_to_part_body(read_pipe)
+        return if read_pipe.closed?
+        temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new
+        temp_io.binmode
+        bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size)
+        temp_io.rewind
+        if bytes_copied == 0
+          if Tempfile === temp_io
+            temp_io.close
+            temp_io.unlink
+          end
+          nil
+        else
+          temp_io
+        end
+      end
+
+      def upload_in_threads(read_pipe, completed, options)
+        mutex = Mutex.new
+        part_number = 0
+        @thread_count.times.map do
+          thread = Thread.new do
+            begin
+              loop do
+                body, thread_part_number = mutex.synchronize do
+                  [read_to_part_body(read_pipe), part_number += 1]
+                end
+                break unless body
+                begin
+                  part = options.merge(
+                    body: body,
+                    part_number: thread_part_number,
+                  )
+                  resp = @client.upload_part(part)
+                  completed << {etag: resp.etag, part_number: part[:part_number]}
+                ensure
+                  if Tempfile === body
+                    body.close
+                    body.unlink
+                  end
+                end
+              end
+              nil
+            rescue => error
+              # keep other threads from uploading other parts
+              mutex.synchronize { read_pipe.close_read }
+              error
+            end
+          end
+          thread.abort_on_exception = true
+          thread
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws-sdk-s3
 version: !ruby/object:Gem::Version
-  version: 1.13.0
+  version: 1.14.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-22 00:00:00.000000000 Z
+date: 2018-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-kms
@@ -109,6 +109,7 @@ files:
 - lib/aws-sdk-s3/file_uploader.rb
 - lib/aws-sdk-s3/legacy_signer.rb
 - lib/aws-sdk-s3/multipart_file_uploader.rb
+- lib/aws-sdk-s3/multipart_stream_uploader.rb
 - lib/aws-sdk-s3/multipart_upload.rb
 - lib/aws-sdk-s3/multipart_upload_error.rb
 - lib/aws-sdk-s3/multipart_upload_part.rb