aws-sdk-s3 1.9.0 → 1.40.0

data/lib/aws-sdk-s3/customizations/bucket.rb

@@ -61,8 +61,8 @@ module Aws
       # You can pass `virtual_host: true` to use the bucket name as the
       # host name.
       #
-      #     bucket = s3.bucket('my.bucket.com', virtual_host: true)
-      #     bucket.url
+      #     bucket = s3.bucket('my.bucket.com')
+      #     bucket.url(virtual_host: true)
       #     #=> "http://my.bucket.com"
       #
       # @option options [Boolean] :virtual_host (false) When `true`,

data/lib/aws-sdk-s3/customizations/object.rb

@@ -214,6 +214,62 @@ module Aws
         url.to_s
       end
 
+      # Uploads a stream in a streaming fashion to the current object in S3.
+      #
+      #     # Passed chunks automatically split into multipart upload parts
+      #     # and the parts are uploaded in parallel. This allows for streaming uploads
+      #     # that never touch the disk.
+      #
+      #  Note that this is known to have issues in JRuby until jruby-9.1.15.0, so avoid using this with older versions of JRuby.
+      #
+      # @example Streaming chunks of data
+      #     obj.upload_stream do |write_stream|
+      #       10.times { write_stream << 'foo' }
+      #     end
+      # @example Streaming chunks of data
+      #     obj.upload_stream do |write_stream|
+      #       IO.copy_stream(IO.popen('ls'), write_stream)
+      #     end
+      # @example Streaming chunks of data
+      #     obj.upload_stream do |write_stream|
+      #       IO.copy_stream(STDIN, write_stream)
+      #     end
+      #
+      # @option options [Integer] :thread_count
+      #   The number of parallel multipart uploads
+      #   Default `:thread_count` is `10`.
+      #
+      # @option options [Boolean] :tempfile
+      #   Normally read data is stored in memory when building the parts in order to complete
+      #   the underlying multipart upload. By passing `:tempfile => true` data read will be
+      #   temporarily stored on disk reducing the memory footprint vastly.
+      #   Default `:tempfile` is `false`.
+      #
+      # @option options [Integer] :part_size
+      #   Define how big each part size but the last should be.
+      #   Default `:part_size` is `5 * 1024 * 1024`.
+      #
+      # @raise [MultipartUploadError] If an object is being uploaded in
+      #   parts, and the upload can not be completed, then the upload is
+      #   aborted and this error is raised.  The raised error has a `#errors`
+      #   method that returns the failures that caused the upload to be
+      #   aborted.
+      #
+      # @return [Boolean] Returns `true` when the object is uploaded
+      #   without any errors.
+      #
+      def upload_stream(options = {}, &block)
+        uploading_options = options.dup
+        uploader = MultipartStreamUploader.new(
+          client: client,
+          thread_count: uploading_options.delete(:thread_count),
+          tempfile: uploading_options.delete(:tempfile),
+          part_size: uploading_options.delete(:part_size),
+        )
+        uploader.upload(uploading_options.merge(bucket: bucket_name, key: key), &block)
+        true
+      end
+
       # Uploads a file from disk to the current object in S3.
       #
       #     # small files are uploaded in a single API call
@@ -277,6 +333,10 @@ module Aws
       # @option options [String] thread_count Customize threads used in multipart
       #   download, if not provided, 10 is default value
       #
+      # @option options [String] version_id The object version id used to retrieve
+      #   the object, to know more about object versioning, see:
+      #   https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
+      #
       # @return [Boolean] Returns `true` when the file is downloaded
       #   without any errors.
       def download_file(destination, options = {})

data/lib/aws-sdk-s3/customizations/object_summary.rb

@@ -60,6 +60,13 @@ module Aws
         object.upload_file(source, options)
       end
 
+      # @options (see Object#upload_stream)
+      # @return (see Object#upload_stream)
+      # @see Object#upload_stream
+      def upload_stream(options = {}, &block)
+        object.upload_stream(options, &block)
+      end
+
       # @param (see Object#download_file)
       # @options (see Object#download_file)
       # @return (see Object#download_file)

data/lib/aws-sdk-s3/customizations.rb

@@ -6,6 +6,7 @@ require 'aws-sdk-s3/file_uploader'
 require 'aws-sdk-s3/file_downloader'
 require 'aws-sdk-s3/legacy_signer'
 require 'aws-sdk-s3/multipart_file_uploader'
+require 'aws-sdk-s3/multipart_stream_uploader'
 require 'aws-sdk-s3/multipart_upload_error'
 require 'aws-sdk-s3/object_copier'
 require 'aws-sdk-s3/object_multipart_copier'

data/lib/aws-sdk-s3/encryption/client.rb

@@ -179,7 +179,7 @@ module Aws
 
         extend Deprecations
 
-        # Creates a new encryption client. You must provide on of the following
+        # Creates a new encryption client. You must provide one of the following
         # options:
         #
         # * `:encryption_key`

data/lib/aws-sdk-s3/encryption/io_auth_decrypter.rb

@@ -22,8 +22,10 @@ module Aws
 
         def write(chunk)
           chunk = truncate_chunk(chunk)
-          @bytes_written += chunk.bytesize
-          @decrypter.write(chunk)
+          if chunk.bytesize > 0
+            @bytes_written += chunk.bytesize
+            @decrypter.write(chunk)
+          end
         end
 
         def finalize
@@ -39,8 +41,12 @@ module Aws
         def truncate_chunk(chunk)
           if chunk.bytesize + @bytes_written <= @max_bytes
             chunk
-          else
+          elsif @bytes_written < @max_bytes
             chunk[0..(@max_bytes - @bytes_written - 1)]
+          else
+            # If the tag was sent over after the full body has been read,
+            # we don't want to accidentally append it.
+            ""
           end
         end
 

data/lib/aws-sdk-s3/event_streams.rb

@@ -0,0 +1,62 @@
+# WARNING ABOUT GENERATED CODE
+#
+# This file is generated. See the contributing guide for more information:
+# https://github.com/aws/aws-sdk-ruby/blob/master/CONTRIBUTING.md
+#
+# WARNING ABOUT GENERATED CODE
+
+module Aws::S3
+  module EventStreams
+    class SelectObjectContentEventStream
+
+      def initialize
+        @event_emitter = Aws::EventEmitter.new
+      end
+
+      def on_records_event(&block)
+        @event_emitter.on(:records, Proc.new)
+      end
+
+      def on_stats_event(&block)
+        @event_emitter.on(:stats, Proc.new)
+      end
+
+      def on_progress_event(&block)
+        @event_emitter.on(:progress, Proc.new)
+      end
+
+      def on_cont_event(&block)
+        @event_emitter.on(:cont, Proc.new)
+      end
+
+      def on_end_event(&block)
+        @event_emitter.on(:end, Proc.new)
+      end
+
+      def on_error_event(&block)
+        @event_emitter.on(:error, Proc.new)
+      end
+
+      def on_initial_response_event(&block)
+        @event_emitter.on(:initial_response, Proc.new)
+      end
+
+      def on_event(&block)
+        on_records_event(&block)
+        on_stats_event(&block)
+        on_progress_event(&block)
+        on_cont_event(&block)
+        on_end_event(&block)
+        on_error_event(&block)
+        on_initial_response_event(&block)
+      end
+
+      # @api private
+      # @return Aws::EventEmitter
+      attr_reader :event_emitter
+
+    end
+
+  end
+end
+

data/lib/aws-sdk-s3/file_downloader.rb

@@ -24,15 +24,18 @@ module Aws
         @mode = options[:mode] || "auto"
         @thread_count = options[:thread_count] || THREAD_COUNT
         @chunk_size = options[:chunk_size]
-        @bucket = options[:bucket]
-        @key = options[:key]
+        @params = {
+          bucket: options[:bucket],
+          key: options[:key],
+        }
+        @params[:version_id] = options[:version_id] if options[:version_id]
 
         case @mode
         when "auto" then multipart_download
         when "single_request" then single_request
         when "get_range"
           if @chunk_size
-            resp = @client.head_object(bucket: @bucket, key: @key)
+            resp = @client.head_object(@params)
             multithreaded_get_by_ranges(construct_chunks(resp.content_length))
           else
             msg = "In :get_range mode, :chunk_size must be provided"
@@ -48,7 +51,7 @@ module Aws
       private
 
       def multipart_download
-        resp = @client.head_object(bucket: @bucket, key: @key, part_number: 1)
+        resp = @client.head_object(@params.merge(part_number: 1))
         count = resp.parts_count
         if count.nil? || count <= 1
           resp.content_length < MIN_CHUNK_SIZE ?
@@ -56,7 +59,7 @@ module Aws
             multithreaded_get_by_ranges(construct_chunks(resp.content_length))
         else
           # partNumber is an option
-          resp = @client.head_object(bucket: @bucket, key: @key)
+          resp = @client.head_object(@params)
           resp.content_length < MIN_CHUNK_SIZE ?
             single_request :
             compute_mode(resp.content_length, count)
@@ -112,9 +115,7 @@ module Aws
           batch.each do |chunk|
             threads << Thread.new do
               resp = @client.get_object(
-                :bucket => @bucket,
-                :key => @key,
-                param.to_sym => chunk
+                @params.merge(param.to_sym => chunk)
               )
               write(resp)
             end
@@ -131,7 +132,7 @@ module Aws
 
       def single_request
         @client.get_object(
-          bucket: @bucket, key: @key, response_target: @path
+          @params.merge(response_target: @path)
         )
       end
     end

data/lib/aws-sdk-s3/file_part.rb

@@ -56,14 +56,12 @@ module Aws
       end
 
       def read_from_file(bytes, output_buffer)
-        if bytes
-          data = @file.read([remaining_bytes, bytes].min)
-          data = nil if data == ''
-        else
-          data = @file.read(remaining_bytes)
-        end
+        length = [remaining_bytes, *bytes].min
+        data   = @file.read(length, output_buffer)
+
         @position += data ? data.bytesize : 0
-        output_buffer ? output_buffer.replace(data || '') : data
+
+        data.to_s unless bytes && (data.nil? || data.empty?)
       end
 
       def remaining_bytes

data/lib/aws-sdk-s3/file_uploader.rb

@@ -45,9 +45,7 @@ module Aws
 
       def open_file(source)
         if String === source || Pathname === source
-          file = File.open(source, 'rb')
-          yield(file)
-          file.close
+          File.open(source, 'rb') { |file| yield(file) }
         else
           yield(source)
         end

data/lib/aws-sdk-s3/multipart_stream_uploader.rb

@@ -0,0 +1,162 @@
+require 'thread'
+require 'set'
+require 'tempfile'
+require 'stringio'
+
+module Aws
+  module S3
+    # @api private
+    class MultipartStreamUploader
+      # api private
+      PART_SIZE = 5 * 1024 * 1024 # 5MB
+
+      # api private
+      THREAD_COUNT = 10
+
+      # api private
+      TEMPFILE_PREIX = 'aws-sdk-s3-upload_stream'.freeze
+
+      # @api private
+      CREATE_OPTIONS =
+        Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names)
+
+      # @api private
+      UPLOAD_PART_OPTIONS =
+        Set.new(Client.api.operation(:upload_part).input.shape.member_names)
+
+      # @option options [Client] :client
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+        @tempfile = options[:tempfile]
+        @part_size = options[:part_size] || PART_SIZE
+        @thread_count = options[:thread_count] || THREAD_COUNT
+      end
+
+      # @return [Client]
+      attr_reader :client
+
+      # @option options [required,String] :bucket
+      # @option options [required,String] :key
+      # @return [void]
+      def upload(options = {}, &block)
+        upload_id = initiate_upload(options)
+        parts = upload_parts(upload_id, options, &block)
+        complete_upload(upload_id, parts, options)
+      end
+
+      private
+
+      def initiate_upload(options)
+        @client.create_multipart_upload(create_opts(options)).upload_id
+      end
+
+      def complete_upload(upload_id, parts, options)
+        @client.complete_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id,
+          multipart_upload: { parts: parts })
+      end
+
+      def upload_parts(upload_id, options, &block)
+        completed = Queue.new
+        errors = IO.pipe do |read_pipe, write_pipe|
+          threads = upload_in_threads(read_pipe, completed, upload_part_opts(options).merge(upload_id: upload_id))
+          block.call(write_pipe)
+          write_pipe.close
+          threads.map(&:value).compact
+        end
+        if errors.empty?
+          Array.new(completed.size) { completed.pop }.sort_by { |part| part[:part_number] }
+        else
+          abort_upload(upload_id, options, errors)
+        end
+      end
+
+      def abort_upload(upload_id, options, errors)
+        @client.abort_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id
+        )
+        msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
+        raise MultipartUploadError.new(msg, errors)
+      rescue MultipartUploadError => error
+        raise error
+      rescue => error
+        msg = "failed to abort multipart upload: #{error.message}"
+        raise MultipartUploadError.new(msg, errors + [error])
+      end
+
+      def create_opts(options)
+        CREATE_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def upload_part_opts(options)
+        UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def read_to_part_body(read_pipe)
+        return if read_pipe.closed?
+        temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new
+        temp_io.binmode
+        bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size)
+        temp_io.rewind
+        if bytes_copied == 0
+          if Tempfile === temp_io
+            temp_io.close
+            temp_io.unlink
+          end
+          nil
+        else
+          temp_io
+        end
+      end
+
+      def upload_in_threads(read_pipe, completed, options)
+        mutex = Mutex.new
+        part_number = 0
+        @thread_count.times.map do
+          thread = Thread.new do
+            begin
+              loop do
+                body, thread_part_number = mutex.synchronize do
+                  [read_to_part_body(read_pipe), part_number += 1]
+                end
+                break unless (body || thread_part_number == 1)
+                begin
+                  part = options.merge(
+                    body: body,
+                    part_number: thread_part_number,
+                  )
+                  resp = @client.upload_part(part)
+                  completed << {etag: resp.etag, part_number: part[:part_number]}
+                ensure
+                  if Tempfile === body
+                    body.close
+                    body.unlink
+                  elsif StringIO === body
+                    body.string.clear
+                  end
+                end
+              end
+              nil
+            rescue => error
+              # keep other threads from uploading other parts
+              mutex.synchronize { read_pipe.close_read }
+              error
+            end
+          end
+          thread.abort_on_exception = true
+          thread
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/multipart_upload.rb

@@ -70,6 +70,7 @@ module Aws::S3
       data[:storage_class]
     end
 
+    
     # @return [Types::Owner]
     def owner
       data[:owner]

data/lib/aws-sdk-s3/multipart_upload_part.rb

@@ -66,7 +66,7 @@ module Aws::S3
       data[:etag]
     end
 
-    # Size of the uploaded part data.
+    # Size in bytes of the uploaded part data.
     # @return [Integer]
     def size
       data[:size]
@@ -235,7 +235,7 @@ module Aws::S3
     #   must use the form bytes=first-last, where the first and last are the
     #   zero-based byte offsets to copy. For example, bytes=0-9 indicates that
     #   you want to copy the first ten bytes of the source. You can copy a
-    #   range only if the source object is greater than 5 GB.
+    #   range only if the source object is greater than 5 MB.
     # @option options [String] :sse_customer_algorithm
     #   Specifies the algorithm to use to when encrypting the object (e.g.,
     #   AES256).