aws-sdk-s3 1.9.0 → 1.24.1

data/lib/aws-sdk-s3/file_downloader.rb

@@ -24,15 +24,18 @@ module Aws
         @mode = options[:mode] || "auto"
         @thread_count = options[:thread_count] || THREAD_COUNT
         @chunk_size = options[:chunk_size]
-        @bucket = options[:bucket]
-        @key = options[:key]
+        @params = {
+          bucket: options[:bucket],
+          key: options[:key],
+        }
+        @params[:version_id] = options[:version_id] if options[:version_id]
 
         case @mode
         when "auto" then multipart_download
         when "single_request" then single_request
         when "get_range"
           if @chunk_size
-            resp = @client.head_object(bucket: @bucket, key: @key)
+            resp = @client.head_object(@params)
             multithreaded_get_by_ranges(construct_chunks(resp.content_length))
           else
             msg = "In :get_range mode, :chunk_size must be provided"
@@ -48,7 +51,7 @@ module Aws
       private
 
       def multipart_download
-        resp = @client.head_object(bucket: @bucket, key: @key, part_number: 1)
+        resp = @client.head_object(@params.merge(part_number: 1))
         count = resp.parts_count
         if count.nil? || count <= 1
           resp.content_length < MIN_CHUNK_SIZE ?
@@ -56,7 +59,7 @@ module Aws
             multithreaded_get_by_ranges(construct_chunks(resp.content_length))
         else
           # partNumber is an option
-          resp = @client.head_object(bucket: @bucket, key: @key)
+          resp = @client.head_object(@params)
           resp.content_length < MIN_CHUNK_SIZE ?
             single_request :
             compute_mode(resp.content_length, count)
@@ -112,9 +115,7 @@ module Aws
           batch.each do |chunk|
             threads << Thread.new do
               resp = @client.get_object(
-                :bucket => @bucket,
-                :key => @key,
-                param.to_sym => chunk
+                @params.merge(param.to_sym => chunk)
               )
               write(resp)
             end
@@ -131,7 +132,7 @@ module Aws
 
       def single_request
         @client.get_object(
-          bucket: @bucket, key: @key, response_target: @path
+          @params.merge(response_target: @path)
         )
       end
     end

data/lib/aws-sdk-s3/file_part.rb

@@ -56,14 +56,12 @@ module Aws
       end
 
       def read_from_file(bytes, output_buffer)
-        if bytes
-          data = @file.read([remaining_bytes, bytes].min)
-          data = nil if data == ''
-        else
-          data = @file.read(remaining_bytes)
-        end
+        length = [remaining_bytes, *bytes].min
+        data   = @file.read(length, output_buffer)
+
         @position += data ? data.bytesize : 0
-        output_buffer ? output_buffer.replace(data || '') : data
+
+        data.to_s unless bytes && (data.nil? || data.empty?)
       end
 
       def remaining_bytes

data/lib/aws-sdk-s3/file_uploader.rb

@@ -45,9 +45,7 @@ module Aws
 
       def open_file(source)
         if String === source || Pathname === source
-          file = File.open(source, 'rb')
-          yield(file)
-          file.close
+          File.open(source, 'rb') { |file| yield(file) }
         else
           yield(source)
         end

data/lib/aws-sdk-s3/multipart_stream_uploader.rb

@@ -0,0 +1,160 @@
+require 'thread'
+require 'set'
+require 'tempfile'
+require 'stringio'
+
+module Aws
+  module S3
+    # @api private
+    class MultipartStreamUploader
+      # api private
+      PART_SIZE = 5 * 1024 * 1024 # 5MB
+
+      # api private
+      THREAD_COUNT = 10
+
+      # api private
+      TEMPFILE_PREIX = 'aws-sdk-s3-upload_stream'.freeze
+
+      # @api private
+      CREATE_OPTIONS =
+        Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names)
+
+      # @api private
+      UPLOAD_PART_OPTIONS =
+        Set.new(Client.api.operation(:upload_part).input.shape.member_names)
+
+      # @option options [Client] :client
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+        @tempfile = options[:tempfile]
+        @part_size = options[:part_size] || PART_SIZE
+        @thread_count = options[:thread_count] || THREAD_COUNT
+      end
+
+      # @return [Client]
+      attr_reader :client
+
+      # @option options [required,String] :bucket
+      # @option options [required,String] :key
+      # @return [void]
+      def upload(options = {}, &block)
+        upload_id = initiate_upload(options)
+        parts = upload_parts(upload_id, options, &block)
+        complete_upload(upload_id, parts, options)
+      end
+
+      private
+
+      def initiate_upload(options)
+        @client.create_multipart_upload(create_opts(options)).upload_id
+      end
+
+      def complete_upload(upload_id, parts, options)
+        @client.complete_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id,
+          multipart_upload: { parts: parts })
+      end
+
+      def upload_parts(upload_id, options, &block)
+        completed = Queue.new
+        errors = IO.pipe do |read_pipe, write_pipe|
+          threads = upload_in_threads(read_pipe, completed, upload_part_opts(options).merge(upload_id: upload_id))
+          block.call(write_pipe)
+          write_pipe.close
+          threads.map(&:value).compact
+        end
+        if errors.empty?
+          Array.new(completed.size) { completed.pop }.sort_by { |part| part[:part_number] }
+        else
+          abort_upload(upload_id, options, errors)
+        end
+      end
+
+      def abort_upload(upload_id, options, errors)
+        @client.abort_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id
+        )
+        msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
+        raise MultipartUploadError.new(msg, errors)
+      rescue MultipartUploadError => error
+        raise error
+      rescue => error
+        msg = "failed to abort multipart upload: #{error.message}"
+        raise MultipartUploadError.new(msg, errors + [error])
+      end
+
+      def create_opts(options)
+        CREATE_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def upload_part_opts(options)
+        UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def read_to_part_body(read_pipe)
+        return if read_pipe.closed?
+        temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new
+        temp_io.binmode
+        bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size)
+        temp_io.rewind
+        if bytes_copied == 0
+          if Tempfile === temp_io
+            temp_io.close
+            temp_io.unlink
+          end
+          nil
+        else
+          temp_io
+        end
+      end
+
+      def upload_in_threads(read_pipe, completed, options)
+        mutex = Mutex.new
+        part_number = 0
+        @thread_count.times.map do
+          thread = Thread.new do
+            begin
+              loop do
+                body, thread_part_number = mutex.synchronize do
+                  [read_to_part_body(read_pipe), part_number += 1]
+                end
+                break unless (body || thread_part_number == 1)
+                begin
+                  part = options.merge(
+                    body: body,
+                    part_number: thread_part_number,
+                  )
+                  resp = @client.upload_part(part)
+                  completed << {etag: resp.etag, part_number: part[:part_number]}
+                ensure
+                  if Tempfile === body
+                    body.close
+                    body.unlink
+                  end
+                end
+              end
+              nil
+            rescue => error
+              # keep other threads from uploading other parts
+              mutex.synchronize { read_pipe.close_read }
+              error
+            end
+          end
+          thread.abort_on_exception = true
+          thread
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/multipart_upload_part.rb

@@ -66,7 +66,7 @@ module Aws::S3
       data[:etag]
     end
 
-    # Size of the uploaded part data.
+    # Size in bytes of the uploaded part data.
     # @return [Integer]
     def size
       data[:size]

data/lib/aws-sdk-s3/object.rb

@@ -866,7 +866,7 @@ module Aws::S3
     #   http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
     # @option options [String] :tagging
     #   The tag-set for the object. The tag-set must be encoded as URL Query
-    #   parameters
+    #   parameters. (For example, "Key1=Value1")
     # @return [Types::PutObjectOutput]
     def put(options = {})
       options = options.merge(
@@ -898,11 +898,14 @@ module Aws::S3
     #             record_delimiter: "RecordDelimiter",
     #             field_delimiter: "FieldDelimiter",
     #             quote_character: "QuoteCharacter",
+    #             allow_quoted_record_delimiter: false,
     #           },
-    #           compression_type: "NONE", # accepts NONE, GZIP
+    #           compression_type: "NONE", # accepts NONE, GZIP, BZIP2
     #           json: {
     #             type: "DOCUMENT", # accepts DOCUMENT, LINES
     #           },
+    #           parquet: {
+    #           },
     #         },
     #         expression_type: "SQL", # required, accepts SQL
     #         expression: "Expression", # required

data/lib/aws-sdk-s3/object_summary.rb

@@ -709,7 +709,7 @@ module Aws::S3
     #   http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
     # @option options [String] :tagging
     #   The tag-set for the object. The tag-set must be encoded as URL Query
-    #   parameters
+    #   parameters. (For example, "Key1=Value1")
     # @return [Types::PutObjectOutput]
     def put(options = {})
       options = options.merge(
@@ -741,11 +741,14 @@ module Aws::S3
     #             record_delimiter: "RecordDelimiter",
     #             field_delimiter: "FieldDelimiter",
     #             quote_character: "QuoteCharacter",
+    #             allow_quoted_record_delimiter: false,
     #           },
-    #           compression_type: "NONE", # accepts NONE, GZIP
+    #           compression_type: "NONE", # accepts NONE, GZIP, BZIP2
     #           json: {
     #             type: "DOCUMENT", # accepts DOCUMENT, LINES
     #           },
+    #           parquet: {
+    #           },
     #         },
     #         expression_type: "SQL", # required, accepts SQL
     #         expression: "Expression", # required

data/lib/aws-sdk-s3/plugins/accelerate.rb

@@ -60,6 +60,8 @@ each bucket.  [Go here for more information](http://docs.aws.amazon.com/AmazonS3
             endpoint.port = 443
             endpoint.host = "#{bucket_name}.s3-accelerate.amazonaws.com"
             context.http_request.endpoint = endpoint.to_s
+            # s3 accelerate endpoint doesn't work with 'expect' header
+            context.http_request.headers.delete('expect')
           end
 
           def use_combined_accelerate_dualstack_endpoint(context)
@@ -70,6 +72,8 @@ each bucket.  [Go here for more information](http://docs.aws.amazon.com/AmazonS3
             endpoint.port = 443
             endpoint.host = "#{bucket_name}.s3-accelerate.dualstack.amazonaws.com"
             context.http_request.endpoint = endpoint.to_s
+            # s3 accelerate endpoint doesn't work with 'expect' header
+            context.http_request.headers.delete('expect')
           end
 
           def validate_bucket_name!(bucket_name)

data/lib/aws-sdk-s3/plugins/md5s.rb

@@ -45,7 +45,7 @@ module Aws
           end
 
           def update_in_chunks(digest, io)
-            while chunk = io.read(CHUNK_SIZE)
+            while chunk = io.read(CHUNK_SIZE, buffer ||= "")
               digest.update(chunk)
             end
             io.rewind

data/lib/aws-sdk-s3/plugins/redirects.rb

@@ -18,7 +18,9 @@ by Amazon S3.
             response = @handler.call(context)
             if context.http_response.status_code == 307
               endpoint = context.http_response.headers['location']
-              context.http_request.endpoint = endpoint
+              unless context.http_request.endpoint.host.include?('fips')
+                context.http_request.endpoint = endpoint
+              end
               context.http_response.body.truncate(0)
               @handler.call(context)
             else

data/lib/aws-sdk-s3/plugins/s3_signer.rb

@@ -113,7 +113,7 @@ module Aws
           private
 
           def handle_region_errors(response)
-            if wrong_sigv4_region?(response)
+            if wrong_sigv4_region?(response) && !fips_region?(response)
               get_region_and_retry(response.context)
             else
               response
@@ -133,6 +133,10 @@ module Aws
             S3::BUCKET_REGIONS[context.params[:bucket]] = actual_region
           end
 
+          def fips_region?(resp)
+            resp.context.http_request.endpoint.host.include?('fips')
+          end
+
           def wrong_sigv4_region?(resp)
             resp.context.http_response.status_code == 400 &&
             (

data/lib/aws-sdk-s3/presigner.rb

@@ -98,6 +98,7 @@ module Aws
         req.handlers.remove(Seahorse::Client::Plugins::ContentLength::Handler)
 
         signer = build_signer(req.context.config)
+        req.context[:presigned_url] = true
 
         req.handle(step: :send) do |context|