aws-sdk-s3 1.13.0 → 1.23.1

data/lib/aws-sdk-s3/client_api.rb

@@ -21,6 +21,7 @@ module Aws::S3
     AccessControlPolicy = Shapes::StructureShape.new(name: 'AccessControlPolicy')
     AccessControlTranslation = Shapes::StructureShape.new(name: 'AccessControlTranslation')
     AccountId = Shapes::StringShape.new(name: 'AccountId')
+    AllowQuotedRecordDelimiter = Shapes::BooleanShape.new(name: 'AllowQuotedRecordDelimiter')
     AllowedHeader = Shapes::StringShape.new(name: 'AllowedHeader')
     AllowedHeaders = Shapes::ListShape.new(name: 'AllowedHeaders', flattened: true)
     AllowedMethod = Shapes::StringShape.new(name: 'AllowedMethod')
@@ -117,6 +118,8 @@ module Aws::S3
     DeleteBucketWebsiteRequest = Shapes::StructureShape.new(name: 'DeleteBucketWebsiteRequest')
     DeleteMarker = Shapes::BooleanShape.new(name: 'DeleteMarker')
     DeleteMarkerEntry = Shapes::StructureShape.new(name: 'DeleteMarkerEntry')
+    DeleteMarkerReplication = Shapes::StructureShape.new(name: 'DeleteMarkerReplication')
+    DeleteMarkerReplicationStatus = Shapes::StringShape.new(name: 'DeleteMarkerReplicationStatus')
     DeleteMarkerVersionId = Shapes::StringShape.new(name: 'DeleteMarkerVersionId')
     DeleteMarkers = Shapes::ListShape.new(name: 'DeleteMarkers', flattened: true)
     DeleteObjectOutput = Shapes::StructureShape.new(name: 'DeleteObjectOutput')
@@ -335,6 +338,7 @@ module Aws::S3
     OutputSerialization = Shapes::StructureShape.new(name: 'OutputSerialization')
     Owner = Shapes::StructureShape.new(name: 'Owner')
     OwnerOverride = Shapes::StringShape.new(name: 'OwnerOverride')
+    ParquetInput = Shapes::StructureShape.new(name: 'ParquetInput')
     Part = Shapes::StructureShape.new(name: 'Part')
     PartNumber = Shapes::IntegerShape.new(name: 'PartNumber')
     PartNumberMarker = Shapes::IntegerShape.new(name: 'PartNumberMarker')
@@ -344,6 +348,7 @@ module Aws::S3
     Permission = Shapes::StringShape.new(name: 'Permission')
     Policy = Shapes::StringShape.new(name: 'Policy')
     Prefix = Shapes::StringShape.new(name: 'Prefix')
+    Priority = Shapes::IntegerShape.new(name: 'Priority')
     Progress = Shapes::StructureShape.new(name: 'Progress')
     ProgressEvent = Shapes::StructureShape.new(name: 'ProgressEvent')
     Protocol = Shapes::StringShape.new(name: 'Protocol')
@@ -389,6 +394,8 @@ module Aws::S3
     ReplicaKmsKeyID = Shapes::StringShape.new(name: 'ReplicaKmsKeyID')
     ReplicationConfiguration = Shapes::StructureShape.new(name: 'ReplicationConfiguration')
     ReplicationRule = Shapes::StructureShape.new(name: 'ReplicationRule')
+    ReplicationRuleAndOperator = Shapes::StructureShape.new(name: 'ReplicationRuleAndOperator')
+    ReplicationRuleFilter = Shapes::StructureShape.new(name: 'ReplicationRuleFilter')
     ReplicationRuleStatus = Shapes::StringShape.new(name: 'ReplicationRuleStatus')
     ReplicationRules = Shapes::ListShape.new(name: 'ReplicationRules', flattened: true)
     ReplicationStatus = Shapes::StringShape.new(name: 'ReplicationStatus')
@@ -558,6 +565,7 @@ module Aws::S3
     CSVInput.add_member(:record_delimiter, Shapes::ShapeRef.new(shape: RecordDelimiter, location_name: "RecordDelimiter"))
     CSVInput.add_member(:field_delimiter, Shapes::ShapeRef.new(shape: FieldDelimiter, location_name: "FieldDelimiter"))
     CSVInput.add_member(:quote_character, Shapes::ShapeRef.new(shape: QuoteCharacter, location_name: "QuoteCharacter"))
+    CSVInput.add_member(:allow_quoted_record_delimiter, Shapes::ShapeRef.new(shape: AllowQuotedRecordDelimiter, location_name: "AllowQuotedRecordDelimiter"))
     CSVInput.struct_class = Types::CSVInput
 
     CSVOutput.add_member(:quote_fields, Shapes::ShapeRef.new(shape: QuoteFields, location_name: "QuoteFields"))
@@ -772,6 +780,9 @@ module Aws::S3
     DeleteMarkerEntry.add_member(:last_modified, Shapes::ShapeRef.new(shape: LastModified, location_name: "LastModified"))
     DeleteMarkerEntry.struct_class = Types::DeleteMarkerEntry
 
+    DeleteMarkerReplication.add_member(:status, Shapes::ShapeRef.new(shape: DeleteMarkerReplicationStatus, location_name: "Status"))
+    DeleteMarkerReplication.struct_class = Types::DeleteMarkerReplication
+
     DeleteMarkers.member = Shapes::ShapeRef.new(shape: DeleteMarkerEntry)
 
     DeleteObjectOutput.add_member(:delete_marker, Shapes::ShapeRef.new(shape: DeleteMarker, location: "header", location_name: "x-amz-delete-marker"))
@@ -1135,6 +1146,7 @@ module Aws::S3
     InputSerialization.add_member(:csv, Shapes::ShapeRef.new(shape: CSVInput, location_name: "CSV"))
     InputSerialization.add_member(:compression_type, Shapes::ShapeRef.new(shape: CompressionType, location_name: "CompressionType"))
     InputSerialization.add_member(:json, Shapes::ShapeRef.new(shape: JSONInput, location_name: "JSON"))
+    InputSerialization.add_member(:parquet, Shapes::ShapeRef.new(shape: ParquetInput, location_name: "Parquet"))
     InputSerialization.struct_class = Types::InputSerialization
 
     InventoryConfiguration.add_member(:destination, Shapes::ShapeRef.new(shape: InventoryDestination, required: true, location_name: "Destination"))
@@ -1463,6 +1475,8 @@ module Aws::S3
     Owner.add_member(:id, Shapes::ShapeRef.new(shape: ID, location_name: "ID"))
     Owner.struct_class = Types::Owner
 
+    ParquetInput.struct_class = Types::ParquetInput
+
     Part.add_member(:part_number, Shapes::ShapeRef.new(shape: PartNumber, location_name: "PartNumber"))
     Part.add_member(:last_modified, Shapes::ShapeRef.new(shape: LastModified, location_name: "LastModified"))
     Part.add_member(:etag, Shapes::ShapeRef.new(shape: ETag, location_name: "ETag"))
@@ -1714,12 +1728,24 @@ module Aws::S3
     ReplicationConfiguration.struct_class = Types::ReplicationConfiguration
 
     ReplicationRule.add_member(:id, Shapes::ShapeRef.new(shape: ID, location_name: "ID"))
-    ReplicationRule.add_member(:prefix, Shapes::ShapeRef.new(shape: Prefix, required: true, location_name: "Prefix"))
+    ReplicationRule.add_member(:priority, Shapes::ShapeRef.new(shape: Priority, location_name: "Priority"))
+    ReplicationRule.add_member(:prefix, Shapes::ShapeRef.new(shape: Prefix, deprecated: true, location_name: "Prefix"))
+    ReplicationRule.add_member(:filter, Shapes::ShapeRef.new(shape: ReplicationRuleFilter, location_name: "Filter"))
     ReplicationRule.add_member(:status, Shapes::ShapeRef.new(shape: ReplicationRuleStatus, required: true, location_name: "Status"))
     ReplicationRule.add_member(:source_selection_criteria, Shapes::ShapeRef.new(shape: SourceSelectionCriteria, location_name: "SourceSelectionCriteria"))
     ReplicationRule.add_member(:destination, Shapes::ShapeRef.new(shape: Destination, required: true, location_name: "Destination"))
+    ReplicationRule.add_member(:delete_marker_replication, Shapes::ShapeRef.new(shape: DeleteMarkerReplication, location_name: "DeleteMarkerReplication"))
     ReplicationRule.struct_class = Types::ReplicationRule
 
+    ReplicationRuleAndOperator.add_member(:prefix, Shapes::ShapeRef.new(shape: Prefix, location_name: "Prefix"))
+    ReplicationRuleAndOperator.add_member(:tags, Shapes::ShapeRef.new(shape: TagSet, location_name: "Tag", metadata: {"flattened"=>true}))
+    ReplicationRuleAndOperator.struct_class = Types::ReplicationRuleAndOperator
+
+    ReplicationRuleFilter.add_member(:prefix, Shapes::ShapeRef.new(shape: Prefix, location_name: "Prefix"))
+    ReplicationRuleFilter.add_member(:tag, Shapes::ShapeRef.new(shape: Tag, location_name: "Tag"))
+    ReplicationRuleFilter.add_member(:and, Shapes::ShapeRef.new(shape: ReplicationRuleAndOperator, location_name: "And"))
+    ReplicationRuleFilter.struct_class = Types::ReplicationRuleFilter
+
     ReplicationRules.member = Shapes::ShapeRef.new(shape: ReplicationRule)
 
     RequestPaymentConfiguration.add_member(:payer, Shapes::ShapeRef.new(shape: Payer, required: true, location_name: "Payer"))
@@ -1957,10 +1983,15 @@ module Aws::S3
       api.version = "2006-03-01"
 
       api.metadata = {
+        "apiVersion" => "2006-03-01",
+        "checksumFormat" => "md5",
         "endpointPrefix" => "s3",
+        "globalEndpoint" => "s3.amazonaws.com",
         "protocol" => "rest-xml",
+        "serviceAbbreviation" => "Amazon S3",
         "serviceFullName" => "Amazon Simple Storage Service",
-        "timestampFormat" => "rfc822",
+        "serviceId" => "S3",
+        "uid" => "s3-2006-03-01",
       }
 
       api.add_operation(:abort_multipart_upload, Seahorse::Model::Operation.new.tap do |o|

data/lib/aws-sdk-s3/customizations/bucket.rb

@@ -61,8 +61,8 @@ module Aws
       # You can pass `virtual_host: true` to use the bucket name as the
       # host name.
       #
-      #     bucket = s3.bucket('my.bucket.com', virtual_host: true)
-      #     bucket.url
+      #     bucket = s3.bucket('my.bucket.com')
+      #     bucket.url(virtual_host: true)
       #     #=> "http://my.bucket.com"
       #
       # @option options [Boolean] :virtual_host (false) When `true`,

data/lib/aws-sdk-s3/customizations/object.rb

@@ -214,6 +214,62 @@ module Aws
         url.to_s
       end
 
+      # Uploads a stream in a streaming fashion to the current object in S3.
+      #
+      #     # Passed chunks automatically split into multipart upload parts
+      #     # and the parts are uploaded in parallel. This allows for streaming uploads
+      #     # that never touch the disk.
+      #
+      #  Note that this is known to have issues in JRuby until jruby-9.1.15.0, so avoid using this with older versions of JRuby.
+      #
+      # @example Streaming chunks of data
+      #     obj.upload_stream do |write_stream|
+      #       10.times { write_stream << 'foo' }
+      #     end
+      # @example Streaming chunks of data
+      #     obj.upload_stream do |write_stream|
+      #       IO.copy_stream(IO.popen('ls'), write_stream)
+      #     end
+      # @example Streaming chunks of data
+      #     obj.upload_stream do |write_stream|
+      #       IO.copy_stream(STDIN, write_stream)
+      #     end
+      #
+      # @option options [Integer] :thread_count
+      #   The number of parallel multipart uploads
+      #   Default `:thread_count` is `10`.
+      #
+      # @option options [Boolean] :tempfile
+      #   Normally read data is stored in memory when building the parts in order to complete
+      #   the underlying multipart upload. By passing `:tempfile => true` data read will be
+      #   temporarily stored on disk reducing the memory footprint vastly.
+      #   Default `:tempfile` is `false`.
+      #
+      # @option options [Integer] :part_size
+      #   Define how big each part size but the last should be.
+      #   Default `:part_size` is `5 * 1024 * 1024`.
+      #
+      # @raise [MultipartUploadError] If an object is being uploaded in
+      #   parts, and the upload can not be completed, then the upload is
+      #   aborted and this error is raised.  The raised error has a `#errors`
+      #   method that returns the failures that caused the upload to be
+      #   aborted.
+      #
+      # @return [Boolean] Returns `true` when the object is uploaded
+      #   without any errors.
+      #
+      def upload_stream(options = {}, &block)
+        uploading_options = options.dup
+        uploader = MultipartStreamUploader.new(
+          client: client,
+          thread_count: uploading_options.delete(:thread_count),
+          tempfile: uploading_options.delete(:tempfile),
+          part_size: uploading_options.delete(:part_size),
+        )
+        uploader.upload(uploading_options.merge(bucket: bucket_name, key: key), &block)
+        true
+      end
+
       # Uploads a file from disk to the current object in S3.
       #
       #     # small files are uploaded in a single API call
@@ -277,6 +333,10 @@ module Aws
       # @option options [String] thread_count Customize threads used in multipart
       #   download, if not provided, 10 is default value
       #
+      # @option options [String] version_id The object version id used to retrieve
+      #   the object, to know more about object versioning, see:
+      #   https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
+      #
       # @return [Boolean] Returns `true` when the file is downloaded
       #   without any errors.
       def download_file(destination, options = {})

data/lib/aws-sdk-s3/customizations/object_summary.rb

@@ -60,6 +60,13 @@ module Aws
         object.upload_file(source, options)
       end
 
+      # @options (see Object#upload_stream)
+      # @return (see Object#upload_stream)
+      # @see Object#upload_stream
+      def upload_stream(options = {}, &block)
+        object.upload_stream(options, &block)
+      end
+
       # @param (see Object#download_file)
       # @options (see Object#download_file)
       # @return (see Object#download_file)

data/lib/aws-sdk-s3/customizations.rb

@@ -6,6 +6,7 @@ require 'aws-sdk-s3/file_uploader'
 require 'aws-sdk-s3/file_downloader'
 require 'aws-sdk-s3/legacy_signer'
 require 'aws-sdk-s3/multipart_file_uploader'
+require 'aws-sdk-s3/multipart_stream_uploader'
 require 'aws-sdk-s3/multipart_upload_error'
 require 'aws-sdk-s3/object_copier'
 require 'aws-sdk-s3/object_multipart_copier'

data/lib/aws-sdk-s3/encryption/io_auth_decrypter.rb

@@ -22,8 +22,10 @@ module Aws
 
         def write(chunk)
           chunk = truncate_chunk(chunk)
-          @bytes_written += chunk.bytesize
-          @decrypter.write(chunk)
+          if chunk.bytesize > 0
+            @bytes_written += chunk.bytesize
+            @decrypter.write(chunk)
+          end
         end
 
         def finalize
@@ -39,8 +41,12 @@ module Aws
         def truncate_chunk(chunk)
           if chunk.bytesize + @bytes_written <= @max_bytes
             chunk
-          else
+          elsif @bytes_written < @max_bytes
             chunk[0..(@max_bytes - @bytes_written - 1)]
+          else
+            # If the tag was sent over after the full body has been read,
+            # we don't want to accidentally append it.
+            ""
           end
         end
 

data/lib/aws-sdk-s3/file_downloader.rb

@@ -24,15 +24,18 @@ module Aws
         @mode = options[:mode] || "auto"
         @thread_count = options[:thread_count] || THREAD_COUNT
         @chunk_size = options[:chunk_size]
-        @bucket = options[:bucket]
-        @key = options[:key]
+        @params = {
+          bucket: options[:bucket],
+          key: options[:key],
+        }
+        @params[:version_id] = options[:version_id] if options[:version_id]
 
         case @mode
         when "auto" then multipart_download
         when "single_request" then single_request
         when "get_range"
           if @chunk_size
-            resp = @client.head_object(bucket: @bucket, key: @key)
+            resp = @client.head_object(@params)
             multithreaded_get_by_ranges(construct_chunks(resp.content_length))
           else
             msg = "In :get_range mode, :chunk_size must be provided"
@@ -48,7 +51,7 @@ module Aws
       private
 
       def multipart_download
-        resp = @client.head_object(bucket: @bucket, key: @key, part_number: 1)
+        resp = @client.head_object(@params.merge(part_number: 1))
         count = resp.parts_count
         if count.nil? || count <= 1
           resp.content_length < MIN_CHUNK_SIZE ?
@@ -56,7 +59,7 @@ module Aws
             multithreaded_get_by_ranges(construct_chunks(resp.content_length))
         else
           # partNumber is an option
-          resp = @client.head_object(bucket: @bucket, key: @key)
+          resp = @client.head_object(@params)
           resp.content_length < MIN_CHUNK_SIZE ?
             single_request :
             compute_mode(resp.content_length, count)
@@ -112,9 +115,7 @@ module Aws
           batch.each do |chunk|
             threads << Thread.new do
               resp = @client.get_object(
-                :bucket => @bucket,
-                :key => @key,
-                param.to_sym => chunk
+                @params.merge(param.to_sym => chunk)
               )
               write(resp)
             end
@@ -131,7 +132,7 @@ module Aws
 
       def single_request
         @client.get_object(
-          bucket: @bucket, key: @key, response_target: @path
+          @params.merge(response_target: @path)
         )
       end
     end

data/lib/aws-sdk-s3/file_part.rb

@@ -56,14 +56,12 @@ module Aws
       end
 
       def read_from_file(bytes, output_buffer)
-        if bytes
-          data = @file.read([remaining_bytes, bytes].min)
-          data = nil if data == ''
-        else
-          data = @file.read(remaining_bytes)
-        end
+        length = [remaining_bytes, *bytes].min
+        data   = @file.read(length, output_buffer)
+
         @position += data ? data.bytesize : 0
-        output_buffer ? output_buffer.replace(data || '') : data
+
+        data.to_s unless bytes && (data.nil? || data.empty?)
       end
 
       def remaining_bytes

data/lib/aws-sdk-s3/file_uploader.rb

@@ -45,9 +45,7 @@ module Aws
 
       def open_file(source)
         if String === source || Pathname === source
-          file = File.open(source, 'rb')
-          yield(file)
-          file.close
+          File.open(source, 'rb') { |file| yield(file) }
         else
           yield(source)
         end

data/lib/aws-sdk-s3/multipart_stream_uploader.rb

@@ -0,0 +1,160 @@
+require 'thread'
+require 'set'
+require 'tempfile'
+require 'stringio'
+
+module Aws
+  module S3
+    # @api private
+    class MultipartStreamUploader
+      # api private
+      PART_SIZE = 5 * 1024 * 1024 # 5MB
+
+      # api private
+      THREAD_COUNT = 10
+
+      # api private
+      TEMPFILE_PREIX = 'aws-sdk-s3-upload_stream'.freeze
+
+      # @api private
+      CREATE_OPTIONS =
+        Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names)
+
+      # @api private
+      UPLOAD_PART_OPTIONS =
+        Set.new(Client.api.operation(:upload_part).input.shape.member_names)
+
+      # @option options [Client] :client
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+        @tempfile = options[:tempfile]
+        @part_size = options[:part_size] || PART_SIZE
+        @thread_count = options[:thread_count] || THREAD_COUNT
+      end
+
+      # @return [Client]
+      attr_reader :client
+
+      # @option options [required,String] :bucket
+      # @option options [required,String] :key
+      # @return [void]
+      def upload(options = {}, &block)
+        upload_id = initiate_upload(options)
+        parts = upload_parts(upload_id, options, &block)
+        complete_upload(upload_id, parts, options)
+      end
+
+      private
+
+      def initiate_upload(options)
+        @client.create_multipart_upload(create_opts(options)).upload_id
+      end
+
+      def complete_upload(upload_id, parts, options)
+        @client.complete_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id,
+          multipart_upload: { parts: parts })
+      end
+
+      def upload_parts(upload_id, options, &block)
+        completed = Queue.new
+        errors = IO.pipe do |read_pipe, write_pipe|
+          threads = upload_in_threads(read_pipe, completed, upload_part_opts(options).merge(upload_id: upload_id))
+          block.call(write_pipe)
+          write_pipe.close
+          threads.map(&:value).compact
+        end
+        if errors.empty?
+          Array.new(completed.size) { completed.pop }.sort_by { |part| part[:part_number] }
+        else
+          abort_upload(upload_id, options, errors)
+        end
+      end
+
+      def abort_upload(upload_id, options, errors)
+        @client.abort_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id
+        )
+        msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
+        raise MultipartUploadError.new(msg, errors)
+      rescue MultipartUploadError => error
+        raise error
+      rescue => error
+        msg = "failed to abort multipart upload: #{error.message}"
+        raise MultipartUploadError.new(msg, errors + [error])
+      end
+
+      def create_opts(options)
+        CREATE_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def upload_part_opts(options)
+        UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
+      def read_to_part_body(read_pipe)
+        return if read_pipe.closed?
+        temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new
+        temp_io.binmode
+        bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size)
+        temp_io.rewind
+        if bytes_copied == 0
+          if Tempfile === temp_io
+            temp_io.close
+            temp_io.unlink
+          end
+          nil
+        else
+          temp_io
+        end
+      end
+
+      def upload_in_threads(read_pipe, completed, options)
+        mutex = Mutex.new
+        part_number = 0
+        @thread_count.times.map do
+          thread = Thread.new do
+            begin
+              loop do
+                body, thread_part_number = mutex.synchronize do
+                  [read_to_part_body(read_pipe), part_number += 1]
+                end
+                break unless (body || thread_part_number == 1)
+                begin
+                  part = options.merge(
+                    body: body,
+                    part_number: thread_part_number,
+                  )
+                  resp = @client.upload_part(part)
+                  completed << {etag: resp.etag, part_number: part[:part_number]}
+                ensure
+                  if Tempfile === body
+                    body.close
+                    body.unlink
+                  end
+                end
+              end
+              nil
+            rescue => error
+              # keep other threads from uploading other parts
+              mutex.synchronize { read_pipe.close_read }
+              error
+            end
+          end
+          thread.abort_on_exception = true
+          thread
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/object.rb

@@ -898,11 +898,14 @@ module Aws::S3
     #             record_delimiter: "RecordDelimiter",
     #             field_delimiter: "FieldDelimiter",
     #             quote_character: "QuoteCharacter",
+    #             allow_quoted_record_delimiter: false,
     #           },
-    #           compression_type: "NONE", # accepts NONE, GZIP
+    #           compression_type: "NONE", # accepts NONE, GZIP, BZIP2
     #           json: {
     #             type: "DOCUMENT", # accepts DOCUMENT, LINES
     #           },
+    #           parquet: {
+    #           },
     #         },
     #         expression_type: "SQL", # required, accepts SQL
     #         expression: "Expression", # required

data/lib/aws-sdk-s3/object_summary.rb

@@ -741,11 +741,14 @@ module Aws::S3
     #             record_delimiter: "RecordDelimiter",
     #             field_delimiter: "FieldDelimiter",
     #             quote_character: "QuoteCharacter",
+    #             allow_quoted_record_delimiter: false,
     #           },
-    #           compression_type: "NONE", # accepts NONE, GZIP
+    #           compression_type: "NONE", # accepts NONE, GZIP, BZIP2
     #           json: {
     #             type: "DOCUMENT", # accepts DOCUMENT, LINES
     #           },
+    #           parquet: {
+    #           },
     #         },
     #         expression_type: "SQL", # required, accepts SQL
     #         expression: "Expression", # required

data/lib/aws-sdk-s3/plugins/md5s.rb

@@ -45,7 +45,7 @@ module Aws
           end
 
           def update_in_chunks(digest, io)
-            while chunk = io.read(CHUNK_SIZE)
+            while chunk = io.read(CHUNK_SIZE, buffer ||= "")
               digest.update(chunk)
             end
             io.rewind

data/lib/aws-sdk-s3/plugins/redirects.rb

@@ -18,7 +18,9 @@ by Amazon S3.
             response = @handler.call(context)
             if context.http_response.status_code == 307
               endpoint = context.http_response.headers['location']
-              context.http_request.endpoint = endpoint
+              unless context.http_request.endpoint.host.include?('fips')
+                context.http_request.endpoint = endpoint
+              end
               context.http_response.body.truncate(0)
               @handler.call(context)
             else

data/lib/aws-sdk-s3/plugins/s3_signer.rb

@@ -113,7 +113,7 @@ module Aws
           private
 
           def handle_region_errors(response)
-            if wrong_sigv4_region?(response)
+            if wrong_sigv4_region?(response) && !fips_region?(response)
               get_region_and_retry(response.context)
             else
               response
@@ -133,6 +133,10 @@ module Aws
             S3::BUCKET_REGIONS[context.params[:bucket]] = actual_region
           end
 
+          def fips_region?(resp)
+            resp.context.http_request.endpoint.host.include?('fips')
+          end
+
           def wrong_sigv4_region?(resp)
             resp.context.http_response.status_code == 400 &&
             (

data/lib/aws-sdk-s3/presigner.rb

@@ -98,6 +98,7 @@ module Aws
         req.handlers.remove(Seahorse::Client::Plugins::ContentLength::Handler)
 
         signer = build_signer(req.context.config)
+        req.context[:presigned_url] = true
 
         req.handle(step: :send) do |context|