aws-sdk-s3 1.69.1 → 1.74.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71499a1c36cb061a8fb6b48a61fba9eec3985149a0e3effb810f8bf45e89f8db
-  data.tar.gz: 05e2a7de980f2c579b09688d1301326ce8a5df60bb9f3487a0ff21486afb58e6
+  metadata.gz: 626495d85d0f33f1a04615e2c65aca4331f8ac04b35306d2e68668eb1cbad727
+  data.tar.gz: 324b43984913dedda5bd63af0608eb9ba01feab28e8ff15ee5ceb7b1cb5d5578
 SHA512:
-  metadata.gz: 522caf62fdf5130b9e8f19da2c06841074ea7da04c15c9e2731992cadc9fcd7372a0705db07902bd672ea48d7b47703876f0f679c402330f13fb33120f6e4478
-  data.tar.gz: 977b14327746226f9abb50cc7984f23abfa6526b6a3b10360c5e1827d4b38166c827290ca416e109a58bebf5debecdb4effc7dbc316c22e921a004720cad8e63
+  metadata.gz: f4958752f8508e3d4ecdb0ad22bc5184f3c9e8c9c6e035bfcb5cabda3897fb8fec2b3da206499689f400c001e83e1d364daeea687bff71cbcc1b910779936058
+  data.tar.gz: ef54aeb2dc6373afe0b806a4e88ca85ed81eedba190da89c7370d10eea5535f6f608ba80539b7c5b31f3198ab64767afda81c8560475c9268155af373e1d23ff

data/lib/aws-sdk-s3.rb

@@ -68,6 +68,6 @@ require_relative 'aws-sdk-s3/event_streams'
 # @service
 module Aws::S3
 
-  GEM_VERSION = '1.69.1'
+  GEM_VERSION = '1.74.0'
 
 end

data/lib/aws-sdk-s3/client.rb

@@ -44,6 +44,7 @@ require 'aws-sdk-s3/plugins/sse_cpk.rb'
 require 'aws-sdk-s3/plugins/url_encoded_keys.rb'
 require 'aws-sdk-s3/plugins/s3_signer.rb'
 require 'aws-sdk-s3/plugins/bucket_name_restrictions.rb'
+require 'aws-sdk-s3/plugins/streaming_retry.rb'
 require 'aws-sdk-core/plugins/event_stream_configuration.rb'
 
 Aws::Plugins::GlobalConfiguration.add_identifier(:s3)
@@ -106,6 +107,7 @@ module Aws::S3
     add_plugin(Aws::S3::Plugins::UrlEncodedKeys)
     add_plugin(Aws::S3::Plugins::S3Signer)
     add_plugin(Aws::S3::Plugins::BucketNameRestrictions)
+    add_plugin(Aws::S3::Plugins::StreamingRetry)
     add_plugin(Aws::Plugins::EventStreamConfiguration)
 
     # @overload initialize(options)
@@ -333,7 +335,7 @@ module Aws::S3
     #   @option options [Boolean] :use_accelerate_endpoint (false)
     #     When set to `true`, accelerated bucket endpoints will be used
     #     for all object operations. You must first enable accelerate for
-    #     each bucket.  [Go here for more information](http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html).
+    #     each bucket. [Go here for more information](http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html).
     #
     #   @option options [Boolean] :use_dualstack_endpoint (false)
     #     When set to `true`, IPv6-compatible bucket endpoints will be used
@@ -11670,7 +11672,7 @@ module Aws::S3
         params: params,
         config: config)
       context[:gem_name] = 'aws-sdk-s3'
-      context[:gem_version] = '1.69.1'
+      context[:gem_version] = '1.74.0'
       Seahorse::Client::Request.new(handlers, context)
     end
 

data/lib/aws-sdk-s3/encryptionV2/io_decrypter.rb

@@ -10,6 +10,7 @@ module Aws
           @cipher = cipher
           # Ensure that IO is reset between retries
           @io = io.tap { |io| io.truncate(0) if io.respond_to?(:truncate) }
+          @cipher_buffer = String.new
         end
 
         # @return [#write]
@@ -17,7 +18,11 @@ module Aws
 
         def write(chunk)
           # decrypt and write
-          @io.write(@cipher.update(chunk))
+          if @cipher.method(:update).arity == 1
+            @io.write(@cipher.update(chunk))
+          else
+            @io.write(@cipher.update(chunk, @cipher_buffer))
+          end
         end
 
         def finalize

data/lib/aws-sdk-s3/encryptionV2/io_encrypter.rb

@@ -52,8 +52,12 @@ module Aws
         def encrypt_to_tempfile(cipher, io)
           encrypted = Tempfile.new(self.object_id.to_s)
           encrypted.binmode
-          while chunk = io.read(ONE_MEGABYTE)
-            encrypted.write(cipher.update(chunk))
+          while chunk = io.read(ONE_MEGABYTE, read_buffer ||= String.new)
+            if cipher.method(:update).arity == 1
+              encrypted.write(cipher.update(chunk))
+            else
+              encrypted.write(cipher.update(chunk, cipher_buffer ||= String.new))
+            end
           end
           encrypted.write(cipher.final)
           encrypted.write(cipher.auth_tag)

data/lib/aws-sdk-s3/plugins/accelerate.rb

@@ -3,36 +3,44 @@
 module Aws
   module S3
     module Plugins
-
       # Provides support for using `Aws::S3::Client` with Amazon S3 Transfer
       # Acceleration.
       #
       # Go here for more information about transfer acceleration:
       # [http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html](http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html)
       class Accelerate < Seahorse::Client::Plugin
-
-        option(:use_accelerate_endpoint,
+        option(
+          :use_accelerate_endpoint,
           default: false,
           doc_type: 'Boolean',
           docstring: <<-DOCS)
 When set to `true`, accelerated bucket endpoints will be used
 for all object operations. You must first enable accelerate for
-each bucket.  [Go here for more information](http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html).
+each bucket. [Go here for more information](http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html).
           DOCS
 
         def add_handlers(handlers, config)
           operations = config.api.operation_names - [
-            :create_bucket, :list_buckets, :delete_bucket,
+            :create_bucket, :list_buckets, :delete_bucket
           ]
-          handlers.add(OptionHandler, step: :initialize, operations: operations)
-          handlers.add(AccelerateHandler, step: :build, priority: 0, operations: operations)
+          # Need 2 handlers so that the context can be set for other plugins
+          # and to remove :use_accelerate_endpoint from the params.
+          handlers.add(
+            OptionHandler, step: :initialize, operations: operations
+          )
+          handlers.add(
+            AccelerateHandler, step: :build, priority: 0, operations: operations
+          )
         end
 
         # @api private
         class OptionHandler < Seahorse::Client::Handler
           def call(context)
+            # Support client configuration and per-operation configuration
             accelerate = context.params.delete(:use_accelerate_endpoint)
-            accelerate = context.config.use_accelerate_endpoint if accelerate.nil?
+            if accelerate.nil?
+              accelerate = context.config.use_accelerate_endpoint
+            end
             context[:use_accelerate_endpoint] = accelerate
             @handler.call(context)
           end
@@ -40,39 +48,24 @@ each bucket.  [Go here for more information](http://docs.aws.amazon.com/AmazonS3
 
         # @api private
         class AccelerateHandler < Seahorse::Client::Handler
-
           def call(context)
             if context[:use_accelerate_endpoint]
-              if context[:use_dualstack_endpoint]
-                use_combined_accelerate_dualstack_endpoint(context)
-              else
-                use_accelerate_endpoint(context)
-              end
+              dualstack = !!context[:use_dualstack_endpoint]
+              use_accelerate_endpoint(context, dualstack)
             end
             @handler.call(context)
           end
 
           private
 
-          def use_accelerate_endpoint(context)
-            bucket_name = context.params[:bucket]
-            validate_bucket_name!(bucket_name)
-            endpoint = URI.parse(context.http_request.endpoint.to_s)
-            endpoint.scheme = 'https'
-            endpoint.port = 443
-            endpoint.host = "#{bucket_name}.s3-accelerate.amazonaws.com"
-            context.http_request.endpoint = endpoint.to_s
-            # s3 accelerate endpoint doesn't work with 'expect' header
-            context.http_request.headers.delete('expect')
-          end
-
-          def use_combined_accelerate_dualstack_endpoint(context)
+          def use_accelerate_endpoint(context, dualstack)
             bucket_name = context.params[:bucket]
             validate_bucket_name!(bucket_name)
             endpoint = URI.parse(context.http_request.endpoint.to_s)
             endpoint.scheme = 'https'
             endpoint.port = 443
-            endpoint.host = "#{bucket_name}.s3-accelerate.dualstack.amazonaws.com"
+            endpoint.host = "#{bucket_name}.s3-accelerate"\
+                            "#{'.dualstack' if dualstack}.amazonaws.com"
             context.http_request.endpoint = endpoint.to_s
             # s3 accelerate endpoint doesn't work with 'expect' header
             context.http_request.headers.delete('expect')
@@ -80,17 +73,11 @@ each bucket.  [Go here for more information](http://docs.aws.amazon.com/AmazonS3
 
           def validate_bucket_name!(bucket_name)
             unless BucketDns.dns_compatible?(bucket_name, _ssl = true)
-              msg = 'unable to use `accelerate: true` on buckets with '\
-                    'non-DNS compatible names'
-              raise ArgumentError, msg
-            end
-            if bucket_name.include?('.')
-              msg = 'unable to use `accelerate: true` on buckets with dots'\
-                    "in their name: #{bucket_name.inspect}"
-              raise ArgumentError, msg
+              raise ArgumentError,
+                    'Unable to use `use_accelerate_endpoint: true` on buckets '\
+                    'with non-DNS compatible names.'
             end
           end
-
         end
       end
     end

data/lib/aws-sdk-s3/plugins/http_200_errors.rb

@@ -40,7 +40,8 @@ module Aws
           end
         end
 
-        handler(Handler,
+        handler(
+          Handler,
           step: :sign,
           operations: [
             :complete_multipart_upload,

data/lib/aws-sdk-s3/plugins/streaming_retry.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Aws
+  module S3
+    module Plugins
+
+      # A wrapper around BlockIO that adds no-ops for truncate and rewind
+      # @api private
+      class RetryableBlockIO
+        extend Forwardable
+        def_delegators :@block_io, :write, :read, :size
+
+        def initialize(block_io)
+          @block_io = block_io
+        end
+
+        def truncate(_integer); end
+
+        def rewind; end
+      end
+
+      # A wrapper around ManagedFile that adds no-ops for truncate and rewind
+      # @api private
+      class RetryableManagedFile
+        extend Forwardable
+        def_delegators :@file, :write, :read, :size, :open?, :close
+
+        def initialize(managed_file)
+          @file = managed_file
+        end
+
+        def truncate(_integer); end
+
+        def rewind; end
+      end
+
+      # This handler works with the ResponseTarget plugin to provide smart
+      # retries of S3 streaming operations that support the range parameter
+      # (currently only: get_object).  When a 200 OK with a TruncatedBodyError
+      # is received this handler will add a range header that excludes the
+      # data that has already been processed (written to file or sent to
+      # the target Proc).
+      # It is important to not write data to the custom target in the case of
+      # a non-success response. We do not want to write an XML error
+      # message to someone's file or pass it to a user's Proc.
+      # @api private
+      class StreamingRetry < Seahorse::Client::Plugin
+
+        class Handler < Seahorse::Client::Handler
+
+          def call(context)
+            target = context.params[:response_target] || context[:response_target]
+
+            # retry is only supported when range is NOT set on the initial request
+            if supported_target?(target) && !context.params[:range]
+              add_event_listeners(context, target)
+            end
+            @handler.call(context)
+          end
+
+          private
+
+          def add_event_listeners(context, target)
+            context.http_response.on_headers(200..299) do
+              case context.http_response.body
+              when Seahorse::Client::BlockIO then
+                context.http_response.body = RetryableBlockIO.new(context.http_response.body)
+              when Seahorse::Client::ManagedFile then
+                context.http_response.body = RetryableManagedFile.new(context.http_response.body)
+              end
+            end
+
+            context.http_response.on_headers(400..599) do
+              context.http_response.body = StringIO.new # something to write the error to
+            end
+
+            context.http_response.on_success(200..299) do
+              body = context.http_response.body
+              if body.is_a?(RetryableManagedFile) && body.open?
+                body.close
+              end
+            end
+
+            context.http_response.on_error do |error|
+              if retryable_body?(context) && truncated_body?(error)
+                context.http_request.headers[:range] = "bytes=#{context.http_response.body.size}-"
+              end
+            end
+          end
+
+          def truncated_body?(error)
+            error.is_a?(Seahorse::Client::NetworkingError) &&
+              error.original_error.is_a?(
+                Seahorse::Client::NetHttp::Handler::TruncatedBodyError
+              )
+          end
+
+          def retryable_body?(context)
+            context.http_response.body.is_a?(RetryableBlockIO) ||
+              context.http_response.body.is_a?(RetryableManagedFile)
+          end
+
+          def supported_target?(target)
+            case target
+            when Proc, String, Pathname then true
+            else false
+            end
+          end
+        end
+
+        handler(Handler, step: :sign, operations: [:get_object], priority: 10)
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/presigned_post.rb

@@ -185,35 +185,58 @@ module Aws
       #   the post will expire. Defaults to one hour from creation of the
       #   presigned post. May not exceed one week from creation time.
       # @option options [String] :key See {PresignedPost#key}.
-      # @option options [String] :key_starts_with See {PresignedPost#key_starts_with}.
+      # @option options [String] :key_starts_with
+      #   See {PresignedPost#key_starts_with}.
       # @option options [String] :acl See {PresignedPost#acl}.
-      # @option options [String] :acl_starts_with See {PresignedPost#acl_starts_with}.
-      # @option options [String] :cache_control See {PresignedPost#cache_control}.
-      # @option options [String] :cache_control_starts_with See {PresignedPost#cache_control_starts_with}.
+      # @option options [String] :acl_starts_with
+      #   See {PresignedPost#acl_starts_with}.
+      # @option options [String] :cache_control
+      #   See {PresignedPost#cache_control}.
+      # @option options [String] :cache_control_starts_with
+      #   See {PresignedPost#cache_control_starts_with}.
       # @option options [String] :content_type See {PresignedPost#content_type}.
-      # @option options [String] :content_type_starts_with See {PresignedPost#content_type_starts_with}.
-      # @option options [String] :content_disposition See {PresignedPost#content_disposition}.
-      # @option options [String] :content_disposition_starts_with See {PresignedPost#content_disposition_starts_with}.
-      # @option options [String] :content_encoding See {PresignedPost#content_encoding}.
-      # @option options [String] :content_encoding_starts_with See {PresignedPost#content_encoding_starts_with}.
+      # @option options [String] :content_type_starts_with
+      #   See {PresignedPost#content_type_starts_with}.
+      # @option options [String] :content_disposition
+      #   See {PresignedPost#content_disposition}.
+      # @option options [String] :content_disposition_starts_with
+      #   See {PresignedPost#content_disposition_starts_with}.
+      # @option options [String] :content_encoding
+      #   See {PresignedPost#content_encoding}.
+      # @option options [String] :content_encoding_starts_with
+      #   See {PresignedPost#content_encoding_starts_with}.
       # @option options [String] :expires See {PresignedPost#expires}.
-      # @option options [String] :expires_starts_with See {PresignedPost#expires_starts_with}.
-      # @option options [Range<Integer>] :content_length_range See {PresignedPost#content_length_range}.
-      # @option options [String] :success_action_redirect See {PresignedPost#success_action_redirect}.
-      # @option options [String] :success_action_redirect_starts_with See {PresignedPost#success_action_redirect_starts_with}.
-      # @option options [String] :success_action_status See {PresignedPost#success_action_status}.
-      # @option options [String] :storage_class See {PresignedPost#storage_class}.
-      # @option options [String] :website_redirect_location See {PresignedPost#website_redirect_location}.
-      # @option options [Hash<String,String>] :metadata See {PresignedPost#metadata}.
-      # @option options [Hash<String,String>] :metadata_starts_with See {PresignedPost#metadata_starts_with}.
-      # @option options [String] :server_side_encryption See {PresignedPost#server_side_encryption}.
-      # @option options [String] :server_side_encryption_aws_kms_key_id See {PresignedPost#server_side_encryption_aws_kms_key_id}.
-      # @option options [String] :server_side_encryption_customer_algorithm See {PresignedPost#server_side_encryption_customer_algorithm}.
-      # @option options [String] :server_side_encryption_customer_key See {PresignedPost#server_side_encryption_customer_key}.
+      # @option options [String] :expires_starts_with
+      #   See {PresignedPost#expires_starts_with}.
+      # @option options [Range<Integer>] :content_length_range
+      #   See {PresignedPost#content_length_range}.
+      # @option options [String] :success_action_redirect
+      #   See {PresignedPost#success_action_redirect}.
+      # @option options [String] :success_action_redirect_starts_with
+      #   See {PresignedPost#success_action_redirect_starts_with}.
+      # @option options [String] :success_action_status
+      #   See {PresignedPost#success_action_status}.
+      # @option options [String] :storage_class
+      #   See {PresignedPost#storage_class}.
+      # @option options [String] :website_redirect_location
+      #   See {PresignedPost#website_redirect_location}.
+      # @option options [Hash<String,String>] :metadata
+      #   See {PresignedPost#metadata}.
+      # @option options [Hash<String,String>] :metadata_starts_with
+      #   See {PresignedPost#metadata_starts_with}.
+      # @option options [String] :server_side_encryption
+      #   See {PresignedPost#server_side_encryption}.
+      # @option options [String] :server_side_encryption_aws_kms_key_id
+      #   See {PresignedPost#server_side_encryption_aws_kms_key_id}.
+      # @option options [String] :server_side_encryption_customer_algorithm
+      #   See {PresignedPost#server_side_encryption_customer_algorithm}.
+      # @option options [String] :server_side_encryption_customer_key
+      #   See {PresignedPost#server_side_encryption_customer_key}.
       def initialize(credentials, bucket_region, bucket_name, options = {})
         @credentials = credentials.credentials
         @bucket_region = bucket_region
         @bucket_name = bucket_name
+        @accelerate = !!options.delete(:use_accelerate_endpoint)
         @url = options.delete(:url) || bucket_url
         @fields = {}
         @key_set = false
@@ -272,7 +295,7 @@ module Aws
 
       # @!group Fields
 
-      # The key to use for the uploaded object. Use can use `${filename}`
+      # The key to use for the uploaded object. You can use `${filename}`
       # as a variable in the key. This will be replaced with the name
       # of the file as provided by the user.
       #
@@ -507,7 +530,10 @@ module Aws
       #   (KMS) master encryption key to use for the object.
       #   @param [String] value
       #   @return [self]
-      define_field(:server_side_encryption_aws_kms_key_id, 'x-amz-server-side-encryption-aws-kms-key-id')
+      define_field(
+        :server_side_encryption_aws_kms_key_id,
+        'x-amz-server-side-encryption-aws-kms-key-id'
+      )
 
       # @!endgroup
 
@@ -520,7 +546,10 @@ module Aws
       #   @param [String] value
       #   @see #server_side_encryption_customer_key
       #   @return [self]
-      define_field(:server_side_encryption_customer_algorithm, 'x-amz-server-side-encryption-customer-algorithm')
+      define_field(
+        :server_side_encryption_customer_algorithm,
+        'x-amz-server-side-encryption-customer-algorithm'
+      )
 
       # Specifies the customer-provided encryption key for Amazon S3 to use
       # in encrypting data. This value is used to store the object and then
@@ -582,10 +611,14 @@ module Aws
       def bucket_url
         url = Aws::Partitions::EndpointProvider.resolve(@bucket_region, 's3')
         url = URI.parse(url)
-        if Plugins::BucketDns.dns_compatible?(@bucket_name, true)
-          url.host = @bucket_name + '.' + url.host
+        if Plugins::BucketDns.dns_compatible?(@bucket_name, _ssl = true)
+          if @accelerate
+            url.host = "#{@bucket_name}.s3-accelerate.amazonaws.com"
+          else
+            url.host = "#{@bucket_name}.#{url.host}"
+          end
         else
-          url.path = '/' + @bucket_name
+          url.path = "/#{@bucket_name}"
         end
         if @bucket_region == 'us-east-1'
           # keep legacy behavior by default