aws-sdk-s3 1.81.1 → 1.114.0

data/lib/aws-sdk-s3/plugins/expect_100_continue.rb

@@ -15,7 +15,8 @@ module Aws
         class Handler < Seahorse::Client::Handler
 
           def call(context)
-            if context.http_request.body && context.http_request.body.size > 0
+            body = context.http_request.body
+            if body.respond_to?(:size) && body.size > 0
               context.http_request.headers['expect'] = '100-continue'
             end
             @handler.call(context)

data/lib/aws-sdk-s3/plugins/get_bucket_location_fix.rb

@@ -11,7 +11,7 @@ module Aws
             @handler.call(context).on(200) do |response|
               response.data = S3::Types::GetBucketLocationOutput.new
               xml = context.http_response.body_contents
-              matches = xml.match(/>(.+?)<\/LocationConstraint>/)
+              matches = xml.match(/<LocationConstraint.*?>(.+?)<\/LocationConstraint>/)
               response.data[:location_constraint] = matches ? matches[1] : ''
             end
           end

data/lib/aws-sdk-s3/plugins/iad_regional_endpoint.rb

@@ -10,14 +10,15 @@ module Aws
           default: 'legacy',
           doc_type: String,
           docstring: <<-DOCS) do |cfg|
-Passing in `regional` to enable regional endpoint for S3's `us-east-1`
-region. Defaults to `legacy` mode using global endpoint.
+Pass in `regional` to enable the `us-east-1` regional endpoint.
+Defaults to `legacy` mode which uses the global endpoint.
           DOCS
           resolve_iad_regional_endpoint(cfg)
         end
 
         def add_handlers(handlers, config)
-          if config.region == 'us-east-1'
+          # only modify non-custom endpoints
+          if config.regional_endpoint && config.region == 'us-east-1'
             handlers.add(Handler)
           end
         end
@@ -26,10 +27,14 @@ region. Defaults to `legacy` mode using global endpoint.
         class Handler < Seahorse::Client::Handler
 
           def call(context)
-            # keep legacy global endpoint pattern by default
-            if context.config.s3_us_east_1_regional_endpoint == 'legacy'
-              context.http_request.endpoint.host = IADRegionalEndpoint.legacy_host(
-                context.http_request.endpoint.host)
+            # WriteGetObjectResponse does not have a global endpoint
+            # ARNs are regionalized, so don't touch those either.
+            if context.operation.name != 'WriteGetObjectResponse' &&
+               context.config.s3_us_east_1_regional_endpoint == 'legacy' &&
+               !context.metadata[:s3_arn]
+              host = context.http_request.endpoint.host
+              legacy_host = IADRegionalEndpoint.legacy_host(host)
+              context.http_request.endpoint.host = legacy_host
             end
             @handler.call(context)
           end
@@ -43,8 +48,14 @@ region. Defaults to `legacy` mode using global endpoint.
         private
 
         def self.resolve_iad_regional_endpoint(cfg)
+          default_mode_value =
+            if cfg.respond_to?(:defaults_mode_config_resolver)
+              cfg.defaults_mode_config_resolver.resolve(:s3_us_east_1_regional_endpoint)
+            end
+
           mode = ENV['AWS_S3_US_EAST_1_REGIONAL_ENDPOINT'] ||
             Aws.shared_config.s3_us_east_1_regional_endpoint(profile: cfg.profile) ||
+            default_mode_value ||
             'legacy'
           mode = mode.downcase
           unless %w(legacy regional).include?(mode)

data/lib/aws-sdk-s3/plugins/md5s.rb

@@ -22,9 +22,11 @@ module Aws
           CHUNK_SIZE = 1 * 1024 * 1024 # one MB
 
           def call(context)
-            body = context.http_request.body
-            if body.size > 0
-              context.http_request.headers['Content-Md5'] ||= md5(body)
+            if !context[:checksum_algorithms] # skip in favor of flexible checksum
+              body = context.http_request.body
+              if body.respond_to?(:size) && body.size > 0
+                context.http_request.headers['Content-Md5'] ||= md5(body)
+              end
             end
             @handler.call(context)
           end

data/lib/aws-sdk-s3/plugins/object_lambda_endpoint.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+      # WriteGetObjectResponse is called from Lambda after a data transform.
+      # If there is no custom endpoint, we change the endpoint from s3 to
+      # s3-object-lambda just for this operation.
+      class ObjectLambdaEndpoint < Seahorse::Client::Plugin
+        class Handler < Seahorse::Client::Handler
+          def call(context)
+            if context.config.regional_endpoint
+              host = context.http_request.endpoint.host
+              host = host.sub('s3.', 's3-object-lambda.')
+              context.http_request.endpoint.host = host
+            end
+            @handler.call(context)
+          end
+        end
+
+        handler(Handler, operations: [:write_get_object_response])
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/s3_signer.rb

@@ -12,15 +12,19 @@ module Aws
 
         option(:sigv4_signer) do |cfg|
           S3Signer.build_v4_signer(
+            service: 's3',
             region: cfg.sigv4_region,
             credentials: cfg.credentials
           )
         end
 
         option(:sigv4_region) do |cfg|
+          # S3 removes core's signature_v4 plugin that checks for this
           raise Aws::Errors::MissingRegionError if cfg.region.nil?
 
-          Aws::Partitions::EndpointProvider.signing_region(cfg.region, 's3')
+          Aws::Partitions::EndpointProvider.signing_region(
+            cfg.region, 's3'
+          )
         end
 
         def add_handlers(handlers, cfg)
@@ -67,9 +71,30 @@ module Aws
             if context[:cached_sigv4_region] &&
                context[:cached_sigv4_region] != context.config.sigv4_signer.region
               S3Signer.build_v4_signer(
+                service: 's3',
                 region: context[:cached_sigv4_region],
                 credentials: context.config.credentials
               )
+            elsif (arn = context.metadata[:s3_arn])
+              if arn[:arn].is_a?(MultiRegionAccessPointARN)
+                signing_region = '*'
+                signing_algorithm = :sigv4a
+              else
+                signing_region = arn[:resolved_region]
+                signing_algorithm = :sigv4
+              end
+              S3Signer.build_v4_signer(
+                service: arn[:arn].service,
+                signing_algorithm: signing_algorithm,
+                region: signing_region,
+                credentials: context.config.credentials
+              )
+            elsif context.operation.name == 'WriteGetObjectResponse'
+              S3Signer.build_v4_signer(
+                service: 's3-object-lambda',
+                region: context.config.sigv4_region,
+                credentials: context.config.credentials
+              )
             else
               context.config.sigv4_signer
             end
@@ -90,7 +115,9 @@ module Aws
           def check_for_cached_region(context, bucket)
             cached_region = S3::BUCKET_REGIONS[bucket]
             if cached_region && cached_region != context.config.region
-              context.http_request.endpoint.host = S3Signer.new_hostname(context, cached_region)
+              context.http_request.endpoint.host = S3Signer.new_hostname(
+                context, cached_region
+              )
               context[:cached_sigv4_region] = cached_region
             end
           end
@@ -111,7 +138,8 @@ module Aws
           def handle_region_errors(response)
             if wrong_sigv4_region?(response) &&
                !fips_region?(response) &&
-               !custom_endpoint?(response)
+               !custom_endpoint?(response) &&
+               !expired_credentials?(response)
               get_region_and_retry(response.context)
             else
               response
@@ -135,9 +163,18 @@ module Aws
             resp.context.http_request.endpoint.host.include?('fips')
           end
 
+          def expired_credentials?(resp)
+            resp.context.http_response.body_contents.match(/<Code>ExpiredToken<\/Code>/)
+          end
+
           def custom_endpoint?(resp)
             resolved_suffix = Aws::Partitions::EndpointProvider.dns_suffix_for(
-              resp.context.config.region
+              resp.context.config.region,
+              's3',
+              {
+                dualstack: resp.context[:use_dualstack_endpoint],
+                fips: resp.context.config.use_fips_endpoint
+              }
             )
             !resp.context.http_request.endpoint.hostname.include?(resolved_suffix)
           end
@@ -150,11 +187,18 @@ module Aws
 
           def resign_with_new_region(context, actual_region)
             context.http_response.body.truncate(0)
-            context.http_request.endpoint.host = S3Signer.new_hostname(context, actual_region)
+            context.http_request.endpoint.host = S3Signer.new_hostname(
+              context, actual_region
+            )
             context.metadata[:redirect_region] = actual_region
+            # if it's an ARN, use the service in the ARN
+            if (arn = context.metadata[:s3_arn])
+              service = arn[:arn].service
+            end
             Aws::Plugins::SignatureV4.apply_signature(
               context: context,
               signer: S3Signer.build_v4_signer(
+                service: service || 's3',
                 region: actual_region,
                 credentials: context.config.credentials
               )
@@ -189,30 +233,35 @@ module Aws
           # @api private
           def build_v4_signer(options = {})
             Aws::Sigv4::Signer.new(
-              service: 's3',
+              service: options[:service],
               region: options[:region],
               credentials_provider: options[:credentials],
+              signing_algorithm: options.fetch(:signing_algorithm, :sigv4),
               uri_escape_path: false,
               unsigned_headers: ['content-length', 'x-amzn-trace-id']
             )
           end
 
+          # Check to see if the bucket is actually an ARN
+          # Otherwise it will retry with the ARN as the bucket name.
           def new_hostname(context, region)
-            # Check to see if the bucket is actually an ARN and resolve it
-            # Otherwise it will retry with the ARN as the bucket name.
-            resolved_bucket, resolved_region, arn = BucketARN.resolve_arn!(
-              context.params[:bucket],
-              region,
-              context.config.s3_use_arn_region
-            )
             uri = URI.parse(
-              Aws::Partitions::EndpointProvider.resolve(resolved_region, 's3')
+              Aws::Partitions::EndpointProvider.resolve(
+                region, 's3', 'regional',
+                {
+                  dualstack: context[:use_dualstack_endpoint],
+                  fips: context.config.use_fips_endpoint
+                }
+              )
             )
 
-            if arn
-              BucketARN.resolve_url!(uri, arn).host
+            if (arn = context.metadata[:s3_arn])
+              # Retry with the response region and not the ARN resolved one
+              ARN.resolve_url!(
+                uri, arn[:arn], region, arn[:fips], arn[:dualstack]
+              ).host
             else
-              resolved_bucket + '.' + uri.host
+              "#{context.params[:bucket]}.#{uri.host}"
             end
           end
         end

data/lib/aws-sdk-s3/plugins/skip_whole_multipart_get_checksums.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+
+      # S3 GetObject results for whole Multipart Objects contain a checksum
+      # that cannot be validated.  These should be skipped by the
+      # ChecksumAlgorithm plugin.
+      class SkipWholeMultipartGetChecksums < Seahorse::Client::Plugin
+
+        class Handler < Seahorse::Client::Handler
+
+          def call(context)
+            context[:http_checksum] ||= {}
+            context[:http_checksum][:skip_on_suffix] = true
+
+            @handler.call(context)
+          end
+
+        end
+
+        handler(
+          Handler,
+          step: :initialize,
+          operations: [:get_object]
+        )
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/streaming_retry.rb

@@ -36,6 +36,17 @@ module Aws
         def rewind; end
       end
 
+      class NonRetryableStreamingError < StandardError
+
+        def initialize(error)
+          super('Unable to retry request - retry could result in processing duplicated chunks.')
+          set_backtrace(error.backtrace)
+          @original_error = error
+        end
+
+        attr_reader :original_error
+      end
+
       # This handler works with the ResponseTarget plugin to provide smart
       # retries of S3 streaming operations that support the range parameter
       # (currently only: get_object).  When a 200 OK with a TruncatedBodyError
@@ -84,8 +95,18 @@ module Aws
             end
 
             context.http_response.on_error do |error|
-              if retryable_body?(context) && truncated_body?(error)
-                context.http_request.headers[:range] = "bytes=#{context.http_response.body.size}-"
+              if retryable_body?(context)
+                if truncated_body?(error)
+                  context.http_request.headers[:range] = "bytes=#{context.http_response.body.size}-"
+                else
+                  case context.http_response.body
+                  when RetryableManagedFile
+                    # call rewind on the underlying file
+                    context.http_response.body.instance_variable_get(:@file).rewind
+                  else
+                    raise NonRetryableStreamingError, error
+                  end
+                end
               end
             end
           end

data/lib/aws-sdk-s3/presigned_post.rb

@@ -98,7 +98,7 @@ module Aws
     # or call the associated method.
     #
     # ```ruby
-    # post = Aws::S3::PresignedPost.new(creds, region, bucket).
+    # post = Aws::S3::PresignedPost.new(creds, region, bucket)
     # post.content_type('text/plain')
     # ```
     #
@@ -176,11 +176,17 @@ module Aws
     # ```
     #
     class PresignedPost
+      @@allowed_fields = []
 
       # @param [Credentials] credentials Security credentials for signing
       #   the post policy.
       # @param [String] bucket_region Region of the target bucket.
       # @param [String] bucket_name Name of the target bucket.
+      # @option options [Boolean] :use_accelerate_endpoint (false) When `true`,
+      #   PresignedPost will attempt to use accelerated endpoint.
+      # @option options [String] :url See {PresignedPost#url}.
+      # @option options [Sting, Array<String>] :allow_any
+      #   See {PresignedPost#allow_any}.
       # @option options [Time] :signature_expiration Specify when the signature on
       #   the post will expire. Defaults to one hour from creation of the
       #   presigned post. May not exceed one week from creation time.
@@ -205,7 +211,7 @@ module Aws
       #   See {PresignedPost#content_encoding}.
       # @option options [String] :content_encoding_starts_with
       #   See {PresignedPost#content_encoding_starts_with}.
-      # @option options [String] :expires See {PresignedPost#expires}.
+      # @option options [Time] :expires See {PresignedPost#expires}.
       # @option options [String] :expires_starts_with
       #   See {PresignedPost#expires_starts_with}.
       # @option options [Range<Integer>] :content_length_range
@@ -232,6 +238,8 @@ module Aws
       #   See {PresignedPost#server_side_encryption_customer_algorithm}.
       # @option options [String] :server_side_encryption_customer_key
       #   See {PresignedPost#server_side_encryption_customer_key}.
+      # @option options [String] :server_side_encryption_customer_key_starts_with
+      #   See {PresignedPost#server_side_encryption_customer_key_starts_with}.
       def initialize(credentials, bucket_region, bucket_name, options = {})
         @credentials = credentials.credentials
         @bucket_region = bucket_region
@@ -247,7 +255,12 @@ module Aws
           case option_name
           when :allow_any then allow_any(option_value)
           when :signature_expiration then @signature_expiration = option_value
-          else send("#{option_name}", option_value)
+          else
+            if @@allowed_fields.include?(option_name)
+              send("#{option_name}", option_value)
+            else
+              raise ArgumentError, "Unsupported option: #{option_name}"
+            end
           end
         end
       end
@@ -279,17 +292,23 @@ module Aws
       end
 
       # @api private
-      def self.define_field(field, *args)
+      def self.define_field(field, *args, &block)
+        @@allowed_fields << field
         options = args.last.is_a?(Hash) ? args.pop : {}
         field_name = args.last || field.to_s
 
-        define_method("#{field}") do |value|
-          with(field_name, value)
-        end
+        if block_given?
+          define_method("#{field}", block)
+        else
+          define_method("#{field}") do |value|
+            with(field_name, value)
+          end
 
-        if options[:starts_with]
-          define_method("#{field}_starts_with") do |value|
-            starts_with(field_name, value)
+          if options[:starts_with]
+            @@allowed_fields << "#{field}_starts_with".to_sym
+            define_method("#{field}_starts_with") do |value|
+              starts_with(field_name, value)
+            end
           end
         end
       end
@@ -307,7 +326,7 @@ module Aws
       # @param [String] key
       # @see http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html)
       # @return [self]
-      def key(key)
+      define_field(:key) do |key|
         @key_set = true
         with('key', key)
       end
@@ -316,7 +335,7 @@ module Aws
       # @param [String] prefix
       # @see #key
       # @return [self]
-      def key_starts_with(prefix)
+      define_field(:key_starts_with) do |prefix|
         @key_set = true
         starts_with('key', prefix)
       end
@@ -399,21 +418,21 @@ module Aws
       # @param [Time] time
       # @see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21
       # @return [self]
-      def expires(time)
+      define_field(:expires) do |time|
         with('Expires', time.httpdate)
       end
 
       # @param [String] prefix
       # @see #expires
       # @return [self]
-      def expires_starts_with(prefix)
+      define_field(:expires_starts_with) do |prefix|
         starts_with('Expires', prefix)
       end
 
       # The minimum and maximum allowable size for the uploaded content.
       # @param [Range<Integer>] byte_range
       # @return [self]
-      def content_length_range(byte_range)
+      define_field(:content_length_range) do |byte_range|
         min = byte_range.begin
         max = byte_range.end
         max -= 1 if byte_range.exclude_end?
@@ -492,7 +511,7 @@ module Aws
       # prefixed with "x-amz-meta-".
       # @param [Hash<String,String>] hash
       # @return [self]
-      def metadata(hash)
+      define_field(:metadata) do |hash|
         hash.each do |key, value|
           with("x-amz-meta-#{key}", value)
         end
@@ -503,7 +522,7 @@ module Aws
       # @param [Hash<String,String>] hash
       # @see #metadata
       # @return [self]
-      def metadata_starts_with(hash)
+      define_field(:metadata_starts_with) do |hash|
         hash.each do |key, value|
           starts_with("x-amz-meta-#{key}", value)
         end
@@ -561,7 +580,7 @@ module Aws
       # @param [String] value
       # @see #server_side_encryption_customer_algorithm
       # @return [self]
-      def server_side_encryption_customer_key(value)
+      define_field(:server_side_encryption_customer_key) do |value|
         field_name = 'x-amz-server-side-encryption-customer-key'
         with(field_name, base64(value))
         with(field_name + '-MD5', base64(OpenSSL::Digest::MD5.digest(value)))
@@ -570,7 +589,7 @@ module Aws
       # @param [String] prefix
       # @see #server_side_encryption_customer_key
       # @return [self]
-      def server_side_encryption_customer_key_starts_with(prefix)
+      define_field(:server_side_encryption_customer_key_starts_with) do |prefix|
         field_name = 'x-amz-server-side-encryption-customer-key'
         starts_with(field_name, prefix)
       end

data/lib/aws-sdk-s3/presigner.rb

@@ -58,8 +58,7 @@ module Aws
       #   is returned instead of the default HTTPS URL.
       #
       # @option params [Boolean] :virtual_host (false) When `true`, the
-      #   bucket name will be used as the hostname. This will cause
-      #   the returned URL to be 'http' and not 'https'.
+      #   bucket name will be used as the hostname.
       #
       # @option params [Boolean] :use_accelerate_endpoint (false) When `true`,
       #   Presigner will attempt to use accelerated endpoint.
@@ -139,6 +138,7 @@ module Aws
 
         req = @client.build_request(method, params)
         use_bucket_as_hostname(req) if virtual_host
+        handle_presigned_url_context(req)
 
         x_amz_headers = sign_but_dont_send(
           req, expires_in, scheme, time, unsigned_headers, hoist
@@ -184,6 +184,17 @@ module Aws
         end
       end
 
+      # Used for excluding presigned_urls from API request count.
+      #
+      # Store context information as early as possible, to allow
+      # handlers to perform decisions based on this flag if need.
+      def handle_presigned_url_context(req)
+        req.handle(step: :initialize, priority: 98) do |context|
+          context[:presigned_url] = true
+          @handler.call(context)
+        end
+      end
+
       # @param [Seahorse::Client::Request] req
       def sign_but_dont_send(
         req, expires_in, scheme, time, unsigned_headers, hoist = true
@@ -196,8 +207,6 @@ module Aws
         req.handlers.remove(Aws::S3::Plugins::S3Signer::V4Handler)
         req.handlers.remove(Seahorse::Client::Plugins::ContentLength::Handler)
 
-        signer = build_signer(req.context.config, unsigned_headers)
-
         req.handle(step: :send) do |context|
           if scheme != http_req.endpoint.scheme
             endpoint = http_req.endpoint.dup
@@ -222,6 +231,26 @@ module Aws
           end
           http_req.endpoint.query = query.join('&') unless query.empty?
 
+          signing_algorithm = :sigv4
+
+          # If it's an ARN, get the resolved region and service
+          if (arn = context.metadata[:s3_arn])
+            region = arn[:resolved_region]
+            service = arn[:arn].service
+            region = arn[:arn].is_a?(MultiRegionAccessPointARN) ? '*': arn[:resolved_region]
+            signing_algorithm = arn[:arn].is_a?(MultiRegionAccessPointARN) ? :sigv4a : :sigv4
+          end
+
+          signer = Aws::Sigv4::Signer.new(
+            service: service || 's3',
+            region: region || context.config.region,
+            signing_algorithm: signing_algorithm,
+            credentials_provider: context.config.credentials,
+            unsigned_headers: unsigned_headers,
+            apply_checksum_header: false,
+            uri_escape_path: false
+          )
+
           url = signer.presign_url(
             http_method: http_req.http_method,
             url: http_req.endpoint,
@@ -231,24 +260,11 @@ module Aws
             time: time
           ).to_s
 
-          # Used for excluding presigned_urls from API request count
-          context[:presigned_url] = true
-
           Seahorse::Client::Response.new(context: context, data: url)
         end
         # Return the headers
         x_amz_headers
       end
-
-      def build_signer(cfg, unsigned_headers)
-        Aws::Sigv4::Signer.new(
-          service: 's3',
-          region: cfg.region,
-          credentials_provider: cfg.credentials,
-          unsigned_headers: unsigned_headers,
-          uri_escape_path: false
-        )
-      end
     end
   end
 end

data/lib/aws-sdk-s3/resource.rb

@@ -3,7 +3,7 @@
 # WARNING ABOUT GENERATED CODE
 #
 # This file is generated. See the contributing guide for more information:
-# https://github.com/aws/aws-sdk-ruby/blob/master/CONTRIBUTING.md
+# https://github.com/aws/aws-sdk-ruby/blob/version-3/CONTRIBUTING.md
 #
 # WARNING ABOUT GENERATED CODE
 
@@ -49,6 +49,7 @@ module Aws::S3
     #     grant_write: "GrantWrite",
     #     grant_write_acp: "GrantWriteACP",
     #     object_lock_enabled_for_bucket: false,
+    #     object_ownership: "BucketOwnerPreferred", # accepts BucketOwnerPreferred, ObjectWriter, BucketOwnerEnforced
     #   })
     # @param [Hash] options ({})
     # @option options [String] :acl
@@ -65,13 +66,32 @@ module Aws::S3
     # @option options [String] :grant_read_acp
     #   Allows grantee to read the bucket ACL.
     # @option options [String] :grant_write
-    #   Allows grantee to create, overwrite, and delete any object in the
-    #   bucket.
+    #   Allows grantee to create new objects in the bucket.
+    #
+    #   For the bucket and object owners of existing objects, also allows
+    #   deletions and overwrites of those objects.
     # @option options [String] :grant_write_acp
     #   Allows grantee to write the ACL for the applicable bucket.
     # @option options [Boolean] :object_lock_enabled_for_bucket
     #   Specifies whether you want S3 Object Lock to be enabled for the new
     #   bucket.
+    # @option options [String] :object_ownership
+    #   The container element for object ownership for a bucket's ownership
+    #   controls.
+    #
+    #   BucketOwnerPreferred - Objects uploaded to the bucket change ownership
+    #   to the bucket owner if the objects are uploaded with the
+    #   `bucket-owner-full-control` canned ACL.
+    #
+    #   ObjectWriter - The uploading account will own the object if the object
+    #   is uploaded with the `bucket-owner-full-control` canned ACL.
+    #
+    #   BucketOwnerEnforced - Access control lists (ACLs) are disabled and no
+    #   longer affect permissions. The bucket owner automatically owns and has
+    #   full control over every object in the bucket. The bucket only accepts
+    #   PUT requests that don't specify an ACL or bucket owner full control
+    #   ACLs, such as the `bucket-owner-full-control` canned ACL or an
+    #   equivalent form of this ACL expressed in the XML format.
     # @return [Bucket]
     def create_bucket(options = {})
       @client.create_bucket(options)