aws-sdk-s3 1.109.0 → 1.156.0

data/lib/aws-sdk-s3/plugins/s3_signer.rb

@@ -4,29 +4,19 @@ require 'aws-sigv4'
 
 module Aws
   module S3
+    # @api private
+    def self.bucket_region_cache
+      @bucket_region_cache ||= BucketRegionCache.new
+    end
+
     module Plugins
-      # This plugin is an implementation detail and may be modified.
+      # This plugin used to have a V4 signer but it was removed in favor of
+      # generic Sign plugin that uses endpoint auth scheme.
+      #
       # @api private
       class S3Signer < Seahorse::Client::Plugin
         option(:signature_version, 'v4')
 
-        option(:sigv4_signer) do |cfg|
-          S3Signer.build_v4_signer(
-            service: 's3',
-            region: cfg.sigv4_region,
-            credentials: cfg.credentials
-          )
-        end
-
-        option(:sigv4_region) do |cfg|
-          # S3 removes core's signature_v4 plugin that checks for this
-          raise Aws::Errors::MissingRegionError if cfg.region.nil?
-
-          Aws::Partitions::EndpointProvider.signing_region(
-            cfg.region, 's3'
-          )
-        end
-
         def add_handlers(handlers, cfg)
           case cfg.signature_version
           when 'v4' then add_v4_handlers(handlers)
@@ -39,11 +29,11 @@ module Aws
 
         def add_v4_handlers(handlers)
           handlers.add(CachedBucketRegionHandler, step: :sign, priority: 60)
-          handlers.add(V4Handler, step: :sign)
           handlers.add(BucketRegionErrorHandler, step: :sign, priority: 40)
         end
 
         def add_legacy_handler(handlers)
+          # generic Sign plugin will be skipped if it sees sigv2
           handlers.add(LegacyHandler, step: :sign)
         end
 
@@ -54,53 +44,6 @@ module Aws
           end
         end
 
-        class V4Handler < Seahorse::Client::Handler
-          def call(context)
-            Aws::Plugins::SignatureV4.apply_signature(
-              context: context,
-              signer: sigv4_signer(context)
-            )
-            @handler.call(context)
-          end
-
-          private
-
-          def sigv4_signer(context)
-            # If the client was configured with the wrong region,
-            # we have to build a new signer.
-            if context[:cached_sigv4_region] &&
-               context[:cached_sigv4_region] != context.config.sigv4_signer.region
-              S3Signer.build_v4_signer(
-                service: 's3',
-                region: context[:cached_sigv4_region],
-                credentials: context.config.credentials
-              )
-            elsif (arn = context.metadata[:s3_arn])
-              if arn[:arn].is_a?(MultiRegionAccessPointARN)
-                signing_region = '*'
-                signing_algorithm = :sigv4a
-              else
-                signing_region = arn[:resolved_region]
-                signing_algorithm = :sigv4
-              end
-              S3Signer.build_v4_signer(
-                service: arn[:arn].service,
-                signing_algorithm: signing_algorithm,
-                region: signing_region,
-                credentials: context.config.credentials
-              )
-            elsif context.operation.name == 'WriteGetObjectResponse'
-              S3Signer.build_v4_signer(
-                service: 's3-object-lambda',
-                region: context.config.sigv4_region,
-                credentials: context.config.credentials
-              )
-            else
-              context.config.sigv4_signer
-            end
-          end
-        end
-
         # This handler will update the http endpoint when the bucket region
         # is known/cached.
         class CachedBucketRegionHandler < Seahorse::Client::Handler
@@ -113,12 +56,14 @@ module Aws
           private
 
           def check_for_cached_region(context, bucket)
-            cached_region = S3::BUCKET_REGIONS[bucket]
-            if cached_region && cached_region != context.config.region
+            cached_region = Aws::S3.bucket_region_cache[bucket]
+            if cached_region &&
+               cached_region != context.config.region &&
+               !S3Signer.custom_endpoint?(context)
               context.http_request.endpoint.host = S3Signer.new_hostname(
                 context, cached_region
               )
-              context[:cached_sigv4_region] = cached_region
+              context[:sigv4_region] = cached_region # Sign plugin will use this
             end
           end
         end
@@ -126,7 +71,8 @@ module Aws
         # This handler detects when a request fails because of a mismatched bucket
         # region. It follows up by making a request to determine the correct
         # region, then finally a version 4 signed request against the correct
-        # regional endpoint.
+        # regional endpoint. This is intended for s3's global endpoint which
+        # will return 400 if the bucket is not in region.
         class BucketRegionErrorHandler < Seahorse::Client::Handler
           def call(context)
             response = @handler.call(context)
@@ -138,7 +84,8 @@ module Aws
           def handle_region_errors(response)
             if wrong_sigv4_region?(response) &&
                !fips_region?(response) &&
-               !custom_endpoint?(response)
+               !S3Signer.custom_endpoint?(response.context) &&
+               !expired_credentials?(response)
               get_region_and_retry(response.context)
             else
               response
@@ -155,23 +102,15 @@ module Aws
           end
 
           def update_bucket_cache(context, actual_region)
-            S3::BUCKET_REGIONS[context.params[:bucket]] = actual_region
+            Aws::S3.bucket_region_cache[context.params[:bucket]] = actual_region
           end
 
           def fips_region?(resp)
-            resp.context.http_request.endpoint.host.include?('fips')
+            resp.context.http_request.endpoint.host.include?('s3-fips.')
           end
 
-          def custom_endpoint?(resp)
-            resolved_suffix = Aws::Partitions::EndpointProvider.dns_suffix_for(
-              resp.context.config.region,
-              's3',
-              {
-                dualstack: resp.context[:use_dualstack_endpoint],
-                fips: resp.context.config.use_fips_endpoint
-              }
-            )
-            !resp.context.http_request.endpoint.hostname.include?(resolved_suffix)
+          def expired_credentials?(resp)
+            resp.context.http_response.body_contents.match(/<Code>ExpiredToken<\/Code>/)
           end
 
           def wrong_sigv4_region?(resp)
@@ -186,18 +125,14 @@ module Aws
               context, actual_region
             )
             context.metadata[:redirect_region] = actual_region
-            # if it's an ARN, use the service in the ARN
-            if (arn = context.metadata[:s3_arn])
-              service = arn[:arn].service
-            end
-            Aws::Plugins::SignatureV4.apply_signature(
-              context: context,
-              signer: S3Signer.build_v4_signer(
-                service: service || 's3',
-                region: actual_region,
-                credentials: context.config.credentials
-              )
+
+            signer = Aws::Plugins::Sign.signer_for(
+              context[:auth_scheme],
+              context.config,
+              actual_region
             )
+
+            signer.sign(context)
           end
 
           def region_from_body(body)
@@ -223,41 +158,22 @@ module Aws
         end
 
         class << self
-          # @option options [required, String] :region
-          # @option options [required, #credentials] :credentials
-          # @api private
-          def build_v4_signer(options = {})
-            Aws::Sigv4::Signer.new(
-              service: options[:service],
-              region: options[:region],
-              credentials_provider: options[:credentials],
-              signing_algorithm: options.fetch(:signing_algorithm, :sigv4),
-              uri_escape_path: false,
-              unsigned_headers: ['content-length', 'x-amzn-trace-id']
-            )
+          def new_hostname(context, region)
+            endpoint_params = context[:endpoint_params].dup
+            endpoint_params.region = region
+            endpoint_params.endpoint = nil
+            endpoint =
+              context.config.endpoint_provider.resolve_endpoint(endpoint_params)
+            URI(endpoint.url).host
           end
 
-          # Check to see if the bucket is actually an ARN
-          # Otherwise it will retry with the ARN as the bucket name.
-          def new_hostname(context, region)
-            uri = URI.parse(
-              Aws::Partitions::EndpointProvider.resolve(
-                region, 's3', 'regional',
-                {
-                  dualstack: context[:use_dualstack_endpoint],
-                  fips: context.config.use_fips_endpoint
-                }
-              )
-            )
+          def custom_endpoint?(context)
+            region = context.config.region
+            partition = Aws::Endpoints::Matchers.aws_partition(region)
+            endpoint = context.http_request.endpoint
 
-            if (arn = context.metadata[:s3_arn])
-              # Retry with the response region and not the ARN resolved one
-              ARN.resolve_url!(
-                uri, arn[:arn], region, arn[:fips], arn[:dualstack]
-              ).host
-            else
-              "#{context.params[:bucket]}.#{uri.host}"
-            end
+            !endpoint.hostname.include?(partition['dnsSuffix']) &&
+              !endpoint.hostname.include?(partition['dualStackDnsSuffix'])
           end
         end
       end

data/lib/aws-sdk-s3/plugins/skip_whole_multipart_get_checksums.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+
+      # S3 GetObject results for whole Multipart Objects contain a checksum
+      # that cannot be validated.  These should be skipped by the
+      # ChecksumAlgorithm plugin.
+      class SkipWholeMultipartGetChecksums < Seahorse::Client::Plugin
+
+        class Handler < Seahorse::Client::Handler
+
+          def call(context)
+            context[:http_checksum] ||= {}
+            context[:http_checksum][:skip_on_suffix] = true
+
+            @handler.call(context)
+          end
+
+        end
+
+        handler(
+          Handler,
+          step: :initialize,
+          operations: [:get_object]
+        )
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/streaming_retry.rb

@@ -36,6 +36,17 @@ module Aws
         def rewind; end
       end
 
+      class NonRetryableStreamingError < StandardError
+
+        def initialize(error)
+          super('Unable to retry request - retry could result in processing duplicated chunks.')
+          set_backtrace(error.backtrace)
+          @original_error = error
+        end
+
+        attr_reader :original_error
+      end
+
       # This handler works with the ResponseTarget plugin to provide smart
       # retries of S3 streaming operations that support the range parameter
       # (currently only: get_object).  When a 200 OK with a TruncatedBodyError
@@ -84,8 +95,18 @@ module Aws
             end
 
             context.http_response.on_error do |error|
-              if retryable_body?(context) && truncated_body?(error)
-                context.http_request.headers[:range] = "bytes=#{context.http_response.body.size}-"
+              if retryable_body?(context)
+                if truncated_body?(error)
+                  context.http_request.headers[:range] = "bytes=#{context.http_response.body.size}-"
+                else
+                  case context.http_response.body
+                  when RetryableManagedFile
+                    # call rewind on the underlying file
+                    context.http_response.body.instance_variable_get(:@file).rewind
+                  else
+                    raise NonRetryableStreamingError, error
+                  end
+                end
               end
             end
           end

data/lib/aws-sdk-s3/presigned_post.rb

@@ -98,7 +98,7 @@ module Aws
     # or call the associated method.
     #
     # ```ruby
-    # post = Aws::S3::PresignedPost.new(creds, region, bucket).
+    # post = Aws::S3::PresignedPost.new(creds, region, bucket)
     # post.content_type('text/plain')
     # ```
     #
@@ -176,11 +176,17 @@ module Aws
     # ```
     #
     class PresignedPost
+      @@allowed_fields = []
 
       # @param [Credentials] credentials Security credentials for signing
       #   the post policy.
       # @param [String] bucket_region Region of the target bucket.
       # @param [String] bucket_name Name of the target bucket.
+      # @option options [Boolean] :use_accelerate_endpoint (false) When `true`,
+      #   PresignedPost will attempt to use accelerated endpoint.
+      # @option options [String] :url See {PresignedPost#url}.
+      # @option options [Sting, Array<String>] :allow_any
+      #   See {PresignedPost#allow_any}.
       # @option options [Time] :signature_expiration Specify when the signature on
       #   the post will expire. Defaults to one hour from creation of the
       #   presigned post. May not exceed one week from creation time.
@@ -205,7 +211,7 @@ module Aws
       #   See {PresignedPost#content_encoding}.
       # @option options [String] :content_encoding_starts_with
       #   See {PresignedPost#content_encoding_starts_with}.
-      # @option options [String] :expires See {PresignedPost#expires}.
+      # @option options [Time] :expires See {PresignedPost#expires}.
       # @option options [String] :expires_starts_with
       #   See {PresignedPost#expires_starts_with}.
       # @option options [Range<Integer>] :content_length_range
@@ -232,6 +238,8 @@ module Aws
       #   See {PresignedPost#server_side_encryption_customer_algorithm}.
       # @option options [String] :server_side_encryption_customer_key
       #   See {PresignedPost#server_side_encryption_customer_key}.
+      # @option options [String] :server_side_encryption_customer_key_starts_with
+      #   See {PresignedPost#server_side_encryption_customer_key_starts_with}.
       def initialize(credentials, bucket_region, bucket_name, options = {})
         @credentials = credentials.credentials
         @bucket_region = bucket_region
@@ -247,7 +255,12 @@ module Aws
           case option_name
           when :allow_any then allow_any(option_value)
           when :signature_expiration then @signature_expiration = option_value
-          else send("#{option_name}", option_value)
+          else
+            if @@allowed_fields.include?(option_name)
+              send("#{option_name}", option_value)
+            else
+              raise ArgumentError, "Unsupported option: #{option_name}"
+            end
           end
         end
       end
@@ -279,44 +292,52 @@ module Aws
       end
 
       # @api private
-      def self.define_field(field, *args)
+      def self.define_field(field, *args, &block)
+        @@allowed_fields << field
         options = args.last.is_a?(Hash) ? args.pop : {}
         field_name = args.last || field.to_s
 
-        define_method("#{field}") do |value|
-          with(field_name, value)
-        end
+        if block_given?
+          define_method("#{field}", block)
+        else
+          define_method("#{field}") do |value|
+            with(field_name, value)
+          end
 
-        if options[:starts_with]
-          define_method("#{field}_starts_with") do |value|
-            starts_with(field_name, value)
+          if options[:starts_with]
+            @@allowed_fields << "#{field}_starts_with".to_sym
+            define_method("#{field}_starts_with") do |value|
+              starts_with(field_name, value)
+            end
           end
         end
       end
 
       # @!group Fields
 
-      # The key to use for the uploaded object. You can use `${filename}`
-      # as a variable in the key. This will be replaced with the name
-      # of the file as provided by the user.
+      # @!method key(key)
+      #   The key to use for the uploaded object. You can use `${filename}`
+      #   as a variable in the key. This will be replaced with the name
+      #   of the file as provided by the user.
       #
-      # For example, if the key is given as `/user/betty/${filename}` and
-      # the file uploaded is named `lolcatz.jpg`, the resultant key will
-      # be `/user/betty/lolcatz.jpg`.
+      #   For example, if the key is given as `/user/betty/${filename}` and
+      #   the file uploaded is named `lolcatz.jpg`, the resultant key will
+      #   be `/user/betty/lolcatz.jpg`.
       #
-      # @param [String] key
-      # @see http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html)
-      # @return [self]
-      def key(key)
+      #   @param [String] key
+      #   @see http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html)
+      #   @return [self]
+      define_field(:key) do |key|
         @key_set = true
         with('key', key)
       end
 
-      # Specify a prefix the uploaded
-      # @param [String] prefix
-      # @see #key
-      # @return [self]
-      def key_starts_with(prefix)
+      # @!method key_starts_with(prefix)
+      #   Specify a prefix the uploaded
+      #   @param [String] prefix
+      #   @see #key
+      #   @return [self]
+      define_field(:key_starts_with) do |prefix|
         @key_set = true
         starts_with('key', prefix)
       end
@@ -393,27 +414,30 @@ module Aws
       #   @return [self]
       define_field(:content_encoding, 'Content-Encoding', starts_with: true)
 
-      # The date and time at which the object is no longer cacheable.
-      # @note This does not affect the expiration of the presigned post
-      #   signature.
-      # @param [Time] time
-      # @see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21
-      # @return [self]
-      def expires(time)
+      # @!method expires(time)
+      #   The date and time at which the object is no longer cacheable.
+      #   @note This does not affect the expiration of the presigned post
+      #     signature.
+      #   @param [Time] time
+      #   @see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21
+      #   @return [self]
+      define_field(:expires) do |time|
         with('Expires', time.httpdate)
       end
 
-      # @param [String] prefix
-      # @see #expires
-      # @return [self]
-      def expires_starts_with(prefix)
+      # @!method expires_starts_with(prefix)
+      #   @param [String] prefix
+      #   @see #expires
+      #   @return [self]
+      define_field(:expires_starts_with) do |prefix|
         starts_with('Expires', prefix)
       end
 
-      # The minimum and maximum allowable size for the uploaded content.
-      # @param [Range<Integer>] byte_range
-      # @return [self]
-      def content_length_range(byte_range)
+      # @!method content_length_range(byte_range)
+      #   The minimum and maximum allowable size for the uploaded content.
+      #   @param [Range<Integer>] byte_range
+      #   @return [self]
+      define_field(:content_length_range) do |byte_range|
         min = byte_range.begin
         max = byte_range.end
         max -= 1 if byte_range.exclude_end?
@@ -488,22 +512,24 @@ module Aws
       #   @return [self]
       define_field(:website_redirect_location, 'x-amz-website-redirect-location')
 
-      # Metadata hash to store with the uploaded object. Hash keys will be
-      # prefixed with "x-amz-meta-".
-      # @param [Hash<String,String>] hash
-      # @return [self]
-      def metadata(hash)
+      # @!method metadata(hash)
+      #   Metadata hash to store with the uploaded object. Hash keys will be
+      #   prefixed with "x-amz-meta-".
+      #   @param [Hash<String,String>] hash
+      #   @return [self]
+      define_field(:metadata) do |hash|
         hash.each do |key, value|
           with("x-amz-meta-#{key}", value)
         end
         self
       end
 
-      # Specify allowable prefix for each key in the metadata hash.
-      # @param [Hash<String,String>] hash
-      # @see #metadata
-      # @return [self]
-      def metadata_starts_with(hash)
+      # @!method metadata_starts_with(hash)
+      #   Specify allowable prefix for each key in the metadata hash.
+      #   @param [Hash<String,String>] hash
+      #   @see #metadata
+      #   @return [self]
+      define_field(:metadata_starts_with) do |hash|
         hash.each do |key, value|
           starts_with("x-amz-meta-#{key}", value)
         end
@@ -552,25 +578,27 @@ module Aws
         'x-amz-server-side-encryption-customer-algorithm'
       )
 
-      # Specifies the customer-provided encryption key for Amazon S3 to use
-      # in encrypting data. This value is used to store the object and then
-      # it is discarded; Amazon does not store the encryption key.
+      # @!method server_side_encryption_customer_key(value)
+      #   Specifies the customer-provided encryption key for Amazon S3 to use
+      #   in encrypting data. This value is used to store the object and then
+      #   it is discarded; Amazon does not store the encryption key.
       #
-      # You must also call {#server_side_encryption_customer_algorithm}.
+      #   You must also call {#server_side_encryption_customer_algorithm}.
       #
-      # @param [String] value
-      # @see #server_side_encryption_customer_algorithm
-      # @return [self]
-      def server_side_encryption_customer_key(value)
+      #   @param [String] value
+      #   @see #server_side_encryption_customer_algorithm
+      #   @return [self]
+      define_field(:server_side_encryption_customer_key) do |value|
         field_name = 'x-amz-server-side-encryption-customer-key'
         with(field_name, base64(value))
         with(field_name + '-MD5', base64(OpenSSL::Digest::MD5.digest(value)))
       end
 
-      # @param [String] prefix
-      # @see #server_side_encryption_customer_key
-      # @return [self]
-      def server_side_encryption_customer_key_starts_with(prefix)
+      # @!method server_side_encryption_customer_key_starts_with(prefix)
+      #   @param [String] prefix
+      #   @see #server_side_encryption_customer_key
+      #   @return [self]
+      define_field(:server_side_encryption_customer_key_starts_with) do |prefix|
         field_name = 'x-amz-server-side-encryption-customer-key'
         starts_with(field_name, prefix)
       end
@@ -610,22 +638,15 @@ module Aws
       end
 
       def bucket_url
-        url = Aws::Partitions::EndpointProvider.resolve(@bucket_region, 's3')
-        url = URI.parse(url)
-        if Plugins::BucketDns.dns_compatible?(@bucket_name, _ssl = true)
-          if @accelerate
-            url.host = "#{@bucket_name}.s3-accelerate.amazonaws.com"
-          else
-            url.host = "#{@bucket_name}.#{url.host}"
-          end
-        else
-          url.path = "/#{@bucket_name}"
-        end
-        if @bucket_region == 'us-east-1'
-          # keep legacy behavior by default
-          url.host = Plugins::IADRegionalEndpoint.legacy_host(url.host)
-        end
-        url.to_s
+        # Taken from Aws::S3::Endpoints module
+        params = Aws::S3::EndpointParameters.new(
+          bucket: @bucket_name,
+          region: @bucket_region,
+          accelerate: @accelerate,
+          use_global_endpoint: true
+        )
+        endpoint = Aws::S3::EndpointProvider.new.resolve_endpoint(params)
+        endpoint.url
       end
 
       # @return [Hash]