aws-sdk-s3 1.87.0 → 1.143.0

data/lib/aws-sdk-s3/plugins/express_session_auth.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+      # @api private
+      class ExpressSessionAuth < Seahorse::Client::Plugin
+        # This should be s3_disable_express_auth instead
+        # But this is not a built in. We're overwriting the generated value
+        option(:disable_s3_express_session_auth,
+          default: false,
+          doc_type: 'Boolean',
+          docstring: <<-DOCS) do |cfg|
+When `true`, S3 Express session authentication is disabled.
+          DOCS
+          resolve_disable_s3_express_session_auth(cfg)
+        end
+
+        option(:express_credentials_provider,
+          doc_type: 'Aws::S3::ExpressCredentialsProvider',
+          rbs_type: 'untyped',
+          docstring: <<-DOCS) do |_cfg|
+Credential Provider for S3 Express endpoints. Manages credentials
+for different buckets.
+          DOCS
+          Aws::S3::ExpressCredentialsProvider.new
+        end
+
+        # @api private
+        class Handler < Seahorse::Client::Handler
+          def call(context)
+            if (props = context[:endpoint_properties])
+              # S3 Express endpoint - turn off md5 and enable crc32 default
+              if (backend = props['backend']) && backend == 'S3Express'
+                if context.operation_name == :put_object || checksum_required?(context)
+                  context[:default_request_checksum_algorithm] = 'CRC32'
+                end
+                context[:s3_express_endpoint] = true
+              end
+
+              # if s3 express auth, use new credentials and sign additional header
+              if context[:auth_scheme]['name'] == 'sigv4-s3express' &&
+                 !context.config.disable_s3_express_session_auth
+                bucket = context.params[:bucket]
+                credentials_provider = context.config.express_credentials_provider
+                credentials = credentials_provider.express_credentials_for(bucket)
+                context[:sigv4_credentials] = credentials # Sign will use this
+              end
+            end
+            @handler.call(context)
+          end
+
+          private
+
+          def checksum_required?(context)
+            context.operation.http_checksum_required ||
+              (context.operation.http_checksum &&
+                context.operation.http_checksum['requestChecksumRequired'])
+          end
+        end
+
+        handler(Handler)
+
+        # Optimization - sets this client as the client to create sessions.
+        def after_initialize(client)
+          provider = client.config.express_credentials_provider
+          provider.client = client unless provider.client
+        end
+
+        class << self
+          private
+
+          def resolve_disable_s3_express_session_auth(cfg)
+            value = ENV['AWS_S3_DISABLE_EXPRESS_SESSION_AUTH'] ||
+                    Aws.shared_config.s3_disable_express_session_auth(profile: cfg.profile) ||
+                    'false'
+            value = Aws::Util.str_2_bool(value)
+            # Raise if provided value is not true or false
+            if value.nil?
+              raise ArgumentError,
+                    'Must provide either `true` or `false` for the '\
+                    '`s3_disable_express_session_auth` profile option or for '\
+                    "ENV['AWS_S3_DISABLE_EXPRESS_SESSION_AUTH']."
+            end
+            value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/get_bucket_location_fix.rb

@@ -11,7 +11,7 @@ module Aws
             @handler.call(context).on(200) do |response|
               response.data = S3::Types::GetBucketLocationOutput.new
               xml = context.http_response.body_contents
-              matches = xml.match(/>(.+?)<\/LocationConstraint>/)
+              matches = xml.match(/<LocationConstraint.*?>(.+?)<\/LocationConstraint>/)
               response.data[:location_constraint] = matches ? matches[1] : ''
             end
           end

data/lib/aws-sdk-s3/plugins/iad_regional_endpoint.rb

@@ -10,46 +10,23 @@ module Aws
           default: 'legacy',
           doc_type: String,
           docstring: <<-DOCS) do |cfg|
-Passing in `regional` to enable regional endpoint for S3's `us-east-1`
-region. Defaults to `legacy` mode using global endpoint.
+Pass in `regional` to enable the `us-east-1` regional endpoint.
+Defaults to `legacy` mode which uses the global endpoint.
           DOCS
           resolve_iad_regional_endpoint(cfg)
         end
 
-        def add_handlers(handlers, config)
-          if config.region == 'us-east-1'
-            handlers.add(Handler)
-          end
-        end
-
-        # @api private
-        class Handler < Seahorse::Client::Handler
-
-          def call(context)
-            # keep legacy global endpoint pattern by default
-            if context.config.s3_us_east_1_regional_endpoint == 'legacy'
-              host = context.http_request.endpoint.host
-              # if it's an ARN, don't touch the endpoint at all
-              # TODO this should use context.metadata[:s3_arn] later
-              unless host.include?('.s3-outposts.') || host.include?('.s3-accesspoint.')
-                legacy_host = IADRegionalEndpoint.legacy_host(host)
-                context.http_request.endpoint.host = legacy_host
-              end
-            end
-            @handler.call(context)
-          end
-
-        end
-
-        def self.legacy_host(host)
-          host.sub(".us-east-1", '')
-        end
-
         private
 
         def self.resolve_iad_regional_endpoint(cfg)
+          default_mode_value =
+            if cfg.respond_to?(:defaults_mode_config_resolver)
+              cfg.defaults_mode_config_resolver.resolve(:s3_us_east_1_regional_endpoint)
+            end
+
           mode = ENV['AWS_S3_US_EAST_1_REGIONAL_ENDPOINT'] ||
             Aws.shared_config.s3_us_east_1_regional_endpoint(profile: cfg.profile) ||
+            default_mode_value ||
             'legacy'
           mode = mode.downcase
           unless %w(legacy regional).include?(mode)

data/lib/aws-sdk-s3/plugins/location_constraint.rb

@@ -22,7 +22,9 @@ module Aws
 
           def populate_location_constraint(params, region)
             params[:create_bucket_configuration] ||= {}
-            params[:create_bucket_configuration][:location_constraint] ||= region
+            unless params[:create_bucket_configuration][:location]
+              params[:create_bucket_configuration][:location_constraint] ||= region
+            end
           end
 
         end

data/lib/aws-sdk-s3/plugins/md5s.rb

@@ -22,9 +22,12 @@ module Aws
           CHUNK_SIZE = 1 * 1024 * 1024 # one MB
 
           def call(context)
-            body = context.http_request.body
-            if body.size > 0
-              context.http_request.headers['Content-Md5'] ||= md5(body)
+            if !context[:checksum_algorithms] && # skip in favor of flexible checksum
+               !context[:s3_express_endpoint] # s3 express endpoints do not support md5
+              body = context.http_request.body
+              if body.respond_to?(:size) && body.size > 0
+                context.http_request.headers['Content-Md5'] ||= md5(body)
+              end
             end
             @handler.call(context)
           end

data/lib/aws-sdk-s3/plugins/s3_signer.rb

@@ -5,26 +5,13 @@ require 'aws-sigv4'
 module Aws
   module S3
     module Plugins
-      # This plugin is an implementation detail and may be modified.
+      # This plugin used to have a V4 signer but it was removed in favor of
+      # generic Sign plugin that uses endpoint auth scheme.
+      #
       # @api private
       class S3Signer < Seahorse::Client::Plugin
         option(:signature_version, 'v4')
 
-        option(:sigv4_signer) do |cfg|
-          S3Signer.build_v4_signer(
-            service: 's3',
-            region: cfg.sigv4_region,
-            credentials: cfg.credentials
-          )
-        end
-
-        option(:sigv4_region) do |cfg|
-          # S3 removes core's signature_v4 plugin that checks for this
-          raise Aws::Errors::MissingRegionError if cfg.region.nil?
-
-          Aws::Partitions::EndpointProvider.signing_region(cfg.region, 's3')
-        end
-
         def add_handlers(handlers, cfg)
           case cfg.signature_version
           when 'v4' then add_v4_handlers(handlers)
@@ -37,11 +24,11 @@ module Aws
 
         def add_v4_handlers(handlers)
           handlers.add(CachedBucketRegionHandler, step: :sign, priority: 60)
-          handlers.add(V4Handler, step: :sign)
           handlers.add(BucketRegionErrorHandler, step: :sign, priority: 40)
         end
 
         def add_legacy_handler(handlers)
+          # generic Sign plugin will be skipped if it sees sigv2
           handlers.add(LegacyHandler, step: :sign)
         end
 
@@ -52,47 +39,6 @@ module Aws
           end
         end
 
-        class V4Handler < Seahorse::Client::Handler
-          def call(context)
-            Aws::Plugins::SignatureV4.apply_signature(
-              context: context,
-              signer: sigv4_signer(context)
-            )
-            @handler.call(context)
-          end
-
-          private
-
-          def sigv4_signer(context)
-            # If the client was configured with the wrong region,
-            # we have to build a new signer.
-            if context[:cached_sigv4_region] &&
-               context[:cached_sigv4_region] != context.config.sigv4_signer.region
-              S3Signer.build_v4_signer(
-                service: 's3',
-                region: context[:cached_sigv4_region],
-                credentials: context.config.credentials
-              )
-            else
-              resolved_region, arn = ARN.resolve_arn!(
-                context.params[:bucket],
-                context.config.sigv4_signer.region,
-                context.config.s3_use_arn_region
-              )
-
-              if arn
-                S3Signer.build_v4_signer(
-                  service: arn.service,
-                  region: resolved_region,
-                  credentials: context.config.credentials
-                )
-              else
-                context.config.sigv4_signer
-              end
-            end
-          end
-        end
-
         # This handler will update the http endpoint when the bucket region
         # is known/cached.
         class CachedBucketRegionHandler < Seahorse::Client::Handler
@@ -106,11 +52,13 @@ module Aws
 
           def check_for_cached_region(context, bucket)
             cached_region = S3::BUCKET_REGIONS[bucket]
-            if cached_region && cached_region != context.config.region
+            if cached_region &&
+               cached_region != context.config.region &&
+               !S3Signer.custom_endpoint?(context)
               context.http_request.endpoint.host = S3Signer.new_hostname(
                 context, cached_region
               )
-              context[:cached_sigv4_region] = cached_region
+              context[:sigv4_region] = cached_region # Sign plugin will use this
             end
           end
         end
@@ -118,7 +66,8 @@ module Aws
         # This handler detects when a request fails because of a mismatched bucket
         # region. It follows up by making a request to determine the correct
         # region, then finally a version 4 signed request against the correct
-        # regional endpoint.
+        # regional endpoint. This is intended for s3's global endpoint which
+        # will return 400 if the bucket is not in region.
         class BucketRegionErrorHandler < Seahorse::Client::Handler
           def call(context)
             response = @handler.call(context)
@@ -130,7 +79,8 @@ module Aws
           def handle_region_errors(response)
             if wrong_sigv4_region?(response) &&
                !fips_region?(response) &&
-               !custom_endpoint?(response)
+               !S3Signer.custom_endpoint?(response.context) &&
+               !expired_credentials?(response)
               get_region_and_retry(response.context)
             else
               response
@@ -151,14 +101,11 @@ module Aws
           end
 
           def fips_region?(resp)
-            resp.context.http_request.endpoint.host.include?('fips')
+            resp.context.http_request.endpoint.host.include?('s3-fips.')
           end
 
-          def custom_endpoint?(resp)
-            resolved_suffix = Aws::Partitions::EndpointProvider.dns_suffix_for(
-              resp.context.config.region
-            )
-            !resp.context.http_request.endpoint.hostname.include?(resolved_suffix)
+          def expired_credentials?(resp)
+            resp.context.http_response.body_contents.match(/<Code>ExpiredToken<\/Code>/)
           end
 
           def wrong_sigv4_region?(resp)
@@ -173,14 +120,14 @@ module Aws
               context, actual_region
             )
             context.metadata[:redirect_region] = actual_region
-            Aws::Plugins::SignatureV4.apply_signature(
-              context: context,
-              signer: S3Signer.build_v4_signer(
-                service: 's3',
-                region: actual_region,
-                credentials: context.config.credentials
-              )
+
+            signer = Aws::Plugins::Sign.signer_for(
+              context[:auth_scheme],
+              context.config,
+              actual_region
             )
+
+            signer.sign(context)
           end
 
           def region_from_body(body)
@@ -206,36 +153,22 @@ module Aws
         end
 
         class << self
-          # @option options [required, String] :region
-          # @option options [required, #credentials] :credentials
-          # @api private
-          def build_v4_signer(options = {})
-            Aws::Sigv4::Signer.new(
-              service: options[:service],
-              region: options[:region],
-              credentials_provider: options[:credentials],
-              uri_escape_path: false,
-              unsigned_headers: ['content-length', 'x-amzn-trace-id']
-            )
+          def new_hostname(context, region)
+            endpoint_params = context[:endpoint_params].dup
+            endpoint_params.region = region
+            endpoint_params.endpoint = nil
+            endpoint =
+              context.config.endpoint_provider.resolve_endpoint(endpoint_params)
+            URI(endpoint.url).host
           end
 
-          def new_hostname(context, region)
-            # Check to see if the bucket is actually an ARN and resolve it
-            # Otherwise it will retry with the ARN as the bucket name.
-            resolved_region, arn = ARN.resolve_arn!(
-              context.params[:bucket],
-              region,
-              context.config.s3_use_arn_region
-            )
-            uri = URI.parse(
-              Aws::Partitions::EndpointProvider.resolve(resolved_region, 's3')
-            )
+          def custom_endpoint?(context)
+            region = context.config.region
+            partition = Aws::Endpoints::Matchers.aws_partition(region)
+            endpoint = context.http_request.endpoint
 
-            if arn
-              ARN.resolve_url!(uri, arn).host
-            else
-              "#{context.params[:bucket]}.#{uri.host}"
-            end
+            !endpoint.hostname.include?(partition['dnsSuffix']) &&
+              !endpoint.hostname.include?(partition['dualStackDnsSuffix'])
           end
         end
       end

data/lib/aws-sdk-s3/plugins/skip_whole_multipart_get_checksums.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+
+      # S3 GetObject results for whole Multipart Objects contain a checksum
+      # that cannot be validated.  These should be skipped by the
+      # ChecksumAlgorithm plugin.
+      class SkipWholeMultipartGetChecksums < Seahorse::Client::Plugin
+
+        class Handler < Seahorse::Client::Handler
+
+          def call(context)
+            context[:http_checksum] ||= {}
+            context[:http_checksum][:skip_on_suffix] = true
+
+            @handler.call(context)
+          end
+
+        end
+
+        handler(
+          Handler,
+          step: :initialize,
+          operations: [:get_object]
+        )
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/streaming_retry.rb

@@ -36,6 +36,17 @@ module Aws
         def rewind; end
       end
 
+      class NonRetryableStreamingError < StandardError
+
+        def initialize(error)
+          super('Unable to retry request - retry could result in processing duplicated chunks.')
+          set_backtrace(error.backtrace)
+          @original_error = error
+        end
+
+        attr_reader :original_error
+      end
+
       # This handler works with the ResponseTarget plugin to provide smart
       # retries of S3 streaming operations that support the range parameter
       # (currently only: get_object).  When a 200 OK with a TruncatedBodyError
@@ -84,8 +95,18 @@ module Aws
             end
 
             context.http_response.on_error do |error|
-              if retryable_body?(context) && truncated_body?(error)
-                context.http_request.headers[:range] = "bytes=#{context.http_response.body.size}-"
+              if retryable_body?(context)
+                if truncated_body?(error)
+                  context.http_request.headers[:range] = "bytes=#{context.http_response.body.size}-"
+                else
+                  case context.http_response.body
+                  when RetryableManagedFile
+                    # call rewind on the underlying file
+                    context.http_response.body.instance_variable_get(:@file).rewind
+                  else
+                    raise NonRetryableStreamingError, error
+                  end
+                end
               end
             end
           end