aws-sdk-s3 1.79.1 → 1.212.0

data/lib/aws-sdk-s3/plugins/get_bucket_location_fix.rb

@@ -11,7 +11,7 @@ module Aws
             @handler.call(context).on(200) do |response|
               response.data = S3::Types::GetBucketLocationOutput.new
               xml = context.http_response.body_contents
-              matches = xml.match(/>(.+?)<\/LocationConstraint>/)
+              matches = xml.match(/<LocationConstraint.*?>(.+?)<\/LocationConstraint>/)
               response.data[:location_constraint] = matches ? matches[1] : ''
             end
           end

data/lib/aws-sdk-s3/plugins/http_200_errors.rb

@@ -3,52 +3,113 @@
 module Aws
   module S3
     module Plugins
-
       # A handful of Amazon S3 operations will respond with a 200 status
       # code but will send an error in the response body. This plugin
       # injects a handler that will parse 200 response bodies for potential
       # errors, allowing them to be retried.
       # @api private
       class Http200Errors < Seahorse::Client::Plugin
-
         class Handler < Seahorse::Client::Handler
+          # A regular expression to match error codes in the response body
+          CODE_PATTERN = %r{<Code>(.+?)</Code>}.freeze
+          private_constant :CODE_PATTERN
+
+          # A list of encodings we force into UTF-8
+          ENCODINGS_TO_FIX = [Encoding::US_ASCII, Encoding::ASCII_8BIT].freeze
+          private_constant :ENCODINGS_TO_FIX
+
+          # A regular expression to match detect errors in the response body
+          ERROR_PATTERN = /<\?xml\s[^>]*\?>\s*<Error>/.freeze
+          private_constant :ERROR_PATTERN
+
+          # A regular expression to match an error message in the response body
+          MESSAGE_PATTERN = %r{<Message>(.+?)</Message>}.freeze
+          private_constant :MESSAGE_PATTERN
 
           def call(context)
             @handler.call(context).on(200) do |response|
-              if error = check_for_error(context)
-                context.http_response.status_code = 500
-                response.data = nil
-                response.error = error
-              end
+              return response if streaming_output?(context.operation.output)
+
+              error = check_for_error(context)
+              return response unless error
+
+              context.http_response.status_code = 500
+              response.data = nil
+              response.error = error
             end
           end
 
+          private
+
+          def build_error(context, code, message)
+            S3::Errors.error_class(code).new(context, message)
+          end
+
           def check_for_error(context)
-            xml = context.http_response.body_contents
-            if xml.match(/<Error>/)
-              error_code = xml.match(/<Code>(.+?)<\/Code>/)[1]
-              error_message = xml.match(/<Message>(.+?)<\/Message>/)[1]
-              S3::Errors.error_class(error_code).new(context, error_message)
-            elsif !xml.match(/<\w/) # Must have the start of an XML Tag
-              # Other incomplete xml bodies will result in XML ParsingError
+            xml = normalize_encoding(context.http_response.body_contents)
+
+            if xml.match?(ERROR_PATTERN)
+              error_code = xml.match(CODE_PATTERN)[1]
+              error_message = xml.match(MESSAGE_PATTERN)[1]
+              build_error(context, error_code, error_message)
+            elsif incomplete_xml_body?(xml, context.operation.output)
               Seahorse::Client::NetworkingError.new(
-                S3::Errors
-                  .error_class('InternalError')
-                  .new(context, 'Empty or incomplete response body')
+                build_error(context, 'InternalError', 'Empty or incomplete response body')
               )
             end
           end
+
+          # Must have a member in the body and have the start of an XML Tag.
+          # Other incomplete xml bodies will result in an XML ParsingError.
+          def incomplete_xml_body?(xml, output)
+            members_in_body?(output) && !xml.match(/<\w/)
+          end
+
+          # Checks if the output shape is a structure shape and has members that
+          # are in the body for the case of a payload and a normal structure. A
+          # non-structure shape will not have members in the body. In the case
+          # of a string or blob, the body contents would have been checked first
+          # before this method is called in incomplete_xml_body?.
+          def members_in_body?(output)
+            shape = resolve_shape(output)
+
+            if structure_shape?(shape)
+              shape.members.any? { |_, k| k.location.nil? }
+            else
+              false
+            end
+          end
+
+          # Fixes encoding issues when S3 returns UTF-8 content with missing charset in Content-Type header or omits
+          # Content-Type header entirely.  Net::HTTP defaults to US-ASCII or ASCII-8BIT when charset is unspecified.
+          def normalize_encoding(xml)
+            return xml unless xml.is_a?(String) && ENCODINGS_TO_FIX.include?(xml.encoding)
+
+            xml.force_encoding('UTF-8')
+          end
+
+          def resolve_shape(output)
+            return output.shape unless output[:payload_member]
+
+            output[:payload_member].shape
+          end
+
+          # Streaming outputs are not subject to 200 errors.
+          def streaming_output?(output)
+            if (payload = output[:payload_member])
+              # checking ref and shape
+              payload['streaming'] || payload.shape['streaming'] || payload.eventstream
+            else
+              false
+            end
+          end
+
+          def structure_shape?(shape)
+            shape.is_a?(Seahorse::Model::Shapes::StructureShape)
+          end
         end
 
-        handler(
-          Handler,
-          step: :sign,
-          operations: [
-            :complete_multipart_upload,
-            :copy_object,
-            :upload_part_copy,
-          ]
-        )
+        handler(Handler, step: :sign)
       end
     end
   end

data/lib/aws-sdk-s3/plugins/iad_regional_endpoint.rb

@@ -10,41 +10,23 @@ module Aws
           default: 'legacy',
           doc_type: String,
           docstring: <<-DOCS) do |cfg|
-Passing in `regional` to enable regional endpoint for S3's `us-east-1`
-region. Defaults to `legacy` mode using global endpoint.
+Pass in `regional` to enable the `us-east-1` regional endpoint.
+Defaults to `legacy` mode which uses the global endpoint.
           DOCS
           resolve_iad_regional_endpoint(cfg)
         end
 
-        def add_handlers(handlers, config)
-          if config.region == 'us-east-1'
-            handlers.add(Handler)
-          end
-        end
-
-        # @api private
-        class Handler < Seahorse::Client::Handler
-
-          def call(context)
-            # keep legacy global endpoint pattern by default
-            if context.config.s3_us_east_1_regional_endpoint == 'legacy'
-              context.http_request.endpoint.host = IADRegionalEndpoint.legacy_host(
-                context.http_request.endpoint.host)
-            end
-            @handler.call(context)
-          end
-
-        end
-
-        def self.legacy_host(host)
-          host.sub(".us-east-1", '')
-        end
-
         private
 
         def self.resolve_iad_regional_endpoint(cfg)
+          default_mode_value =
+            if cfg.respond_to?(:defaults_mode_config_resolver)
+              cfg.defaults_mode_config_resolver.resolve(:s3_us_east_1_regional_endpoint)
+            end
+
           mode = ENV['AWS_S3_US_EAST_1_REGIONAL_ENDPOINT'] ||
             Aws.shared_config.s3_us_east_1_regional_endpoint(profile: cfg.profile) ||
+            default_mode_value ||
             'legacy'
           mode = mode.downcase
           unless %w(legacy regional).include?(mode)

data/lib/aws-sdk-s3/plugins/location_constraint.rb

@@ -22,7 +22,9 @@ module Aws
 
           def populate_location_constraint(params, region)
             params[:create_bucket_configuration] ||= {}
-            params[:create_bucket_configuration][:location_constraint] ||= region
+            unless params[:create_bucket_configuration][:location]
+              params[:create_bucket_configuration][:location_constraint] ||= region
+            end
           end
 
         end

data/lib/aws-sdk-s3/plugins/md5s.rb

@@ -6,78 +6,20 @@ module Aws
   module S3
     module Plugins
       # @api private
-      # This plugin is effectively deprecated in favor of modeled
+      # This plugin is deprecated in favor of modeled
       # httpChecksumRequired traits.
       class Md5s < Seahorse::Client::Plugin
-        # These operations allow Content MD5 but are not required by
-        # httpChecksumRequired. This list should not grow.
-        OPTIONAL_OPERATIONS = [
-          :put_object,
-          :upload_part
-        ]
-
-        # @api private
-        class Handler < Seahorse::Client::Handler
-
-          CHUNK_SIZE = 1 * 1024 * 1024 # one MB
-
-          def call(context)
-            body = context.http_request.body
-            if body.size > 0
-              context.http_request.headers['Content-Md5'] ||= md5(body)
-            end
-            @handler.call(context)
-          end
-
-          private
-
-          # @param [File, Tempfile, IO#read, String] value
-          # @return [String<MD5>]
-          def md5(value)
-            if (File === value || Tempfile === value) && !value.path.nil? && File.exist?(value.path)
-              OpenSSL::Digest::MD5.file(value).base64digest
-            elsif value.respond_to?(:read)
-              md5 = OpenSSL::Digest::MD5.new
-              update_in_chunks(md5, value)
-              md5.base64digest
-            else
-              OpenSSL::Digest::MD5.digest(value).base64digest
-            end
-          end
-
-          def update_in_chunks(digest, io)
-            loop do
-              chunk = io.read(CHUNK_SIZE)
-              break unless chunk
-              digest.update(chunk)
-            end
-            io.rewind
-          end
-
-        end
-
         option(:compute_checksums,
-          default: true,
-          doc_type: 'Boolean',
-          docstring: <<-DOCS)
-When `true` a MD5 checksum will be computed and sent in the Content Md5
-header for :put_object and :upload_part. When `false`, MD5 checksums
-will not be computed for these operations. Checksums are still computed
-for operations requiring them. Checksum errors returned by Amazon S3 are
-automatically retried up to `:retry_limit` times.
-          DOCS
-
-        def add_handlers(handlers, config)
-          if config.compute_checksums
-            # priority set low to ensure md5 is computed AFTER the request is
-            # built but before it is signed
-            handlers.add(
-              Handler,
-              priority: 10, step: :build, operations: OPTIONAL_OPERATIONS
-            )
-          end
+               default: true,
+               doc_type: 'Boolean',
+               docstring: <<~DOCS)
+                 This option is deprecated. Please use `:request_checksum_calculation` instead.
+                 When `false`, `request_checksum_calculation` is overridden to `when_required`.
+               DOCS
+
+        def after_initialize(client)
+          client.config.request_checksum_calculation = 'when_required' unless client.config.compute_checksums
         end
-
       end
     end
   end

data/lib/aws-sdk-s3/plugins/s3_signer.rb

@@ -4,25 +4,19 @@ require 'aws-sigv4'
 
 module Aws
   module S3
+    # @api private
+    def self.bucket_region_cache
+      @bucket_region_cache ||= BucketRegionCache.new
+    end
+
     module Plugins
-      # This plugin is an implementation detail and may be modified.
+      # This plugin used to have a V4 signer but it was removed in favor of
+      # generic Sign plugin that uses endpoint auth scheme.
+      #
       # @api private
       class S3Signer < Seahorse::Client::Plugin
         option(:signature_version, 'v4')
 
-        option(:sigv4_signer) do |cfg|
-          S3Signer.build_v4_signer(
-            region: cfg.sigv4_region,
-            credentials: cfg.credentials
-          )
-        end
-
-        option(:sigv4_region) do |cfg|
-          raise Aws::Errors::MissingRegionError if cfg.region.nil?
-
-          Aws::Partitions::EndpointProvider.signing_region(cfg.region, 's3')
-        end
-
         def add_handlers(handlers, cfg)
           case cfg.signature_version
           when 'v4' then add_v4_handlers(handlers)
@@ -35,11 +29,11 @@ module Aws
 
         def add_v4_handlers(handlers)
           handlers.add(CachedBucketRegionHandler, step: :sign, priority: 60)
-          handlers.add(V4Handler, step: :sign)
           handlers.add(BucketRegionErrorHandler, step: :sign, priority: 40)
         end
 
         def add_legacy_handler(handlers)
+          # generic Sign plugin will be skipped if it sees sigv2
           handlers.add(LegacyHandler, step: :sign)
         end
 
@@ -50,32 +44,6 @@ module Aws
           end
         end
 
-        class V4Handler < Seahorse::Client::Handler
-          def call(context)
-            Aws::Plugins::SignatureV4.apply_signature(
-              context: context,
-              signer: sigv4_signer(context)
-            )
-            @handler.call(context)
-          end
-
-          private
-
-          def sigv4_signer(context)
-            # If the client was configured with the wrong region,
-            # we have to build a new signer.
-            if context[:cached_sigv4_region] &&
-               context[:cached_sigv4_region] != context.config.sigv4_signer.region
-              S3Signer.build_v4_signer(
-                region: context[:cached_sigv4_region],
-                credentials: context.config.credentials
-              )
-            else
-              context.config.sigv4_signer
-            end
-          end
-        end
-
         # This handler will update the http endpoint when the bucket region
         # is known/cached.
         class CachedBucketRegionHandler < Seahorse::Client::Handler
@@ -88,10 +56,14 @@ module Aws
           private
 
           def check_for_cached_region(context, bucket)
-            cached_region = S3::BUCKET_REGIONS[bucket]
-            if cached_region && cached_region != context.config.region
-              context.http_request.endpoint.host = S3Signer.new_hostname(context, cached_region)
-              context[:cached_sigv4_region] = cached_region
+            cached_region = Aws::S3.bucket_region_cache[bucket]
+            if cached_region &&
+               cached_region != context.config.region &&
+               !S3Signer.custom_endpoint?(context)
+              context.http_request.endpoint.host = S3Signer.new_hostname(
+                context, cached_region
+              )
+              context[:sigv4_region] = cached_region # Sign plugin will use this
             end
           end
         end
@@ -99,7 +71,8 @@ module Aws
         # This handler detects when a request fails because of a mismatched bucket
         # region. It follows up by making a request to determine the correct
         # region, then finally a version 4 signed request against the correct
-        # regional endpoint.
+        # regional endpoint. This is intended for s3's global endpoint which
+        # will return 400 if the bucket is not in region.
         class BucketRegionErrorHandler < Seahorse::Client::Handler
           def call(context)
             response = @handler.call(context)
@@ -111,7 +84,8 @@ module Aws
           def handle_region_errors(response)
             if wrong_sigv4_region?(response) &&
                !fips_region?(response) &&
-               !custom_endpoint?(response)
+               !S3Signer.custom_endpoint?(response.context) &&
+               !expired_credentials?(response)
               get_region_and_retry(response.context)
             else
               response
@@ -128,18 +102,15 @@ module Aws
           end
 
           def update_bucket_cache(context, actual_region)
-            S3::BUCKET_REGIONS[context.params[:bucket]] = actual_region
+            Aws::S3.bucket_region_cache[context.params[:bucket]] = actual_region
           end
 
           def fips_region?(resp)
-            resp.context.http_request.endpoint.host.include?('fips')
+            resp.context.http_request.endpoint.host.include?('s3-fips.')
           end
 
-          def custom_endpoint?(resp)
-            resolved_suffix = Aws::Partitions::EndpointProvider.dns_suffix_for(
-              resp.context.config.region
-            )
-            !resp.context.http_request.endpoint.hostname.include?(resolved_suffix)
+          def expired_credentials?(resp)
+            resp.context.http_response.body_contents.match(/<Code>ExpiredToken<\/Code>/)
           end
 
           def wrong_sigv4_region?(resp)
@@ -150,15 +121,18 @@ module Aws
 
           def resign_with_new_region(context, actual_region)
             context.http_response.body.truncate(0)
-            context.http_request.endpoint.host = S3Signer.new_hostname(context, actual_region)
+            context.http_request.endpoint.host = S3Signer.new_hostname(
+              context, actual_region
+            )
             context.metadata[:redirect_region] = actual_region
-            Aws::Plugins::SignatureV4.apply_signature(
-              context: context,
-              signer: S3Signer.build_v4_signer(
-                region: actual_region,
-                credentials: context.config.credentials
-              )
+
+            signer = Aws::Plugins::Sign.signer_for(
+              context[:auth_scheme],
+              context.config,
+              actual_region
             )
+
+            signer.sign(context)
           end
 
           def region_from_body(body)
@@ -184,36 +158,22 @@ module Aws
         end
 
         class << self
-          # @option options [required, String] :region
-          # @option options [required, #credentials] :credentials
-          # @api private
-          def build_v4_signer(options = {})
-            Aws::Sigv4::Signer.new(
-              service: 's3',
-              region: options[:region],
-              credentials_provider: options[:credentials],
-              uri_escape_path: false,
-              unsigned_headers: ['content-length', 'x-amzn-trace-id']
-            )
+          def new_hostname(context, region)
+            endpoint_params = context[:endpoint_params].dup
+            endpoint_params.region = region
+            endpoint_params.endpoint = nil
+            endpoint =
+              context.config.endpoint_provider.resolve_endpoint(endpoint_params)
+            URI(endpoint.url).host
           end
 
-          def new_hostname(context, region)
-            # Check to see if the bucket is actually an ARN and resolve it
-            # Otherwise it will retry with the ARN as the bucket name.
-            resolved_bucket, resolved_region, arn = BucketARN.resolve_arn!(
-              context.params[:bucket],
-              region,
-              context.config.s3_use_arn_region
-            )
-            uri = URI.parse(
-              Aws::Partitions::EndpointProvider.resolve(resolved_region, 's3')
-            )
+          def custom_endpoint?(context)
+            region = context.config.region
+            partition = Aws::Endpoints::Matchers.aws_partition(region)
+            endpoint = context.http_request.endpoint
 
-            if arn
-              BucketARN.resolve_url!(uri, arn).host
-            else
-              resolved_bucket + '.' + uri.host
-            end
+            !endpoint.hostname.include?(partition['dnsSuffix']) &&
+              !endpoint.hostname.include?(partition['dualStackDnsSuffix'])
           end
         end
       end

data/lib/aws-sdk-s3/plugins/streaming_retry.rb

@@ -36,6 +36,17 @@ module Aws
         def rewind; end
       end
 
+      class NonRetryableStreamingError < StandardError
+
+        def initialize(error)
+          super('Unable to retry request - retry could result in processing duplicated chunks.')
+          set_backtrace(error.backtrace)
+          @original_error = error
+        end
+
+        attr_reader :original_error
+      end
+
       # This handler works with the ResponseTarget plugin to provide smart
       # retries of S3 streaming operations that support the range parameter
       # (currently only: get_object).  When a 200 OK with a TruncatedBodyError
@@ -51,18 +62,16 @@ module Aws
         class Handler < Seahorse::Client::Handler
 
           def call(context)
-            target = context.params[:response_target] || context[:response_target]
-
             # retry is only supported when range is NOT set on the initial request
-            if supported_target?(target) && !context.params[:range]
-              add_event_listeners(context, target)
+            if supported_target?(context) && !context.params[:range]
+              add_event_listeners(context)
             end
             @handler.call(context)
           end
 
           private
 
-          def add_event_listeners(context, target)
+          def add_event_listeners(context)
             context.http_response.on_headers(200..299) do
               case context.http_response.body
               when Seahorse::Client::BlockIO then
@@ -84,8 +93,18 @@ module Aws
             end
 
             context.http_response.on_error do |error|
-              if retryable_body?(context) && truncated_body?(error)
-                context.http_request.headers[:range] = "bytes=#{context.http_response.body.size}-"
+              if retryable_body?(context)
+                if truncated_body?(error)
+                  context.http_request.headers[:range] = "bytes=#{context.http_response.body.size}-"
+                else
+                  case context.http_response.body
+                  when RetryableManagedFile
+                    # call rewind on the underlying file
+                    context.http_response.body.instance_variable_get(:@file).rewind
+                  else
+                    raise NonRetryableStreamingError, error
+                  end
+                end
               end
             end
           end
@@ -102,8 +121,8 @@ module Aws
               context.http_response.body.is_a?(RetryableManagedFile)
           end
 
-          def supported_target?(target)
-            case target
+          def supported_target?(context)
+            case context[:response_target]
             when Proc, String, Pathname then true
             else false
             end

data/lib/aws-sdk-s3/plugins/url_encoded_keys.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 require 'uri'
-require 'cgi'
+require "cgi/escape"
+require "cgi/util" if RUBY_VERSION < "3.5"
 
 module Aws
   module S3