aws-sdk-s3 1.48.0 → 1.169.0

data/lib/aws-sdk-s3/plugins/express_session_auth.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+      # @api private
+      class ExpressSessionAuth < Seahorse::Client::Plugin
+        # This should be s3_disable_express_auth instead
+        # But this is not a built in. We're overwriting the generated value
+        option(:disable_s3_express_session_auth,
+          default: false,
+          doc_type: 'Boolean',
+          docstring: <<-DOCS) do |cfg|
+When `true`, S3 Express session authentication is disabled.
+          DOCS
+          resolve_disable_s3_express_session_auth(cfg)
+        end
+
+        option(:express_credentials_provider,
+          doc_type: 'Aws::S3::ExpressCredentialsProvider',
+          rbs_type: 'untyped',
+          docstring: <<-DOCS) do |_cfg|
+Credential Provider for S3 Express endpoints. Manages credentials
+for different buckets.
+          DOCS
+          Aws::S3::ExpressCredentialsProvider.new
+        end
+
+        # @api private
+        class Handler < Seahorse::Client::Handler
+          def call(context)
+            if (props = context[:endpoint_properties])
+              # S3 Express endpoint - turn off md5 and enable crc32 default
+              if props['backend'] == 'S3Express'
+                if context.operation_name == :put_object || checksum_required?(context)
+                  context[:default_request_checksum_algorithm] = 'CRC32'
+                end
+                context[:s3_express_endpoint] = true
+              end
+
+              # if s3 express auth, use new credentials and sign additional header
+              if context[:auth_scheme]['name'] == 'sigv4-s3express' &&
+                 !context.config.disable_s3_express_session_auth
+                bucket = context.params[:bucket]
+                credentials_provider = context.config.express_credentials_provider
+                credentials = credentials_provider.express_credentials_for(bucket)
+                context[:sigv4_credentials] = credentials # Sign will use this
+              end
+            end
+            with_metric(credentials) { @handler.call(context) }
+          end
+
+          private
+
+          def with_metric(credentials, &block)
+            return block.call unless credentials
+
+            Aws::Plugins::UserAgent.metric('S3_EXPRESS_BUCKET', &block)
+          end
+
+          def checksum_required?(context)
+            context.operation.http_checksum_required ||
+              (context.operation.http_checksum &&
+                context.operation.http_checksum['requestChecksumRequired'])
+          end
+        end
+
+        handler(Handler)
+
+        # Optimization - sets this client as the client to create sessions.
+        def after_initialize(client)
+          provider = client.config.express_credentials_provider
+          provider.client = client unless provider.client
+        end
+
+        class << self
+          private
+
+          def resolve_disable_s3_express_session_auth(cfg)
+            value = ENV['AWS_S3_DISABLE_EXPRESS_SESSION_AUTH'] ||
+                    Aws.shared_config.s3_disable_express_session_auth(profile: cfg.profile) ||
+                    'false'
+            value = Aws::Util.str_2_bool(value)
+            # Raise if provided value is not true or false
+            if value.nil?
+              raise ArgumentError,
+                    'Must provide either `true` or `false` for the '\
+                    '`s3_disable_express_session_auth` profile option or for '\
+                    "ENV['AWS_S3_DISABLE_EXPRESS_SESSION_AUTH']."
+            end
+            value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/get_bucket_location_fix.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     module Plugins
@@ -9,7 +11,7 @@ module Aws
             @handler.call(context).on(200) do |response|
               response.data = S3::Types::GetBucketLocationOutput.new
               xml = context.http_response.body_contents
-              matches = xml.match(/>(.+?)<\/LocationConstraint>/)
+              matches = xml.match(/<LocationConstraint.*?>(.+?)<\/LocationConstraint>/)
               response.data[:location_constraint] = matches ? matches[1] : ''
             end
           end

data/lib/aws-sdk-s3/plugins/http_200_errors.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     module Plugins
@@ -13,34 +15,77 @@ module Aws
 
           def call(context)
             @handler.call(context).on(200) do |response|
-              if error = check_for_error(context)
-                context.http_response.status_code = 500
-                response.data = nil
-                response.error = error
+              return response if streaming_output?(context.operation.output)
+
+              error = check_for_error(context)
+              return response unless error
+
+              context.http_response.status_code = 500
+              response.data = nil
+              response.error = error
+            end
+          end
+
+          private
+
+          # Streaming outputs are not subject to 200 errors.
+          def streaming_output?(output)
+            if (payload = output[:payload_member])
+              # checking ref and shape
+              payload['streaming'] || payload.shape['streaming'] ||
+                payload.eventstream
+            else
+              false
+            end
+          end
+
+          # Checks if the output shape is a structure shape and has members that
+          # are in the body for the case of a payload and a normal structure. A
+          # non-structure shape will not have members in the body. In the case
+          # of a string or blob, the body contents would have been checked first
+          # before this method is called in incomplete_xml_body?.
+          def members_in_body?(output)
+            shape =
+              if output[:payload_member]
+                output[:payload_member].shape
+              else
+                output.shape
               end
+
+            if structure_shape?(shape)
+              shape.members.any? { |_, k| k.location.nil? }
+            else
+              false
             end
           end
 
+          def structure_shape?(shape)
+            shape.is_a?(Seahorse::Model::Shapes::StructureShape)
+          end
+
+          # Must have a member in the body and have the start of an XML Tag.
+          # Other incomplete xml bodies will result in an XML ParsingError.
+          def incomplete_xml_body?(xml, output)
+            members_in_body?(output) && !xml.match(/<\w/)
+          end
+
           def check_for_error(context)
             xml = context.http_response.body_contents
-            if xml.match(/<Error>/)
+            if xml.match(/\?>\s*<Error>/)
               error_code = xml.match(/<Code>(.+?)<\/Code>/)[1]
               error_message = xml.match(/<Message>(.+?)<\/Message>/)[1]
               S3::Errors.error_class(error_code).new(context, error_message)
+            elsif incomplete_xml_body?(xml, context.operation.output)
+              Seahorse::Client::NetworkingError.new(
+                S3::Errors
+                  .error_class('InternalError')
+                  .new(context, 'Empty or incomplete response body')
+              )
             end
           end
-
         end
 
-        handler(Handler,
-          step: :sign,
-          operations: [
-            :complete_multipart_upload,
-            :copy_object,
-            :upload_part_copy,
-          ]
-        )
-
+        handler(Handler, step: :sign)
       end
     end
   end

data/lib/aws-sdk-s3/plugins/iad_regional_endpoint.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+
+      class IADRegionalEndpoint < Seahorse::Client::Plugin
+
+        option(:s3_us_east_1_regional_endpoint,
+          default: 'legacy',
+          doc_type: String,
+          docstring: <<-DOCS) do |cfg|
+Pass in `regional` to enable the `us-east-1` regional endpoint.
+Defaults to `legacy` mode which uses the global endpoint.
+          DOCS
+          resolve_iad_regional_endpoint(cfg)
+        end
+
+        private
+
+        def self.resolve_iad_regional_endpoint(cfg)
+          default_mode_value =
+            if cfg.respond_to?(:defaults_mode_config_resolver)
+              cfg.defaults_mode_config_resolver.resolve(:s3_us_east_1_regional_endpoint)
+            end
+
+          mode = ENV['AWS_S3_US_EAST_1_REGIONAL_ENDPOINT'] ||
+            Aws.shared_config.s3_us_east_1_regional_endpoint(profile: cfg.profile) ||
+            default_mode_value ||
+            'legacy'
+          mode = mode.downcase
+          unless %w(legacy regional).include?(mode)
+            raise ArgumentError, "expected :s3_us_east_1_regional_endpoint or"\
+              " ENV['AWS_S3_US_EAST_1_REGIONAL_ENDPOINT'] to be `legacy` or"\
+              " `regional`."
+          end
+          mode
+        end
+
+      end
+
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/location_constraint.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     module Plugins
@@ -20,7 +22,9 @@ module Aws
 
           def populate_location_constraint(params, region)
             params[:create_bucket_configuration] ||= {}
-            params[:create_bucket_configuration][:location_constraint] ||= region
+            unless params[:create_bucket_configuration][:location]
+              params[:create_bucket_configuration][:location_constraint] ||= region
+            end
           end
 
         end

data/lib/aws-sdk-s3/plugins/md5s.rb

@@ -1,21 +1,19 @@
+# frozen_string_literal: true
+
 require 'openssl'
-require 'base64'
 
 module Aws
   module S3
     module Plugins
+      # @api private
+      # This plugin is effectively deprecated in favor of modeled
+      # httpChecksumRequired traits.
       class Md5s < Seahorse::Client::Plugin
-
-        # Amazon S3 requires these operations to have an MD5 checksum
-        REQUIRED_OPERATIONS = [
-          :delete_objects,
-          :put_bucket_cors,
-          :put_bucket_lifecycle,
-          :put_bucket_policy,
-          :put_bucket_tagging,
-          :put_object_legal_hold,
-          :put_object_lock_configuration,
-          :put_object_retention
+        # These operations allow Content MD5 but are not required by
+        # httpChecksumRequired. This list should not grow.
+        OPTIONAL_OPERATIONS = [
+          :put_object,
+          :upload_part
         ]
 
         # @api private
@@ -24,9 +22,12 @@ module Aws
           CHUNK_SIZE = 1 * 1024 * 1024 # one MB
 
           def call(context)
-            body = context.http_request.body
-            if body.size > 0
-              context.http_request.headers['Content-Md5'] ||= md5(body)
+            if !context[:checksum_algorithms] && # skip in favor of flexible checksum
+               !context[:s3_express_endpoint] # s3 express endpoints do not support md5
+              body = context.http_request.body
+              if body.respond_to?(:size) && body.size > 0
+                context.http_request.headers['Content-Md5'] ||= md5(body)
+              end
             end
             @handler.call(context)
           end
@@ -37,18 +38,20 @@ module Aws
           # @return [String<MD5>]
           def md5(value)
             if (File === value || Tempfile === value) && !value.path.nil? && File.exist?(value.path)
-              Base64.encode64(OpenSSL::Digest::MD5.file(value).digest).strip
+              OpenSSL::Digest::MD5.file(value).base64digest
             elsif value.respond_to?(:read)
               md5 = OpenSSL::Digest::MD5.new
               update_in_chunks(md5, value)
-              Base64.encode64(md5.digest).strip
+              md5.base64digest
             else
-              Base64.encode64(OpenSSL::Digest::MD5.digest(value)).strip
+              OpenSSL::Digest::MD5.digest(value).base64digest
             end
           end
 
           def update_in_chunks(digest, io)
-            while chunk = io.read(CHUNK_SIZE, buffer ||= "")
+            loop do
+              chunk = io.read(CHUNK_SIZE)
+              break unless chunk
               digest.update(chunk)
             end
             io.rewind
@@ -60,20 +63,22 @@ module Aws
           default: true,
           doc_type: 'Boolean',
           docstring: <<-DOCS)
-When `true` a MD5 checksum will be computed for every request that
-sends a body.  When `false`, MD5 checksums will only be computed
-for operations that require them.  Checksum errors returned by Amazon
-S3 are automatically retried up to `:retry_limit` times.
+When `true` a MD5 checksum will be computed and sent in the Content Md5
+header for :put_object and :upload_part. When `false`, MD5 checksums
+will not be computed for these operations. Checksums are still computed
+for operations requiring them. Checksum errors returned by Amazon S3 are
+automatically retried up to `:retry_limit` times.
           DOCS
 
         def add_handlers(handlers, config)
-          # priority set low to ensure md5 is computed AFTER the request is
-          # built but before it is signed
-          handlers.add(Handler, {
-            priority: 10,
-            step: :build,
-            operations: config.compute_checksums ? nil : REQUIRED_OPERATIONS,
-          })
+          if config.compute_checksums
+            # priority set low to ensure md5 is computed AFTER the request is
+            # built but before it is signed
+            handlers.add(
+              Handler,
+              priority: 10, step: :build, operations: OPTIONAL_OPERATIONS
+            )
+          end
         end
 
       end

data/lib/aws-sdk-s3/plugins/redirects.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     module Plugins

data/lib/aws-sdk-s3/plugins/s3_host_id.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     module Plugins

data/lib/aws-sdk-s3/plugins/s3_signer.rb

@@ -1,25 +1,22 @@
+# frozen_string_literal: true
+
 require 'aws-sigv4'
 
 module Aws
   module S3
+    # @api private
+    def self.bucket_region_cache
+      @bucket_region_cache ||= BucketRegionCache.new
+    end
+
     module Plugins
-      # This plugin is an implementation detail and may be modified.
+      # This plugin used to have a V4 signer but it was removed in favor of
+      # generic Sign plugin that uses endpoint auth scheme.
+      #
       # @api private
       class S3Signer < Seahorse::Client::Plugin
-
         option(:signature_version, 'v4')
 
-        option(:sigv4_signer) do |cfg|
-          S3Signer.build_v4_signer(
-            region: cfg.sigv4_region,
-            credentials: cfg.credentials
-          )
-        end
-
-        option(:sigv4_region) do |cfg|
-          Aws::Partitions::EndpointProvider.signing_region(cfg.region, 's3')
-        end
-
         def add_handlers(handlers, cfg)
           case cfg.signature_version
           when 'v4' then add_v4_handlers(handlers)
@@ -32,11 +29,11 @@ module Aws
 
         def add_v4_handlers(handlers)
           handlers.add(CachedBucketRegionHandler, step: :sign, priority: 60)
-          handlers.add(V4Handler, step: :sign)
           handlers.add(BucketRegionErrorHandler, step: :sign, priority: 40)
         end
 
         def add_legacy_handler(handlers)
+          # generic Sign plugin will be skipped if it sees sigv2
           handlers.add(LegacyHandler, step: :sign)
         end
 
@@ -47,40 +44,9 @@ module Aws
           end
         end
 
-        class V4Handler < Seahorse::Client::Handler
-
-          def call(context)
-            Aws::Plugins::SignatureV4.apply_signature(
-              context: context,
-              signer: sigv4_signer(context)
-            )
-            @handler.call(context)
-          end
-
-          private
-
-          def sigv4_signer(context)
-            # If the client was configured with the wrong region,
-            # we have to build a new signer.
-            if
-              context[:cached_sigv4_region] &&
-              context[:cached_sigv4_region] != context.config.sigv4_signer.region
-            then
-              S3Signer.build_v4_signer(
-                region: context[:cached_sigv4_region],
-                credentials: context.config.credentials
-              )
-            else
-              context.config.sigv4_signer
-            end
-          end
-
-        end
-
         # This handler will update the http endpoint when the bucket region
         # is known/cached.
         class CachedBucketRegionHandler < Seahorse::Client::Handler
-
           def call(context)
             bucket = context.params[:bucket]
             check_for_cached_region(context, bucket) if bucket
@@ -90,21 +56,24 @@ module Aws
           private
 
           def check_for_cached_region(context, bucket)
-            cached_region = S3::BUCKET_REGIONS[bucket]
-            if cached_region && cached_region != context.config.region
-              context.http_request.endpoint.host = S3Signer.new_hostname(context, cached_region)
-              context[:cached_sigv4_region] = cached_region
+            cached_region = Aws::S3.bucket_region_cache[bucket]
+            if cached_region &&
+               cached_region != context.config.region &&
+               !S3Signer.custom_endpoint?(context)
+              context.http_request.endpoint.host = S3Signer.new_hostname(
+                context, cached_region
+              )
+              context[:sigv4_region] = cached_region # Sign plugin will use this
             end
           end
-
         end
 
         # This handler detects when a request fails because of a mismatched bucket
         # region. It follows up by making a request to determine the correct
         # region, then finally a version 4 signed request against the correct
-        # regional endpoint.
+        # regional endpoint. This is intended for s3's global endpoint which
+        # will return 400 if the bucket is not in region.
         class BucketRegionErrorHandler < Seahorse::Client::Handler
-
           def call(context)
             response = @handler.call(context)
             handle_region_errors(response)
@@ -113,7 +82,10 @@ module Aws
           private
 
           def handle_region_errors(response)
-            if wrong_sigv4_region?(response) && !fips_region?(response)
+            if wrong_sigv4_region?(response) &&
+               !fips_region?(response) &&
+               !S3Signer.custom_endpoint?(response.context) &&
+               !expired_credentials?(response)
               get_region_and_retry(response.context)
             else
               response
@@ -130,37 +102,42 @@ module Aws
           end
 
           def update_bucket_cache(context, actual_region)
-            S3::BUCKET_REGIONS[context.params[:bucket]] = actual_region
+            Aws::S3.bucket_region_cache[context.params[:bucket]] = actual_region
           end
 
           def fips_region?(resp)
-            resp.context.http_request.endpoint.host.include?('fips')
+            resp.context.http_request.endpoint.host.include?('s3-fips.')
+          end
+
+          def expired_credentials?(resp)
+            resp.context.http_response.body_contents.match(/<Code>ExpiredToken<\/Code>/)
           end
 
           def wrong_sigv4_region?(resp)
             resp.context.http_response.status_code == 400 &&
-            (
-              resp.context.http_response.headers['x-amz-bucket-region'] ||
-              resp.context.http_response.body_contents.match(/<Region>.+?<\/Region>/)
-            )
+              (resp.context.http_response.headers['x-amz-bucket-region'] ||
+               resp.context.http_response.body_contents.match(/<Region>.+?<\/Region>/))
           end
 
           def resign_with_new_region(context, actual_region)
             context.http_response.body.truncate(0)
-            context.http_request.endpoint.host = S3Signer.new_hostname(context, actual_region)
+            context.http_request.endpoint.host = S3Signer.new_hostname(
+              context, actual_region
+            )
             context.metadata[:redirect_region] = actual_region
-            Aws::Plugins::SignatureV4.apply_signature(
-              context: context,
-              signer: S3Signer.build_v4_signer(
-                region: actual_region,
-                credentials: context.config.credentials
-              )
+
+            signer = Aws::Plugins::Sign.signer_for(
+              context[:auth_scheme],
+              context.config,
+              actual_region
             )
+
+            signer.sign(context)
           end
 
           def region_from_body(body)
             region = body.match(/<Region>(.+?)<\/Region>/)[1]
-            if region.nil? || region == ""
+            if region.nil? || region == ''
               raise "couldn't get region from body: #{body}"
             else
               region
@@ -168,44 +145,36 @@ module Aws
           end
 
           def log_warning(context, actual_region)
-            msg = "S3 client configured for #{context.config.region.inspect} " +
-              "but the bucket #{context.params[:bucket].inspect} is in " +
-              "#{actual_region.inspect}; Please configure the proper region " +
-              "to avoid multiple unnecessary redirects and signing attempts\n"
-            if logger = context.config.logger
+            msg = "S3 client configured for #{context.config.region.inspect} " \
+                  "but the bucket #{context.params[:bucket].inspect} is in " \
+                  "#{actual_region.inspect}; Please configure the proper region " \
+                  "to avoid multiple unnecessary redirects and signing attempts\n"
+            if (logger = context.config.logger)
               logger.warn(msg)
             else
               warn(msg)
             end
           end
-
         end
 
         class << self
-
-          # @option options [required, String] :region
-          # @option options [required, #credentials] :credentials
-          # @api private
-          def build_v4_signer(options = {})
-            Aws::Sigv4::Signer.new({
-              service: 's3',
-              region: options[:region],
-              credentials_provider: options[:credentials],
-              uri_escape_path: false,
-              unsigned_headers: ['content-length', 'x-amzn-trace-id'],
-            })
-          end
-
           def new_hostname(context, region)
-            bucket = context.params[:bucket]
-            if region == 'us-east-1'
-              "#{bucket}.s3.amazonaws.com"
-            else
-              endpoint = Aws::Partitions::EndpointProvider.resolve(region, 's3')
-              bucket + '.' + URI.parse(endpoint).host
-            end
+            endpoint_params = context[:endpoint_params].dup
+            endpoint_params.region = region
+            endpoint_params.endpoint = nil
+            endpoint =
+              context.config.endpoint_provider.resolve_endpoint(endpoint_params)
+            URI(endpoint.url).host
           end
 
+          def custom_endpoint?(context)
+            region = context.config.region
+            partition = Aws::Endpoints::Matchers.aws_partition(region)
+            endpoint = context.http_request.endpoint
+
+            !endpoint.hostname.include?(partition['dnsSuffix']) &&
+              !endpoint.hostname.include?(partition['dualStackDnsSuffix'])
+          end
         end
       end
     end

data/lib/aws-sdk-s3/plugins/skip_whole_multipart_get_checksums.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+
+      # S3 GetObject results for whole Multipart Objects contain a checksum
+      # that cannot be validated.  These should be skipped by the
+      # ChecksumAlgorithm plugin.
+      class SkipWholeMultipartGetChecksums < Seahorse::Client::Plugin
+
+        class Handler < Seahorse::Client::Handler
+
+          def call(context)
+            context[:http_checksum] ||= {}
+            context[:http_checksum][:skip_on_suffix] = true
+
+            @handler.call(context)
+          end
+
+        end
+
+        handler(
+          Handler,
+          step: :initialize,
+          operations: [:get_object]
+        )
+      end
+    end
+  end
+end