aws-sdk-s3 1.79.1 → 1.212.0

data/lib/aws-sdk-s3/plugins/accelerate.rb

@@ -23,20 +23,16 @@ each bucket. [Go here for more information](http://docs.aws.amazon.com/AmazonS3/
           operations = config.api.operation_names - [
             :create_bucket, :list_buckets, :delete_bucket
           ]
-          # Need 2 handlers so that the context can be set for other plugins
-          # and to remove :use_accelerate_endpoint from the params.
           handlers.add(
             OptionHandler, step: :initialize, operations: operations
           )
-          handlers.add(
-            AccelerateHandler, step: :build, priority: 0, operations: operations
-          )
         end
 
         # @api private
         class OptionHandler < Seahorse::Client::Handler
           def call(context)
             # Support client configuration and per-operation configuration
+            # TODO: move this to an options hash and warn here.
             if context.params.is_a?(Hash)
               accelerate = context.params.delete(:use_accelerate_endpoint)
             end
@@ -47,40 +43,6 @@ each bucket. [Go here for more information](http://docs.aws.amazon.com/AmazonS3/
             @handler.call(context)
           end
         end
-
-        # @api private
-        class AccelerateHandler < Seahorse::Client::Handler
-          def call(context)
-            if context[:use_accelerate_endpoint]
-              dualstack = !!context[:use_dualstack_endpoint]
-              use_accelerate_endpoint(context, dualstack)
-            end
-            @handler.call(context)
-          end
-
-          private
-
-          def use_accelerate_endpoint(context, dualstack)
-            bucket_name = context.params[:bucket]
-            validate_bucket_name!(bucket_name)
-            endpoint = URI.parse(context.http_request.endpoint.to_s)
-            endpoint.scheme = 'https'
-            endpoint.port = 443
-            endpoint.host = "#{bucket_name}.s3-accelerate"\
-                            "#{'.dualstack' if dualstack}.amazonaws.com"
-            context.http_request.endpoint = endpoint.to_s
-            # s3 accelerate endpoint doesn't work with 'expect' header
-            context.http_request.headers.delete('expect')
-          end
-
-          def validate_bucket_name!(bucket_name)
-            unless BucketDns.dns_compatible?(bucket_name, _ssl = true)
-              raise ArgumentError,
-                    'Unable to use `use_accelerate_endpoint: true` on buckets '\
-                    'with non-DNS compatible names.'
-            end
-          end
-        end
       end
     end
   end

data/lib/aws-sdk-s3/plugins/access_grants.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+      # @api private
+      class AccessGrants < Seahorse::Client::Plugin
+        @s3control =
+          begin
+            require 'aws-sdk-s3control'
+            true
+          rescue LoadError
+            false
+          end
+
+        option(
+          :access_grants,
+          default: false,
+          doc_type: 'Boolean',
+          docstring: <<-DOCS)
+When `true`, the S3 client will use the S3 Access Grants feature to
+authenticate requests. Bucket credentials will be fetched from S3
+Control using the `get_data_access` API.
+          DOCS
+
+        option(:access_grants_credentials_provider,
+          doc_type: 'Aws::S3::AccessGrantsCredentialsProvider',
+          rbs_type: 'untyped',
+          docstring: <<-DOCS) do |_cfg|
+When `access_grants` is `true`, this option can be used to provide
+additional options to the credentials provider, including a privilege
+setting, caching, and fallback behavior.
+          DOCS
+          Aws::S3::AccessGrantsCredentialsProvider.new
+        end
+
+        # @api private
+        class Handler < Seahorse::Client::Handler
+          PERMISSION_MAP = {
+            head_object: 'READ',
+            get_object: 'READ',
+            get_object_acl: 'READ',
+            list_multipart_uploads: 'READ',
+            list_objects_v2: 'READ',
+            list_object_versions: 'READ',
+            list_parts: 'READ',
+            head_bucket: 'READ',
+            get_object_attributes: 'READ',
+            put_object: 'WRITE',
+            put_object_acl: 'WRITE',
+            delete_object: 'WRITE',
+            abort_multipart_upload: 'WRITE',
+            create_multipart_upload: 'WRITE',
+            upload_part: 'WRITE',
+            complete_multipart_upload: 'WRITE',
+            delete_objects: 'WRITE',
+            copy_object: 'READWRITE'
+          }.freeze
+
+          def call(context)
+            provider = context.config.access_grants_credentials_provider
+
+            if access_grants_operation?(context) &&
+               !s3_express_endpoint?(context) &&
+               !credentials_head_bucket_call?(provider)
+              params = context[:endpoint_params]
+              permission = PERMISSION_MAP[context.operation_name]
+
+              key =
+                case context.operation_name
+                when :delete_objects
+                  delete_params = context.params[:delete]
+                  common_prefixes(delete_params[:objects].map { |o| o[:key] })
+                when :copy_object
+                  source_bucket, source_key = params[:copy_source].split('/', 2)
+                  if params[:bucket] != source_bucket
+                    raise ArgumentError,
+                          'source and destination bucket must be the same'
+                  end
+                  common_prefixes([params[:key], source_key])
+                else
+                  params[:key]
+                end
+
+              credentials = provider.access_grants_credentials_for(
+                bucket: params[:bucket],
+                key: key,
+                prefix: params[:prefix],
+                permission: permission
+              )
+              context[:sigv4_credentials] = credentials # Sign will use this
+            end
+
+            with_metric(credentials) { @handler.call(context) }
+          end
+
+          private
+
+          def with_metric(credentials, &block)
+            return block.call unless credentials
+
+            Aws::Plugins::UserAgent.metric('S3_ACCESS_GRANTS', &block)
+          end
+
+          # HeadBucket is a supported call. When fetching credentials,
+          # this plugin is executed again, and becomes recursive.
+          def credentials_head_bucket_call?(provider)
+            provider.instance_variable_get(:@head_bucket_call)
+          end
+
+          def access_grants_operation?(context)
+            params = context[:endpoint_params]
+            params[:bucket] && PERMISSION_MAP[context.operation_name]
+          end
+
+          def s3_express_endpoint?(context)
+            context[:endpoint_properties]['backend'] == 'S3Express'
+          end
+
+          # Return the common prefix of the keys, regardless of the delimiter.
+          # For example, given keys ['foo/bar', 'foo/baz'], the common prefix
+          # is 'foo/ba'.
+          def common_prefixes(keys)
+            return '' if keys.empty?
+
+            first_key = keys[0]
+            common_ancestor = first_key
+            last_prefix = ''
+            keys.each do |k|
+              until common_ancestor.empty?
+                break if k.start_with?(common_ancestor)
+
+                last_index = common_ancestor.rindex('/')
+                return '' if last_index.nil?
+
+                last_prefix = common_ancestor[(last_index + 1)..-1]
+                common_ancestor = common_ancestor[0...last_index]
+              end
+            end
+            new_common_ancestor = "#{common_ancestor}/#{last_prefix}"
+            keys.each do |k|
+              until last_prefix.empty?
+                break if k.start_with?(new_common_ancestor)
+
+                last_prefix = last_prefix[0...-1]
+                new_common_ancestor = "#{common_ancestor}/#{last_prefix}"
+              end
+            end
+            if new_common_ancestor == "#{first_key}/"
+              first_key
+            else
+              new_common_ancestor
+            end
+          end
+        end
+
+        def add_handlers(handlers, config)
+          return unless AccessGrants.s3control? && config.access_grants
+
+          handlers.add(Handler)
+        end
+
+        def after_initialize(client)
+          return unless AccessGrants.s3control? && client.config.access_grants
+
+          provider = client.config.access_grants_credentials_provider
+          provider.s3_client = client unless provider.s3_client
+        end
+
+        class << self
+          def s3control?
+            @s3control
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/arn.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+      # When an accesspoint ARN is provided for :bucket in S3 operations, this
+      # plugin resolves the request endpoint from the ARN when possible.
+      # @api private
+      class ARN < Seahorse::Client::Plugin
+        option(
+          :s3_use_arn_region,
+          default: true,
+          doc_type: 'Boolean',
+          docstring: <<-DOCS) do |cfg|
+For S3 ARNs passed into the `:bucket` parameter, this option will
+use the region in the ARN, allowing for cross-region requests to
+be made. Set to `false` to use the client's region instead.
+          DOCS
+          resolve_s3_use_arn_region(cfg)
+        end
+
+        option(
+          :s3_disable_multiregion_access_points,
+          default: false,
+          doc_type: 'Boolean',
+          docstring: <<-DOCS) do |cfg|
+When set to `false` this will option will raise errors when multi-region
+access point ARNs are used.  Multi-region access points can potentially
+result in cross region requests.
+        DOCS
+          resolve_s3_disable_multiregion_access_points(cfg)
+        end
+
+        class << self
+          private
+
+          def resolve_s3_use_arn_region(cfg)
+            value = ENV['AWS_S3_USE_ARN_REGION'] ||
+                    Aws.shared_config.s3_use_arn_region(profile: cfg.profile) ||
+                    'true'
+            value = Aws::Util.str_2_bool(value)
+            # Raise if provided value is not true or false
+            if value.nil?
+              raise ArgumentError,
+                    'Must provide either `true` or `false` for the '\
+                    '`s3_use_arn_region` profile option or for '\
+                    "ENV['AWS_S3_USE_ARN_REGION']."
+            end
+            value
+          end
+
+          def resolve_s3_disable_multiregion_access_points(cfg)
+            value = ENV['AWS_S3_DISABLE_MULTIREGION_ACCESS_POINTS'] ||
+              Aws.shared_config.s3_disable_multiregion_access_points(profile: cfg.profile) ||
+              'false'
+            value = Aws::Util.str_2_bool(value)
+            # Raise if provided value is not true or false
+            if value.nil?
+              raise ArgumentError,
+                    'Must provide either `true` or `false` for '\
+                    's3_use_arn_region profile option or for '\
+                    "ENV['AWS_S3_USE_ARN_REGION']"
+            end
+            value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/bucket_dns.rb

@@ -23,45 +23,10 @@ When set to `true`, the bucket name is always left in the
 request URI and never moved to the host as a sub-domain.
           DOCS
 
-        def add_handlers(handlers, config)
-          handlers.add(Handler) unless config.force_path_style
-        end
-
-        # @api private
-        class Handler < Seahorse::Client::Handler
-
-          def call(context)
-            move_dns_compat_bucket_to_subdomain(context)
-            @handler.call(context)
-          end
-
-          private
-
-          def move_dns_compat_bucket_to_subdomain(context)
-            bucket_name = context.params[:bucket]
-            endpoint = context.http_request.endpoint
-            if bucket_name &&
-               BucketDns.dns_compatible?(bucket_name, https?(endpoint)) &&
-               context.operation_name.to_s != 'get_bucket_location'
-              move_bucket_to_subdomain(bucket_name, endpoint)
-            end
-          end
-
-          def move_bucket_to_subdomain(bucket_name, endpoint)
-            endpoint.host = "#{bucket_name}.#{endpoint.host}"
-            path = endpoint.path.sub("/#{bucket_name}", '')
-            path = "/#{path}" unless path.match(/^\//)
-            endpoint.path = path
-          end
-
-          def https?(uri)
-            uri.scheme == 'https'
-          end
-
-        end
-
+        # These class methods were originally used in a handler in this plugin.
+        # SigV2 legacy signer needs this logic so we keep it here as utility.
+        # New endpoint resolution will check this as a matcher.
         class << self
-
           # @param [String] bucket_name
           # @param [Boolean] ssl
           # @return [Boolean]
@@ -73,8 +38,6 @@ request URI and never moved to the host as a sub-domain.
             end
           end
 
-          # Checks for a valid RFC-3986 host name
-          # @see https://tools.ietf.org/html/rfc3986#section-3.2.2
           # @param [String] bucket_name
           # @return [Boolean]
           def valid_subdomain?(bucket_name)
@@ -83,7 +46,6 @@ request URI and never moved to the host as a sub-domain.
             bucket_name !~ /(\d+\.){3}\d+/ &&
             bucket_name !~ /[.-]{2}/
           end
-
         end
       end
     end

data/lib/aws-sdk-s3/plugins/bucket_name_restrictions.rb

@@ -13,12 +13,7 @@ module Aws
           def call(context)
             bucket_member = _bucket_member(context.operation.input.shape)
             if bucket_member && (bucket = context.params[bucket_member])
-              _resolved_bucket, _resolved_region, arn = BucketARN.resolve_arn!(
-                bucket,
-                context.config.region,
-                context.config.s3_use_arn_region
-              )
-              if !arn && bucket.include?('/')
+              if !Aws::ARNParser.arn?(bucket) && bucket.include?('/')
                 raise ArgumentError,
                       'bucket name must not contain a forward-slash (/)'
               end

data/lib/aws-sdk-s3/plugins/checksum_algorithm.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+      # @api private
+      class ChecksumAlgorithm < Seahorse::Client::Plugin
+
+        # S3 GetObject results for whole Multipart Objects contain a checksum
+        # that cannot be validated. These should be skipped by the
+        # ChecksumAlgorithm plugin.
+        class SkipWholeMultipartGetChecksumsHandler < Seahorse::Client::Handler
+          def call(context)
+            context[:http_checksum] ||= {}
+            context[:http_checksum][:skip_on_suffix] = true
+
+            @handler.call(context)
+          end
+        end
+
+        # Handler to disable trailer checksums for S3-compatible services
+        # that don't support STREAMING-UNSIGNED-PAYLOAD-TRAILER
+        # See: https://github.com/aws/aws-sdk-ruby/issues/3338
+        class SkipTrailerChecksumsHandler < Seahorse::Client::Handler
+          def call(context)
+            context[:skip_trailer_checksums] = true if custom_endpoint?(context.config)
+            @handler.call(context)
+          end
+
+          private
+
+          def custom_endpoint?(config)
+            !config.regional_endpoint || !config.endpoint_provider.instance_of?(Aws::S3::EndpointProvider)
+          end
+        end
+
+        def add_handlers(handlers, _config)
+          handlers.add(SkipWholeMultipartGetChecksumsHandler, step: :initialize, operations: [:get_object])
+          handlers.add(SkipTrailerChecksumsHandler, step: :build, priority: 16, operations: %i[put_object upload_part])
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/dualstack.rb

@@ -5,23 +5,14 @@ module Aws
     module Plugins
       # @api private
       class Dualstack < Seahorse::Client::Plugin
-
-        option(:use_dualstack_endpoint,
-          default: false,
-          doc_type: 'Boolean',
-          docstring: <<-DOCS)
-When set to `true`, IPv6-compatible bucket endpoints will be used
-for all operations.
-          DOCS
-
-        def add_handlers(handlers, config)
+        def add_handlers(handlers, _config)
           handlers.add(OptionHandler, step: :initialize)
-          handlers.add(DualstackHandler, step: :build, priority: 0)
         end
 
         # @api private
         class OptionHandler < Seahorse::Client::Handler
           def call(context)
+            # Support client configuration and per-operation configuration
             if context.params.is_a?(Hash)
               dualstack = context.params.delete(:use_dualstack_endpoint)
             end
@@ -30,44 +21,6 @@ for all operations.
             @handler.call(context)
           end
         end
-
-        # @api private
-        class DualstackHandler < Seahorse::Client::Handler
-          def call(context)
-            apply_dualstack_endpoint(context) if use_dualstack_endpoint?(context)
-            @handler.call(context)
-          end
-
-          private
-          def apply_dualstack_endpoint(context)
-            bucket_name = context.params[:bucket]
-            region = context.config.region
-            context.config.force_path_style
-            dns_suffix = Aws::Partitions::EndpointProvider.dns_suffix_for(region)
-
-            if use_bucket_dns?(bucket_name, context)
-              host = "#{bucket_name}.s3.dualstack.#{region}.#{dns_suffix}"
-            else
-              host = "s3.dualstack.#{region}.#{dns_suffix}"
-            end
-            endpoint = URI.parse(context.http_request.endpoint.to_s)
-            endpoint.scheme = context.http_request.endpoint.scheme
-            endpoint.port = context.http_request.endpoint.port
-            endpoint.host = host
-            context.http_request.endpoint = endpoint.to_s
-          end
-
-          def use_bucket_dns?(bucket_name, context)
-            ssl = context.http_request.endpoint.scheme == "https"
-            bucket_name && BucketDns.dns_compatible?(bucket_name, ssl) &&
-              !context.config.force_path_style
-          end
-
-          def use_dualstack_endpoint?(context)
-            context[:use_dualstack_endpoint] && !context[:use_accelerate_endpoint]
-          end
-        end
-
       end
     end
   end

data/lib/aws-sdk-s3/plugins/endpoints.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+# WARNING ABOUT GENERATED CODE
+#
+# This file is generated. See the contributing guide for more information:
+# https://github.com/aws/aws-sdk-ruby/blob/version-3/CONTRIBUTING.md
+#
+# WARNING ABOUT GENERATED CODE
+
+
+module Aws::S3
+  module Plugins
+    class Endpoints < Seahorse::Client::Plugin
+      option(
+        :endpoint_provider,
+        doc_type: 'Aws::S3::EndpointProvider',
+        rbs_type: 'untyped',
+        docstring: <<~DOCS) do |_cfg|
+The endpoint provider used to resolve endpoints. Any object that responds to
+`#resolve_endpoint(parameters)` where `parameters` is a Struct similar to
+`Aws::S3::EndpointParameters`.
+        DOCS
+        Aws::S3::EndpointProvider.new
+      end
+
+      option(
+        :disable_s3_express_session_auth,
+        doc_type: 'boolean',
+        docstring: <<~DOCS) do |cfg|
+Parameter to indicate whether S3Express session auth should be disabled
+        DOCS
+        nil
+      end
+
+      # @api private
+      class Handler < Seahorse::Client::Handler
+        def call(context)
+          unless context[:discovered_endpoint]
+            params = Aws::S3::Endpoints.parameters_for_operation(context)
+            endpoint = context.config.endpoint_provider.resolve_endpoint(params)
+
+            context.http_request.endpoint = endpoint.url
+            apply_endpoint_headers(context, endpoint.headers)
+
+            context[:endpoint_params] = params
+            context[:endpoint_properties] = endpoint.properties
+          end
+
+          context[:auth_scheme] =
+            Aws::Endpoints.resolve_auth_scheme(context, endpoint)
+
+          with_metrics(context) { @handler.call(context) }
+        end
+
+        private
+
+        def with_metrics(context, &block)
+          metrics = []
+          metrics << 'ENDPOINT_OVERRIDE' unless context.config.regional_endpoint
+          if context[:auth_scheme] && context[:auth_scheme]['name'] == 'sigv4a'
+            metrics << 'SIGV4A_SIGNING'
+          end
+          if context.config.credentials&.credentials&.account_id
+            metrics << 'RESOLVED_ACCOUNT_ID'
+          end
+          Aws::Plugins::UserAgent.metric(*metrics, &block)
+        end
+
+        def apply_endpoint_headers(context, headers)
+          headers.each do |key, values|
+            value = values
+              .compact
+              .map { |s| Seahorse::Util.escape_header_list_string(s.to_s) }
+              .join(',')
+
+            context.http_request.headers[key] = value
+          end
+        end
+      end
+
+      def add_handlers(handlers, _config)
+        handlers.add(Handler, step: :build, priority: 75)
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/expect_100_continue.rb

@@ -15,7 +15,9 @@ module Aws
         class Handler < Seahorse::Client::Handler
 
           def call(context)
-            if context.http_request.body && context.http_request.body.size > 0
+            body = context.http_request.body
+            if body.respond_to?(:size) && body.size > 0 &&
+               !context[:use_accelerate_endpoint]
               context.http_request.headers['expect'] = '100-continue'
             end
             @handler.call(context)

data/lib/aws-sdk-s3/plugins/express_session_auth.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+      # @api private
+      class ExpressSessionAuth < Seahorse::Client::Plugin
+        # This should be s3_disable_express_auth instead
+        # But this is not a built in. We're overwriting the generated value
+        option(:disable_s3_express_session_auth,
+          default: false,
+          doc_type: 'Boolean',
+          docstring: <<-DOCS) do |cfg|
+When `true`, S3 Express session authentication is disabled.
+          DOCS
+          resolve_disable_s3_express_session_auth(cfg)
+        end
+
+        option(:express_credentials_provider,
+          doc_type: 'Aws::S3::ExpressCredentialsProvider',
+          rbs_type: 'untyped',
+          docstring: <<-DOCS) do |_cfg|
+Credential Provider for S3 Express endpoints. Manages credentials
+for different buckets.
+          DOCS
+          Aws::S3::ExpressCredentialsProvider.new
+        end
+
+        # @api private
+        class Handler < Seahorse::Client::Handler
+          def call(context)
+            context[:s3_express_endpoint] = true if s3_express_endpoint?(context)
+
+            # if s3 express auth, use new credentials and sign additional header
+            if context[:auth_scheme]['name'] == 'sigv4-s3express' &&
+               !context.config.disable_s3_express_session_auth
+              bucket = context.params[:bucket]
+              credentials_provider = context.config.express_credentials_provider
+              credentials = credentials_provider.express_credentials_for(bucket)
+              context[:sigv4_credentials] = credentials # Sign will use this
+            end
+
+            with_metric(credentials) { @handler.call(context) }
+          end
+
+          private
+
+          def with_metric(credentials, &block)
+            return block.call unless credentials
+
+            Aws::Plugins::UserAgent.metric('S3_EXPRESS_BUCKET', &block)
+          end
+
+          def s3_express_endpoint?(context)
+            context[:endpoint_properties]['backend'] == 'S3Express'
+          end
+        end
+
+        handler(Handler)
+
+        # Optimization - sets this client as the client to create sessions.
+        def after_initialize(client)
+          provider = client.config.express_credentials_provider
+          provider.client = client unless provider.client
+        end
+
+        class << self
+          private
+
+          def resolve_disable_s3_express_session_auth(cfg)
+            value = ENV['AWS_S3_DISABLE_EXPRESS_SESSION_AUTH'] ||
+                    Aws.shared_config.s3_disable_express_session_auth(profile: cfg.profile) ||
+                    'false'
+            value = Aws::Util.str_2_bool(value)
+            # Raise if provided value is not true or false
+            if value.nil?
+              raise ArgumentError,
+                    'Must provide either `true` or `false` for the '\
+                    '`s3_disable_express_session_auth` profile option or for '\
+                    "ENV['AWS_S3_DISABLE_EXPRESS_SESSION_AUTH']."
+            end
+            value
+          end
+        end
+      end
+    end
+  end
+end