aws-sdk-s3 1.119.2 → 1.158.0

data/lib/aws-sdk-s3/plugins/access_grants.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+      # @api private
+      class AccessGrants < Seahorse::Client::Plugin
+        @s3control =
+          begin
+            require 'aws-sdk-s3control'
+            true
+          rescue LoadError
+            false
+          end
+
+        option(
+          :access_grants,
+          default: false,
+          doc_type: 'Boolean',
+          docstring: <<-DOCS)
+When `true`, the S3 client will use the S3 Access Grants feature to
+authenticate requests. Bucket credentials will be fetched from S3
+Control using the `get_data_access` API.
+          DOCS
+
+        option(:access_grants_credentials_provider,
+          doc_type: 'Aws::S3::AccessGrantsCredentialsProvider',
+          rbs_type: 'untyped',
+          docstring: <<-DOCS) do |_cfg|
+When `access_grants` is `true`, this option can be used to provide
+additional options to the credentials provider, including a privilege
+setting, caching, and fallback behavior.
+          DOCS
+          Aws::S3::AccessGrantsCredentialsProvider.new
+        end
+
+        # @api private
+        class Handler < Seahorse::Client::Handler
+          PERMISSION_MAP = {
+            head_object: 'READ',
+            get_object: 'READ',
+            get_object_acl: 'READ',
+            list_multipart_uploads: 'READ',
+            list_objects_v2: 'READ',
+            list_object_versions: 'READ',
+            list_parts: 'READ',
+            head_bucket: 'READ',
+            get_object_attributes: 'READ',
+            put_object: 'WRITE',
+            put_object_acl: 'WRITE',
+            delete_object: 'WRITE',
+            abort_multipart_upload: 'WRITE',
+            create_multipart_upload: 'WRITE',
+            upload_part: 'WRITE',
+            complete_multipart_upload: 'WRITE',
+            delete_objects: 'WRITE',
+            copy_object: 'READWRITE'
+          }.freeze
+
+          def call(context)
+            provider = context.config.access_grants_credentials_provider
+
+            if access_grants_operation?(context) &&
+               !s3_express_endpoint?(context) &&
+               !credentials_head_bucket_call?(provider)
+              params = context[:endpoint_params]
+              permission = PERMISSION_MAP[context.operation_name]
+
+              key =
+                case context.operation_name
+                when :delete_objects
+                  delete_params = context.params[:delete]
+                  common_prefixes(delete_params[:objects].map { |o| o[:key] })
+                when :copy_object
+                  source_bucket, source_key = params[:copy_source].split('/', 2)
+                  if params[:bucket] != source_bucket
+                    raise ArgumentError,
+                          'source and destination bucket must be the same'
+                  end
+                  common_prefixes([params[:key], source_key])
+                else
+                  params[:key]
+                end
+
+              credentials = provider.access_grants_credentials_for(
+                bucket: params[:bucket],
+                key: key,
+                prefix: params[:prefix],
+                permission: permission
+              )
+              context[:sigv4_credentials] = credentials # Sign will use this
+            end
+
+            with_metric(credentials) { @handler.call(context) }
+          end
+
+          private
+
+          def with_metric(credentials, &block)
+            return block.call unless credentials
+
+            Aws::Plugins::UserAgent.metric('S3_ACCESS_GRANTS', &block)
+          end
+
+          # HeadBucket is a supported call. When fetching credentials,
+          # this plugin is executed again, and becomes recursive.
+          def credentials_head_bucket_call?(provider)
+            provider.instance_variable_get(:@head_bucket_call)
+          end
+
+          def access_grants_operation?(context)
+            params = context[:endpoint_params]
+            params[:bucket] && PERMISSION_MAP[context.operation_name]
+          end
+
+          def s3_express_endpoint?(context)
+            context[:endpoint_properties]['backend'] == 'S3Express'
+          end
+
+          # Return the common prefix of the keys, regardless of the delimiter.
+          # For example, given keys ['foo/bar', 'foo/baz'], the common prefix
+          # is 'foo/ba'.
+          def common_prefixes(keys)
+            return '' if keys.empty?
+
+            first_key = keys[0]
+            common_ancestor = first_key
+            last_prefix = ''
+            keys.each do |k|
+              until common_ancestor.empty?
+                break if k.start_with?(common_ancestor)
+
+                last_index = common_ancestor.rindex('/')
+                return '' if last_index.nil?
+
+                last_prefix = common_ancestor[(last_index + 1)..-1]
+                common_ancestor = common_ancestor[0...last_index]
+              end
+            end
+            new_common_ancestor = "#{common_ancestor}/#{last_prefix}"
+            keys.each do |k|
+              until last_prefix.empty?
+                break if k.start_with?(new_common_ancestor)
+
+                last_prefix = last_prefix[0...-1]
+                new_common_ancestor = "#{common_ancestor}/#{last_prefix}"
+              end
+            end
+            if new_common_ancestor == "#{first_key}/"
+              first_key
+            else
+              new_common_ancestor
+            end
+          end
+        end
+
+        def add_handlers(handlers, config)
+          return unless AccessGrants.s3control? && config.access_grants
+
+          handlers.add(Handler)
+        end
+
+        def after_initialize(client)
+          return unless AccessGrants.s3control? && client.config.access_grants
+
+          provider = client.config.access_grants_credentials_provider
+          provider.s3_client = client unless provider.s3_client
+        end
+
+        class << self
+          def s3control?
+            @s3control
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/endpoints.rb

@@ -14,6 +14,7 @@ module Aws::S3
       option(
         :endpoint_provider,
         doc_type: 'Aws::S3::EndpointProvider',
+        rbs_type: 'untyped',
         docstring: 'The endpoint provider used to resolve endpoints. Any '\
                    'object that responds to `#resolve_endpoint(parameters)` '\
                    'where `parameters` is a Struct similar to '\
@@ -22,19 +23,26 @@ module Aws::S3
         Aws::S3::EndpointProvider.new
       end
 
+      option(
+        :disable_s3_express_session_auth,
+        doc_type: 'Boolean',
+        default: nil,
+        docstring: "Parameter to indicate whether S3Express session auth should be disabled")
+
       # @api private
       class Handler < Seahorse::Client::Handler
         def call(context)
-          # If endpoint was discovered, do not resolve or apply the endpoint.
           unless context[:discovered_endpoint]
             params = parameters_for_operation(context)
             endpoint = context.config.endpoint_provider.resolve_endpoint(params)
 
             context.http_request.endpoint = endpoint.url
             apply_endpoint_headers(context, endpoint.headers)
+
+            context[:endpoint_params] = params
+            context[:endpoint_properties] = endpoint.properties
           end
 
-          context[:endpoint_params] = params
           context[:auth_scheme] =
             Aws::Endpoints.resolve_auth_scheme(context, endpoint)
 
@@ -66,6 +74,8 @@ module Aws::S3
             Aws::S3::Endpoints::CreateBucket.build(context)
           when :create_multipart_upload
             Aws::S3::Endpoints::CreateMultipartUpload.build(context)
+          when :create_session
+            Aws::S3::Endpoints::CreateSession.build(context)
           when :delete_bucket
             Aws::S3::Endpoints::DeleteBucket.build(context)
           when :delete_bucket_analytics_configuration
@@ -176,6 +186,8 @@ module Aws::S3
             Aws::S3::Endpoints::ListBucketMetricsConfigurations.build(context)
           when :list_buckets
             Aws::S3::Endpoints::ListBuckets.build(context)
+          when :list_directory_buckets
+            Aws::S3::Endpoints::ListDirectoryBuckets.build(context)
           when :list_multipart_uploads
             Aws::S3::Endpoints::ListMultipartUploads.build(context)
           when :list_object_versions

data/lib/aws-sdk-s3/plugins/express_session_auth.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+      # @api private
+      class ExpressSessionAuth < Seahorse::Client::Plugin
+        # This should be s3_disable_express_auth instead
+        # But this is not a built in. We're overwriting the generated value
+        option(:disable_s3_express_session_auth,
+          default: false,
+          doc_type: 'Boolean',
+          docstring: <<-DOCS) do |cfg|
+When `true`, S3 Express session authentication is disabled.
+          DOCS
+          resolve_disable_s3_express_session_auth(cfg)
+        end
+
+        option(:express_credentials_provider,
+          doc_type: 'Aws::S3::ExpressCredentialsProvider',
+          rbs_type: 'untyped',
+          docstring: <<-DOCS) do |_cfg|
+Credential Provider for S3 Express endpoints. Manages credentials
+for different buckets.
+          DOCS
+          Aws::S3::ExpressCredentialsProvider.new
+        end
+
+        # @api private
+        class Handler < Seahorse::Client::Handler
+          def call(context)
+            if (props = context[:endpoint_properties])
+              # S3 Express endpoint - turn off md5 and enable crc32 default
+              if props['backend'] == 'S3Express'
+                if context.operation_name == :put_object || checksum_required?(context)
+                  context[:default_request_checksum_algorithm] = 'CRC32'
+                end
+                context[:s3_express_endpoint] = true
+              end
+
+              # if s3 express auth, use new credentials and sign additional header
+              if context[:auth_scheme]['name'] == 'sigv4-s3express' &&
+                 !context.config.disable_s3_express_session_auth
+                bucket = context.params[:bucket]
+                credentials_provider = context.config.express_credentials_provider
+                credentials = credentials_provider.express_credentials_for(bucket)
+                context[:sigv4_credentials] = credentials # Sign will use this
+              end
+            end
+            with_metric(credentials) { @handler.call(context) }
+          end
+
+          private
+
+          def with_metric(credentials, &block)
+            return block.call unless credentials
+
+            Aws::Plugins::UserAgent.metric('S3_EXPRESS_BUCKET', &block)
+          end
+
+          def checksum_required?(context)
+            context.operation.http_checksum_required ||
+              (context.operation.http_checksum &&
+                context.operation.http_checksum['requestChecksumRequired'])
+          end
+        end
+
+        handler(Handler)
+
+        # Optimization - sets this client as the client to create sessions.
+        def after_initialize(client)
+          provider = client.config.express_credentials_provider
+          provider.client = client unless provider.client
+        end
+
+        class << self
+          private
+
+          def resolve_disable_s3_express_session_auth(cfg)
+            value = ENV['AWS_S3_DISABLE_EXPRESS_SESSION_AUTH'] ||
+                    Aws.shared_config.s3_disable_express_session_auth(profile: cfg.profile) ||
+                    'false'
+            value = Aws::Util.str_2_bool(value)
+            # Raise if provided value is not true or false
+            if value.nil?
+              raise ArgumentError,
+                    'Must provide either `true` or `false` for the '\
+                    '`s3_disable_express_session_auth` profile option or for '\
+                    "ENV['AWS_S3_DISABLE_EXPRESS_SESSION_AUTH']."
+            end
+            value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/http_200_errors.rb

@@ -15,22 +15,67 @@ module Aws
 
           def call(context)
             @handler.call(context).on(200) do |response|
-              if error = check_for_error(context)
-                context.http_response.status_code = 500
-                response.data = nil
-                response.error = error
+              return response if streaming_output?(context.operation.output)
+
+              error = check_for_error(context)
+              return response unless error
+
+              context.http_response.status_code = 500
+              response.data = nil
+              response.error = error
+            end
+          end
+
+          private
+
+          # Streaming outputs are not subject to 200 errors.
+          def streaming_output?(output)
+            if (payload = output[:payload_member])
+              # checking ref and shape
+              payload['streaming'] || payload.shape['streaming'] ||
+                payload.eventstream
+            else
+              false
+            end
+          end
+
+          # Checks if the output shape is a structure shape and has members that
+          # are in the body for the case of a payload and a normal structure. A
+          # non-structure shape will not have members in the body. In the case
+          # of a string or blob, the body contents would have been checked first
+          # before this method is called in incomplete_xml_body?.
+          def members_in_body?(output)
+            shape =
+              if output[:payload_member]
+                output[:payload_member].shape
+              else
+                output.shape
               end
+
+            if structure_shape?(shape)
+              shape.members.any? { |_, k| k.location.nil? }
+            else
+              false
             end
           end
 
+          def structure_shape?(shape)
+            shape.is_a?(Seahorse::Model::Shapes::StructureShape)
+          end
+
+          # Must have a member in the body and have the start of an XML Tag.
+          # Other incomplete xml bodies will result in an XML ParsingError.
+          def incomplete_xml_body?(xml, output)
+            members_in_body?(output) && !xml.match(/<\w/)
+          end
+
           def check_for_error(context)
             xml = context.http_response.body_contents
-            if xml.match(/<Error>/)
+            if xml.match(/\?>\s*<Error>/)
               error_code = xml.match(/<Code>(.+?)<\/Code>/)[1]
               error_message = xml.match(/<Message>(.+?)<\/Message>/)[1]
               S3::Errors.error_class(error_code).new(context, error_message)
-            elsif !xml.match(/<\w/) # Must have the start of an XML Tag
-              # Other incomplete xml bodies will result in XML ParsingError
+            elsif incomplete_xml_body?(xml, context.operation.output)
               Seahorse::Client::NetworkingError.new(
                 S3::Errors
                   .error_class('InternalError')
@@ -40,15 +85,7 @@ module Aws
           end
         end
 
-        handler(
-          Handler,
-          step: :sign,
-          operations: [
-            :complete_multipart_upload,
-            :copy_object,
-            :upload_part_copy,
-          ]
-        )
+        handler(Handler, step: :sign)
       end
     end
   end

data/lib/aws-sdk-s3/plugins/location_constraint.rb

@@ -22,7 +22,9 @@ module Aws
 
           def populate_location_constraint(params, region)
             params[:create_bucket_configuration] ||= {}
-            params[:create_bucket_configuration][:location_constraint] ||= region
+            unless params[:create_bucket_configuration][:location]
+              params[:create_bucket_configuration][:location_constraint] ||= region
+            end
           end
 
         end

data/lib/aws-sdk-s3/plugins/md5s.rb

@@ -22,7 +22,8 @@ module Aws
           CHUNK_SIZE = 1 * 1024 * 1024 # one MB
 
           def call(context)
-            if !context[:checksum_algorithms] # skip in favor of flexible checksum
+            if !context[:checksum_algorithms] && # skip in favor of flexible checksum
+               !context[:s3_express_endpoint] # s3 express endpoints do not support md5
               body = context.http_request.body
               if body.respond_to?(:size) && body.size > 0
                 context.http_request.headers['Content-Md5'] ||= md5(body)

data/lib/aws-sdk-s3/plugins/s3_signer.rb

@@ -4,6 +4,11 @@ require 'aws-sigv4'
 
 module Aws
   module S3
+    # @api private
+    def self.bucket_region_cache
+      @bucket_region_cache ||= BucketRegionCache.new
+    end
+
     module Plugins
       # This plugin used to have a V4 signer but it was removed in favor of
       # generic Sign plugin that uses endpoint auth scheme.
@@ -51,8 +56,10 @@ module Aws
           private
 
           def check_for_cached_region(context, bucket)
-            cached_region = S3::BUCKET_REGIONS[bucket]
-            if cached_region && cached_region != context.config.region
+            cached_region = Aws::S3.bucket_region_cache[bucket]
+            if cached_region &&
+               cached_region != context.config.region &&
+               !S3Signer.custom_endpoint?(context)
               context.http_request.endpoint.host = S3Signer.new_hostname(
                 context, cached_region
               )
@@ -77,7 +84,7 @@ module Aws
           def handle_region_errors(response)
             if wrong_sigv4_region?(response) &&
                !fips_region?(response) &&
-               !custom_endpoint?(response) &&
+               !S3Signer.custom_endpoint?(response.context) &&
                !expired_credentials?(response)
               get_region_and_retry(response.context)
             else
@@ -95,7 +102,7 @@ module Aws
           end
 
           def update_bucket_cache(context, actual_region)
-            S3::BUCKET_REGIONS[context.params[:bucket]] = actual_region
+            Aws::S3.bucket_region_cache[context.params[:bucket]] = actual_region
           end
 
           def fips_region?(resp)
@@ -106,15 +113,6 @@ module Aws
             resp.context.http_response.body_contents.match(/<Code>ExpiredToken<\/Code>/)
           end
 
-          def custom_endpoint?(resp)
-            region = resp.context.config.region
-            partition = Aws::Endpoints::Matchers.aws_partition(region)
-            endpoint = resp.context.http_request.endpoint
-
-            !endpoint.hostname.include?(partition['dnsSuffix']) &&
-              !endpoint.hostname.include?(partition['dualStackDnsSuffix'])
-          end
-
           def wrong_sigv4_region?(resp)
             resp.context.http_response.status_code == 400 &&
               (resp.context.http_response.headers['x-amz-bucket-region'] ||
@@ -168,6 +166,15 @@ module Aws
               context.config.endpoint_provider.resolve_endpoint(endpoint_params)
             URI(endpoint.url).host
           end
+
+          def custom_endpoint?(context)
+            region = context.config.region
+            partition = Aws::Endpoints::Matchers.aws_partition(region)
+            endpoint = context.http_request.endpoint
+
+            !endpoint.hostname.include?(partition['dnsSuffix']) &&
+              !endpoint.hostname.include?(partition['dualStackDnsSuffix'])
+          end
         end
       end
     end

data/lib/aws-sdk-s3/presigned_post.rb

@@ -315,26 +315,28 @@ module Aws
 
       # @!group Fields
 
-      # The key to use for the uploaded object. You can use `${filename}`
-      # as a variable in the key. This will be replaced with the name
-      # of the file as provided by the user.
+      # @!method key(key)
+      #   The key to use for the uploaded object. You can use `${filename}`
+      #   as a variable in the key. This will be replaced with the name
+      #   of the file as provided by the user.
       #
-      # For example, if the key is given as `/user/betty/${filename}` and
-      # the file uploaded is named `lolcatz.jpg`, the resultant key will
-      # be `/user/betty/lolcatz.jpg`.
+      #   For example, if the key is given as `/user/betty/${filename}` and
+      #   the file uploaded is named `lolcatz.jpg`, the resultant key will
+      #   be `/user/betty/lolcatz.jpg`.
       #
-      # @param [String] key
-      # @see http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html)
-      # @return [self]
+      #   @param [String] key
+      #   @see http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html)
+      #   @return [self]
       define_field(:key) do |key|
         @key_set = true
         with('key', key)
       end
 
-      # Specify a prefix the uploaded
-      # @param [String] prefix
-      # @see #key
-      # @return [self]
+      # @!method key_starts_with(prefix)
+      #   Specify a prefix the uploaded
+      #   @param [String] prefix
+      #   @see #key
+      #   @return [self]
       define_field(:key_starts_with) do |prefix|
         @key_set = true
         starts_with('key', prefix)
@@ -412,26 +414,29 @@ module Aws
       #   @return [self]
       define_field(:content_encoding, 'Content-Encoding', starts_with: true)
 
-      # The date and time at which the object is no longer cacheable.
-      # @note This does not affect the expiration of the presigned post
-      #   signature.
-      # @param [Time] time
-      # @see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21
-      # @return [self]
+      # @!method expires(time)
+      #   The date and time at which the object is no longer cacheable.
+      #   @note This does not affect the expiration of the presigned post
+      #     signature.
+      #   @param [Time] time
+      #   @see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21
+      #   @return [self]
       define_field(:expires) do |time|
         with('Expires', time.httpdate)
       end
 
-      # @param [String] prefix
-      # @see #expires
-      # @return [self]
+      # @!method expires_starts_with(prefix)
+      #   @param [String] prefix
+      #   @see #expires
+      #   @return [self]
       define_field(:expires_starts_with) do |prefix|
         starts_with('Expires', prefix)
       end
 
-      # The minimum and maximum allowable size for the uploaded content.
-      # @param [Range<Integer>] byte_range
-      # @return [self]
+      # @!method content_length_range(byte_range)
+      #   The minimum and maximum allowable size for the uploaded content.
+      #   @param [Range<Integer>] byte_range
+      #   @return [self]
       define_field(:content_length_range) do |byte_range|
         min = byte_range.begin
         max = byte_range.end
@@ -507,10 +512,11 @@ module Aws
       #   @return [self]
       define_field(:website_redirect_location, 'x-amz-website-redirect-location')
 
-      # Metadata hash to store with the uploaded object. Hash keys will be
-      # prefixed with "x-amz-meta-".
-      # @param [Hash<String,String>] hash
-      # @return [self]
+      # @!method metadata(hash)
+      #   Metadata hash to store with the uploaded object. Hash keys will be
+      #   prefixed with "x-amz-meta-".
+      #   @param [Hash<String,String>] hash
+      #   @return [self]
       define_field(:metadata) do |hash|
         hash.each do |key, value|
           with("x-amz-meta-#{key}", value)
@@ -518,10 +524,11 @@ module Aws
         self
       end
 
-      # Specify allowable prefix for each key in the metadata hash.
-      # @param [Hash<String,String>] hash
-      # @see #metadata
-      # @return [self]
+      # @!method metadata_starts_with(hash)
+      #   Specify allowable prefix for each key in the metadata hash.
+      #   @param [Hash<String,String>] hash
+      #   @see #metadata
+      #   @return [self]
       define_field(:metadata_starts_with) do |hash|
         hash.each do |key, value|
           starts_with("x-amz-meta-#{key}", value)
@@ -571,24 +578,26 @@ module Aws
         'x-amz-server-side-encryption-customer-algorithm'
       )
 
-      # Specifies the customer-provided encryption key for Amazon S3 to use
-      # in encrypting data. This value is used to store the object and then
-      # it is discarded; Amazon does not store the encryption key.
+      # @!method server_side_encryption_customer_key(value)
+      #   Specifies the customer-provided encryption key for Amazon S3 to use
+      #   in encrypting data. This value is used to store the object and then
+      #   it is discarded; Amazon does not store the encryption key.
       #
-      # You must also call {#server_side_encryption_customer_algorithm}.
+      #   You must also call {#server_side_encryption_customer_algorithm}.
       #
-      # @param [String] value
-      # @see #server_side_encryption_customer_algorithm
-      # @return [self]
+      #   @param [String] value
+      #   @see #server_side_encryption_customer_algorithm
+      #   @return [self]
       define_field(:server_side_encryption_customer_key) do |value|
         field_name = 'x-amz-server-side-encryption-customer-key'
         with(field_name, base64(value))
         with(field_name + '-MD5', base64(OpenSSL::Digest::MD5.digest(value)))
       end
 
-      # @param [String] prefix
-      # @see #server_side_encryption_customer_key
-      # @return [self]
+      # @!method server_side_encryption_customer_key_starts_with(prefix)
+      #   @param [String] prefix
+      #   @see #server_side_encryption_customer_key
+      #   @return [self]
       define_field(:server_side_encryption_customer_key_starts_with) do |prefix|
         field_name = 'x-amz-server-side-encryption-customer-key'
         starts_with(field_name, prefix)