aws-sdk-s3 1.48.0 → 1.60.0

data/lib/aws-sdk-s3/object_version.rb

@@ -27,6 +27,7 @@ module Aws::S3
       @id = extract_id(args, options)
       @data = options.delete(:data)
       @client = options.delete(:client) || Client.new(options)
+      @waiter_block_warned = false
     end
 
     # @!group Read-Only Attributes
@@ -46,7 +47,7 @@ module Aws::S3
       @id
     end
 
-    
+    # The entity tag is an MD5 hash of that version of the object.
     # @return [String]
     def etag
       data[:etag]
@@ -89,7 +90,7 @@ module Aws::S3
       data[:last_modified]
     end
 
-    
+    # Specifies the owner of the object.
     # @return [Types::Owner]
     def owner
       data[:owner]
@@ -233,14 +234,20 @@ module Aws::S3
     # @option options [String] :mfa
     #   The concatenation of the authentication device's serial number, a
     #   space, and the value that is displayed on your authentication device.
+    #   Required to permanently delete a versioned object if versioning is
+    #   configured with MFA delete enabled.
     # @option options [String] :request_payer
-    #   Confirms that the requester knows that she or he will be charged for
-    #   the request. Bucket owners need not specify this parameter in their
-    #   requests. Documentation on downloading objects from requester pays
-    #   buckets can be found at
-    #   http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+    #   Confirms that the requester knows that they will be charged for the
+    #   request. Bucket owners need not specify this parameter in their
+    #   requests. For information about downloading objects from requester
+    #   pays buckets, see [Downloading Objects in Requestor Pays Buckets][1]
+    #   in the *Amazon S3 Developer Guide*.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
     # @option options [Boolean] :bypass_governance_retention
-    #   Indicates whether Amazon S3 object lock should bypass governance-mode
+    #   Indicates whether S3 Object Lock should bypass Governance-mode
     #   restrictions to process this operation.
     # @return [Types::DeleteObjectOutput]
     def delete(options = {})
@@ -288,39 +295,43 @@ module Aws::S3
     #   time, otherwise return a 412 (precondition failed).
     # @option options [String] :range
     #   Downloads the specified range bytes of an object. For more information
-    #   about the HTTP Range header, go to
-    #   http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.
+    #   about the HTTP Range header, see
+    #   [http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35]().
     # @option options [String] :response_cache_control
-    #   Sets the Cache-Control header of the response.
+    #   Sets the `Cache-Control` header of the response.
     # @option options [String] :response_content_disposition
-    #   Sets the Content-Disposition header of the response
+    #   Sets the `Content-Disposition` header of the response
     # @option options [String] :response_content_encoding
-    #   Sets the Content-Encoding header of the response.
+    #   Sets the `Content-Encoding` header of the response.
     # @option options [String] :response_content_language
-    #   Sets the Content-Language header of the response.
+    #   Sets the `Content-Language` header of the response.
     # @option options [String] :response_content_type
-    #   Sets the Content-Type header of the response.
+    #   Sets the `Content-Type` header of the response.
     # @option options [Time,DateTime,Date,Integer,String] :response_expires
-    #   Sets the Expires header of the response.
+    #   Sets the `Expires` header of the response.
     # @option options [String] :sse_customer_algorithm
-    #   Specifies the algorithm to use to when encrypting the object (e.g.,
-    #   AES256).
+    #   Specifies the algorithm to use to when encrypting the object (for
+    #   example, AES256).
     # @option options [String] :sse_customer_key
     #   Specifies the customer-provided encryption key for Amazon S3 to use in
     #   encrypting data. This value is used to store the object and then it is
-    #   discarded; Amazon does not store the encryption key. The key must be
-    #   appropriate for use with the algorithm specified in the
-    #   x-amz-server-side​-encryption​-customer-algorithm header.
+    #   discarded; Amazon S3 does not store the encryption key. The key must
+    #   be appropriate for use with the algorithm specified in the
+    #   `x-amz-server-side​-encryption​-customer-algorithm` header.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
     #   RFC 1321. Amazon S3 uses this header for a message integrity check to
-    #   ensure the encryption key was transmitted without error.
+    #   ensure that the encryption key was transmitted without error.
     # @option options [String] :request_payer
-    #   Confirms that the requester knows that she or he will be charged for
-    #   the request. Bucket owners need not specify this parameter in their
-    #   requests. Documentation on downloading objects from requester pays
-    #   buckets can be found at
-    #   http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+    #   Confirms that the requester knows that they will be charged for the
+    #   request. Bucket owners need not specify this parameter in their
+    #   requests. For information about downloading objects from requester
+    #   pays buckets, see [Downloading Objects in Requestor Pays Buckets][1]
+    #   in the *Amazon S3 Developer Guide*.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
     # @option options [Integer] :part_number
     #   Part number of the object being read. This is a positive integer
     #   between 1 and 10,000. Effectively performs a 'ranged' GET request
@@ -366,27 +377,31 @@ module Aws::S3
     #   time, otherwise return a 412 (precondition failed).
     # @option options [String] :range
     #   Downloads the specified range bytes of an object. For more information
-    #   about the HTTP Range header, go to
-    #   http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.
+    #   about the HTTP Range header, see
+    #   [http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35]().
     # @option options [String] :sse_customer_algorithm
-    #   Specifies the algorithm to use to when encrypting the object (e.g.,
-    #   AES256).
+    #   Specifies the algorithm to use to when encrypting the object (for
+    #   example, AES256).
     # @option options [String] :sse_customer_key
     #   Specifies the customer-provided encryption key for Amazon S3 to use in
     #   encrypting data. This value is used to store the object and then it is
-    #   discarded; Amazon does not store the encryption key. The key must be
-    #   appropriate for use with the algorithm specified in the
-    #   x-amz-server-side​-encryption​-customer-algorithm header.
+    #   discarded; Amazon S3 does not store the encryption key. The key must
+    #   be appropriate for use with the algorithm specified in the
+    #   `x-amz-server-side​-encryption​-customer-algorithm` header.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
     #   RFC 1321. Amazon S3 uses this header for a message integrity check to
-    #   ensure the encryption key was transmitted without error.
+    #   ensure that the encryption key was transmitted without error.
     # @option options [String] :request_payer
-    #   Confirms that the requester knows that she or he will be charged for
-    #   the request. Bucket owners need not specify this parameter in their
-    #   requests. Documentation on downloading objects from requester pays
-    #   buckets can be found at
-    #   http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+    #   Confirms that the requester knows that they will be charged for the
+    #   request. Bucket owners need not specify this parameter in their
+    #   requests. For information about downloading objects from requester
+    #   pays buckets, see [Downloading Objects in Requestor Pays Buckets][1]
+    #   in the *Amazon S3 Developer Guide*.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
     # @option options [Integer] :part_number
     #   Part number of the object being read. This is a positive integer
     #   between 1 and 10,000. Effectively performs a 'ranged' HEAD request
@@ -475,15 +490,21 @@ module Aws::S3
       # @option options [String] :mfa
       #   The concatenation of the authentication device's serial number, a
       #   space, and the value that is displayed on your authentication device.
+      #   Required to permanently delete a versioned object if versioning is
+      #   configured with MFA delete enabled.
       # @option options [String] :request_payer
-      #   Confirms that the requester knows that she or he will be charged for
-      #   the request. Bucket owners need not specify this parameter in their
-      #   requests. Documentation on downloading objects from requester pays
-      #   buckets can be found at
-      #   http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+      #   Confirms that the requester knows that they will be charged for the
+      #   request. Bucket owners need not specify this parameter in their
+      #   requests. For information about downloading objects from requester
+      #   pays buckets, see [Downloading Objects in Requestor Pays Buckets][1]
+      #   in the *Amazon S3 Developer Guide*.
+      #
+      #
+      #
+      #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
       # @option options [Boolean] :bypass_governance_retention
       #   Specifies whether you want to delete this object even if it has a
-      #   Governance-type object lock in place. You must have sufficient
+      #   Governance-type Object Lock in place. You must have sufficient
       #   permissions to perform this operation.
       # @return [void]
       def batch_delete!(options = {})

data/lib/aws-sdk-s3/plugins/bucket_arn.rb

@@ -0,0 +1,211 @@
+module Aws
+  module S3
+    module Plugins
+      # When an accesspoint ARN is provided for :bucket in S3 operations, this
+      # plugin resolves the request endpoint from the ARN when possible.
+      class BucketARN < Seahorse::Client::Plugin
+        option(
+          :s3_use_arn_region,
+          default: true,
+          doc_type: 'Boolean',
+          docstring: <<-DOCS) do |cfg|
+By default, the SDK will use the S3 ARN region, and cross-region
+requests could be made. Set to `false` to not use the region from
+the S3 ARN.
+          DOCS
+          resolve_s3_use_arn_region(cfg)
+        end
+
+        def add_handlers(handlers, _config)
+          handlers.add(Handler)
+        end
+
+        # @api private
+        class Handler < Seahorse::Client::Handler
+          def call(context)
+            bucket_member = _bucket_member(context.operation.input.shape)
+            if bucket_member && (bucket = context.params[bucket_member])
+              _resolved_bucket, _resolved_region, arn = BucketARN.resolve_arn!(
+                bucket,
+                context.config.region,
+                context.config.s3_use_arn_region
+              )
+              if arn
+                if arn.resource.start_with?('accesspoint')
+                  validate_config!(context.config)
+                end
+
+                dualstack = extract_dualstack_config!(context)
+
+                BucketARN.resolve_url!(
+                  context.http_request.endpoint,
+                  arn,
+                  dualstack
+                )
+              end
+            end
+            @handler.call(context)
+          end
+
+          private
+
+          def _bucket_member(input)
+            input.members.each do |member, ref|
+              return member if ref.shape.name == 'BucketName'
+            end
+            nil
+          end
+
+          # other plugins use dualstack so disable it when we're done
+          def extract_dualstack_config!(context)
+            dualstack = context[:use_dualstack_endpoint]
+            context[:use_dualstack_endpoint] = false if dualstack
+            dualstack
+          end
+
+          def validate_config!(config)
+            unless config.regional_endpoint
+              raise ArgumentError,
+                    'Cannot provide both an accesspoint ARN and :endpoint.'
+            end
+
+            if config.use_accelerate_endpoint
+              raise ArgumentError,
+                    'Cannot provide both an accesspoint ARN and setting '\
+                    ':use_accelerate_endpoint to true.'
+            end
+
+            if config.force_path_style
+              raise ArgumentError,
+                    'Cannot provide both an accesspoint ARN and setting '\
+                    ':force_path_style to true.'
+            end
+          end
+        end
+
+        class << self
+
+          # @api private
+          def resolve_arn!(bucket_name, region, s3_use_arn_region)
+            if Aws::ARNParser.arn?(bucket_name)
+              arn = Aws::ARNParser.parse(bucket_name)
+              validate_s3_arn!(arn)
+              validate_region!(arn, region, s3_use_arn_region)
+              if arn.resource.start_with?('accesspoint')
+                region = arn.region if s3_use_arn_region
+                [bucket_name, region, arn]
+              else
+                raise ArgumentError,
+                      'Only accesspoint type ARNs are currently supported.'
+              end
+            else
+              [bucket_name, region]
+            end
+          end
+
+          # @api private
+          def resolve_url!(url, arn, dualstack = false)
+            if arn.resource.start_with?('accesspoint')
+              url.host = accesspoint_arn_host(arn, dualstack)
+            else
+              raise ArgumentError,
+                    'Only accesspoint type ARNs are currently supported.'
+            end
+            url.path = url_path(url.path, arn)
+            url
+          end
+
+          private
+
+          def accesspoint_arn_host(arn, dualstack)
+            _resource_type, resource_name = parse_resource(arn.resource)
+            accesspoint = "#{resource_name}-#{arn.account_id}"
+            accesspoint << '.s3-accesspoint'
+            accesspoint << '.dualstack' if dualstack
+            sfx = Aws::Partitions::EndpointProvider.dns_suffix_for(arn.region)
+            accesspoint << ".#{arn.region}.#{sfx}"
+            accesspoint
+          end
+
+          def parse_resource(str)
+            slash = str.index('/') || str.length
+            colon = str.index(':') || str.length
+            delimiter = slash < colon ? slash : colon
+            if delimiter < str.length
+              [str[0..(delimiter - 1)], str[(delimiter + 1)..-1]]
+            else
+              [nil, str]
+            end
+          end
+
+          def resolve_s3_use_arn_region(cfg)
+            value = ENV['AWS_S3_USE_ARN_REGION'] ||
+                    Aws.shared_config.s3_use_arn_region(profile: cfg.profile) ||
+                    'true'
+
+            # Raise if provided value is not true or false
+            if value != 'true' && value != 'false'
+              raise ArgumentError,
+                    'Must provide either `true` or `false` for '\
+                    's3_use_arn_region profile option or for '\
+                    'ENV[\'AWS_S3_USE_ARN_REGION\']'
+            end
+
+            value == 'true'
+          end
+
+          def url_path(path, arn)
+            path = path.sub("/#{Seahorse::Util.uri_escape(arn.to_s)}", '')
+                       .sub("/#{arn}", '')
+            "/#{path}" unless path.match(/^\//)
+            path
+          end
+
+          def validate_s3_arn!(arn)
+            _resource_type, resource_name = parse_resource(arn.resource)
+
+            unless arn.service == 's3'
+              raise ArgumentError, 'Must provide an S3 ARN.'
+            end
+
+            if arn.region.empty? || arn.account_id.empty?
+              raise ArgumentError,
+                    'S3 Access Point ARNs must contain both a valid region '\
+                    ' and a valid account id.'
+            end
+
+            if resource_name.include?(':') || resource_name.include?('/')
+              raise ArgumentError,
+                    'ARN resource id must be a single value.'
+            end
+
+            unless Plugins::BucketDns.valid_subdomain?(
+              "#{resource_name}-#{arn.account_id}"
+            )
+              raise ArgumentError,
+                    "#{resource_name}-#{arn.account_id} is not a "\
+                    'valid subdomain.'
+            end
+          end
+
+          def validate_region!(arn, region, s3_use_arn_region)
+            if region.include?('fips')
+              raise ArgumentError,
+                    'FIPS client regions are currently not supported with '\
+                    'accesspoint ARNs.'
+            end
+
+            if s3_use_arn_region &&
+               !Aws::Partitions.partition(arn.partition).region?(region)
+              raise Aws::Errors::InvalidARNPartitionError
+            end
+
+            if !s3_use_arn_region && region != arn.region
+              raise Aws::Errors::InvalidARNRegionError
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/bucket_dns.rb

@@ -38,11 +38,9 @@ request URI and never moved to the host as a sub-domain.
           def move_dns_compat_bucket_to_subdomain(context)
             bucket_name = context.params[:bucket]
             endpoint = context.http_request.endpoint
-            if
-              bucket_name &&
-              BucketDns.dns_compatible?(bucket_name, https?(endpoint)) &&
-              context.operation_name.to_s != 'get_bucket_location'
-            then
+            if bucket_name &&
+               BucketDns.dns_compatible?(bucket_name, https?(endpoint)) &&
+               context.operation_name.to_s != 'get_bucket_location'
               move_bucket_to_subdomain(bucket_name, endpoint)
             end
           end
@@ -73,8 +71,10 @@ request URI and never moved to the host as a sub-domain.
             end
           end
 
-          private
-
+          # Checks for a valid RFC-3986 host name
+          # @see https://tools.ietf.org/html/rfc3986#section-3.2.2
+          # @param [String] bucket_name
+          # @return [Boolean]
           def valid_subdomain?(bucket_name)
             bucket_name.size < 64 &&
             bucket_name =~ /^[a-z0-9][a-z0-9.-]+[a-z0-9]$/ &&

data/lib/aws-sdk-s3/plugins/bucket_name_restrictions.rb

@@ -5,14 +5,34 @@ module Aws
       class BucketNameRestrictions < Seahorse::Client::Plugin
         class Handler < Seahorse::Client::Handler
 
+          # Useful because Aws::S3::Errors::SignatureDoesNotMatch is thrown
+          # when passed a bucket with a forward slash. Instead provide a more
+          # helpful error. Ideally should not be a plugin?
           def call(context)
-            if context.params.key?(:bucket) && context.params[:bucket].include?('/')
-              msg = ":bucket option must not contain a forward-slash (/)"
-              raise ArgumentError, msg
+            bucket_member = _bucket_member(context.operation.input.shape)
+            if bucket_member && (bucket = context.params[bucket_member])
+              _resolved_bucket, _resolved_region, arn = BucketARN.resolve_arn!(
+                bucket,
+                context.config.region,
+                context.config.s3_use_arn_region
+              )
+              if !arn && bucket.include?('/')
+                raise ArgumentError,
+                      'bucket name must not contain a forward-slash (/)'
+              end
             end
             @handler.call(context)
           end
 
+          private
+
+          def _bucket_member(input)
+            input.members.each do |member, ref|
+              return member if ref.shape.name == 'BucketName'
+            end
+            nil
+          end
+
         end
 
         handler(Handler)

data/lib/aws-sdk-s3/plugins/iad_regional_endpoint.rb

@@ -0,0 +1,59 @@
+module Aws
+  module S3
+    module Plugins
+
+      class IADRegionalEndpoint < Seahorse::Client::Plugin
+        
+        option(:s3_us_east_1_regional_endpoint,
+          default: 'legacy',
+          doc_type: String,
+          docstring: <<-DOCS) do |cfg|
+Passing in `regional` to enable regional endpoint for S3's `us-east-1`
+region. Defaults to `legacy` mode using global endpoint.
+          DOCS
+          resolve_iad_regional_endpoint(cfg)
+        end
+
+        def add_handlers(handlers, config)
+          if config.region == 'us-east-1'
+            handlers.add(Handler)
+          end
+        end
+
+        # @api private
+        class Handler < Seahorse::Client::Handler
+
+          def call(context)
+            # keep legacy global endpoint pattern by default
+            if context.config.s3_us_east_1_regional_endpoint == 'legacy'
+              context.http_request.endpoint.host = IADRegionalEndpoint.legacy_host(
+                context.http_request.endpoint.host)
+            end
+            @handler.call(context)
+          end
+
+        end
+
+        def self.legacy_host(host)
+          host.sub(".us-east-1", '')
+        end
+
+        private
+
+        def self.resolve_iad_regional_endpoint(cfg)
+          mode = ENV['AWS_S3_US_EAST_1_REGIONAL_ENDPOINT'] ||
+            Aws.shared_config.s3_us_east_1_regional_endpoint(profile: cfg.profile) ||
+            'legacy'
+          unless %w(legacy regional).include?(mode)
+            raise ArgumentError, "expected :s3_us_east_1_regional_endpoint or"\
+              " ENV['AWS_S3_US_EAST_1_REGIONAL_ENDPOINT'] to be `legacy` or"\
+              " `regional`."
+          end
+          mode
+        end
+
+      end
+
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/md5s.rb

@@ -1,5 +1,4 @@
 require 'openssl'
-require 'base64'
 
 module Aws
   module S3
@@ -37,13 +36,13 @@ module Aws
           # @return [String<MD5>]
           def md5(value)
             if (File === value || Tempfile === value) && !value.path.nil? && File.exist?(value.path)
-              Base64.encode64(OpenSSL::Digest::MD5.file(value).digest).strip
+              OpenSSL::Digest::MD5.file(value).base64digest
             elsif value.respond_to?(:read)
               md5 = OpenSSL::Digest::MD5.new
               update_in_chunks(md5, value)
-              Base64.encode64(md5.digest).strip
+              md5.base64digest
             else
-              Base64.encode64(OpenSSL::Digest::MD5.digest(value)).strip
+              OpenSSL::Digest::MD5.digest(value).base64digest
             end
           end
 

data/lib/aws-sdk-s3/plugins/s3_signer.rb

@@ -17,6 +17,8 @@ module Aws
         end
 
         option(:sigv4_region) do |cfg|
+          raise Aws::Errors::MissingRegionError if cfg.region.nil?
+
           Aws::Partitions::EndpointProvider.signing_region(cfg.region, 's3')
         end
 
@@ -197,15 +199,23 @@ module Aws
           end
 
           def new_hostname(context, region)
-            bucket = context.params[:bucket]
-            if region == 'us-east-1'
-              "#{bucket}.s3.amazonaws.com"
+            # Check to see if the bucket is actually an ARN and resolve it
+            # Otherwise it will retry with the ARN as the bucket name.
+            resolved_bucket, resolved_region, arn = BucketARN.resolve_arn!(
+              context.params[:bucket],
+              region,
+              context.config.s3_use_arn_region
+            )
+            uri = URI.parse(
+              Aws::Partitions::EndpointProvider.resolve(resolved_region, 's3')
+            )
+
+            if arn
+              BucketARN.resolve_url!(uri, arn).host
             else
-              endpoint = Aws::Partitions::EndpointProvider.resolve(region, 's3')
-              bucket + '.' + URI.parse(endpoint).host
+              resolved_bucket + '.' + uri.host
             end
           end
-
         end
       end
     end

data/lib/aws-sdk-s3/presigned_post.rb

@@ -585,6 +585,10 @@ module Aws
         else
           url.path = '/' + @bucket_name
         end
+        if @bucket_region == 'us-east-1'
+          # keep legacy behavior by default
+          url.host = Plugins::IADRegionalEndpoint.legacy_host(url.host)
+        end
         url.to_s
       end
 

data/lib/aws-sdk-s3/presigner.rb

@@ -103,9 +103,13 @@ module Aws
       def expires_in(params)
         if (expires_in = params.delete(:expires_in))
           if expires_in > ONE_WEEK
-            msg = "expires_in value of #{expires_in} exceeds one-week maximum"
-            raise ArgumentError, msg
+            raise ArgumentError,
+                  "expires_in value of #{expires_in} exceeds one-week maximum"
+          elsif expires_in <= 0 
+            raise ArgumentError,
+                  "expires_in value of #{expires_in} cannot be 0 or less"
           end
+
           expires_in
         else
           FIFTEEN_MINUTES

data/lib/aws-sdk-s3/resource.rb

@@ -40,7 +40,9 @@ module Aws::S3
     # @option options [String] :acl
     #   The canned ACL to apply to the bucket.
     # @option options [required, String] :bucket
+    #   The name of the bucket to create.
     # @option options [Types::CreateBucketConfiguration] :create_bucket_configuration
+    #   The configuration information for the bucket.
     # @option options [String] :grant_full_control
     #   Allows grantee the read, write, read ACP, and write ACP permissions on
     #   the bucket.
@@ -54,8 +56,8 @@ module Aws::S3
     # @option options [String] :grant_write_acp
     #   Allows grantee to write the ACL for the applicable bucket.
     # @option options [Boolean] :object_lock_enabled_for_bucket
-    #   Specifies whether you want Amazon S3 object lock to be enabled for the
-    #   new bucket.
+    #   Specifies whether you want S3 Object Lock to be enabled for the new
+    #   bucket.
     # @return [Bucket]
     def create_bucket(options = {})
       resp = @client.create_bucket(options)