aws-sdk-s3 1.30.1 → 1.112.0

data/lib/aws-sdk-s3/presigned_post.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'openssl'
 require 'base64'
 
@@ -174,6 +176,7 @@ module Aws
     # ```
     #
     class PresignedPost
+      @@allowed_fields = []
 
       # @param [Credentials] credentials Security credentials for signing
       #   the post policy.
@@ -183,35 +186,59 @@ module Aws
       #   the post will expire. Defaults to one hour from creation of the
       #   presigned post. May not exceed one week from creation time.
       # @option options [String] :key See {PresignedPost#key}.
-      # @option options [String] :key_starts_with See {PresignedPost#key_starts_with}.
+      # @option options [String] :key_starts_with
+      #   See {PresignedPost#key_starts_with}.
       # @option options [String] :acl See {PresignedPost#acl}.
-      # @option options [String] :acl_starts_with See {PresignedPost#acl_starts_with}.
-      # @option options [String] :cache_control See {PresignedPost#cache_control}.
-      # @option options [String] :cache_control_starts_with See {PresignedPost#cache_control_starts_with}.
+      # @option options [String] :acl_starts_with
+      #   See {PresignedPost#acl_starts_with}.
+      # @option options [String] :cache_control
+      #   See {PresignedPost#cache_control}.
+      # @option options [String] :cache_control_starts_with
+      #   See {PresignedPost#cache_control_starts_with}.
       # @option options [String] :content_type See {PresignedPost#content_type}.
-      # @option options [String] :content_type_starts_with See {PresignedPost#content_type_starts_with}.
-      # @option options [String] :content_disposition See {PresignedPost#content_disposition}.
-      # @option options [String] :content_disposition_starts_with See {PresignedPost#content_disposition_starts_with}.
-      # @option options [String] :content_encoding See {PresignedPost#content_encoding}.
-      # @option options [String] :content_encoding_starts_with See {PresignedPost#content_encoding_starts_with}.
+      # @option options [String] :content_type_starts_with
+      #   See {PresignedPost#content_type_starts_with}.
+      # @option options [String] :content_disposition
+      #   See {PresignedPost#content_disposition}.
+      # @option options [String] :content_disposition_starts_with
+      #   See {PresignedPost#content_disposition_starts_with}.
+      # @option options [String] :content_encoding
+      #   See {PresignedPost#content_encoding}.
+      # @option options [String] :content_encoding_starts_with
+      #   See {PresignedPost#content_encoding_starts_with}.
       # @option options [String] :expires See {PresignedPost#expires}.
-      # @option options [String] :expires_starts_with See {PresignedPost#expires_starts_with}.
-      # @option options [Range<Integer>] :content_length_range See {PresignedPost#content_length_range}.
-      # @option options [String] :success_action_redirect See {PresignedPost#success_action_redirect}.
-      # @option options [String] :success_action_redirect_starts_with See {PresignedPost#success_action_redirect_starts_with}.
-      # @option options [String] :success_action_status See {PresignedPost#success_action_status}.
-      # @option options [String] :storage_class See {PresignedPost#storage_class}.
-      # @option options [String] :website_redirect_location See {PresignedPost#website_redirect_location}.
-      # @option options [Hash<String,String>] :metadata See {PresignedPost#metadata}.
-      # @option options [Hash<String,String>] :metadata_starts_with See {PresignedPost#metadata_starts_with}.
-      # @option options [String] :server_side_encryption See {PresignedPost#server_side_encryption}.
-      # @option options [String] :server_side_encryption_aws_kms_key_id See {PresignedPost#server_side_encryption_aws_kms_key_id}.
-      # @option options [String] :server_side_encryption_customer_algorithm See {PresignedPost#server_side_encryption_customer_algorithm}.
-      # @option options [String] :server_side_encryption_customer_key See {PresignedPost#server_side_encryption_customer_key}.
+      # @option options [String] :expires_starts_with
+      #   See {PresignedPost#expires_starts_with}.
+      # @option options [Range<Integer>] :content_length_range
+      #   See {PresignedPost#content_length_range}.
+      # @option options [String] :success_action_redirect
+      #   See {PresignedPost#success_action_redirect}.
+      # @option options [String] :success_action_redirect_starts_with
+      #   See {PresignedPost#success_action_redirect_starts_with}.
+      # @option options [String] :success_action_status
+      #   See {PresignedPost#success_action_status}.
+      # @option options [String] :storage_class
+      #   See {PresignedPost#storage_class}.
+      # @option options [String] :website_redirect_location
+      #   See {PresignedPost#website_redirect_location}.
+      # @option options [Hash<String,String>] :metadata
+      #   See {PresignedPost#metadata}.
+      # @option options [Hash<String,String>] :metadata_starts_with
+      #   See {PresignedPost#metadata_starts_with}.
+      # @option options [String] :server_side_encryption
+      #   See {PresignedPost#server_side_encryption}.
+      # @option options [String] :server_side_encryption_aws_kms_key_id
+      #   See {PresignedPost#server_side_encryption_aws_kms_key_id}.
+      # @option options [String] :server_side_encryption_customer_algorithm
+      #   See {PresignedPost#server_side_encryption_customer_algorithm}.
+      # @option options [String] :server_side_encryption_customer_key
+      #   See {PresignedPost#server_side_encryption_customer_key}.
       def initialize(credentials, bucket_region, bucket_name, options = {})
         @credentials = credentials.credentials
         @bucket_region = bucket_region
         @bucket_name = bucket_name
+        @accelerate = !!options.delete(:use_accelerate_endpoint)
+        options.delete(:url) if @accelerate # resource methods pass url
         @url = options.delete(:url) || bucket_url
         @fields = {}
         @key_set = false
@@ -221,7 +248,12 @@ module Aws
           case option_name
           when :allow_any then allow_any(option_value)
           when :signature_expiration then @signature_expiration = option_value
-          else send("#{option_name}", option_value)
+          else
+            if @@allowed_fields.include?(option_name)
+              send("#{option_name}", option_value)
+            else
+              raise ArgumentError, "Unsupported option: #{option_name}"
+            end
           end
         end
       end
@@ -234,7 +266,7 @@ module Aws
       #   as hidden input fields.
       def fields
         check_required_values!
-        datetime = Time.now.utc.strftime("%Y%m%dT%H%M%SZ")
+        datetime = Time.now.utc.strftime('%Y%m%dT%H%M%SZ')
         fields = @fields.dup
         fields.update('policy' => policy(datetime))
         fields.update(signature_fields(datetime))
@@ -253,24 +285,30 @@ module Aws
       end
 
       # @api private
-      def self.define_field(field, *args)
+      def self.define_field(field, *args, &block)
+        @@allowed_fields << field
         options = args.last.is_a?(Hash) ? args.pop : {}
         field_name = args.last || field.to_s
 
-        define_method("#{field}") do |value|
-          with(field_name, value)
-        end
+        if block_given?
+          define_method("#{field}", block)
+        else
+          define_method("#{field}") do |value|
+            with(field_name, value)
+          end
 
-        if options[:starts_with]
-          define_method("#{field}_starts_with") do |value|
-            starts_with(field_name, value)
+          if options[:starts_with]
+            @@allowed_fields << "#{field}_starts_with".to_sym
+            define_method("#{field}_starts_with") do |value|
+              starts_with(field_name, value)
+            end
           end
         end
       end
 
       # @!group Fields
 
-      # The key to use for the uploaded object. Use can use `${filename}`
+      # The key to use for the uploaded object. You can use `${filename}`
       # as a variable in the key. This will be replaced with the name
       # of the file as provided by the user.
       #
@@ -281,7 +319,7 @@ module Aws
       # @param [String] key
       # @see http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html)
       # @return [self]
-      def key(key)
+      define_field(:key) do |key|
         @key_set = true
         with('key', key)
       end
@@ -290,7 +328,7 @@ module Aws
       # @param [String] prefix
       # @see #key
       # @return [self]
-      def key_starts_with(prefix)
+      define_field(:key_starts_with) do |prefix|
         @key_set = true
         starts_with('key', prefix)
       end
@@ -373,21 +411,21 @@ module Aws
       # @param [Time] time
       # @see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21
       # @return [self]
-      def expires(time)
+      define_field(:expires) do |time|
         with('Expires', time.httpdate)
       end
 
       # @param [String] prefix
       # @see #expires
       # @return [self]
-      def expires_starts_with(prefix)
+      define_field(:expires_starts_with) do |prefix|
         starts_with('Expires', prefix)
       end
 
       # The minimum and maximum allowable size for the uploaded content.
       # @param [Range<Integer>] byte_range
       # @return [self]
-      def content_length_range(byte_range)
+      define_field(:content_length_range) do |byte_range|
         min = byte_range.begin
         max = byte_range.end
         max -= 1 if byte_range.exclude_end?
@@ -466,7 +504,7 @@ module Aws
       # prefixed with "x-amz-meta-".
       # @param [Hash<String,String>] hash
       # @return [self]
-      def metadata(hash)
+      define_field(:metadata) do |hash|
         hash.each do |key, value|
           with("x-amz-meta-#{key}", value)
         end
@@ -477,7 +515,7 @@ module Aws
       # @param [Hash<String,String>] hash
       # @see #metadata
       # @return [self]
-      def metadata_starts_with(hash)
+      define_field(:metadata_starts_with) do |hash|
         hash.each do |key, value|
           starts_with("x-amz-meta-#{key}", value)
         end
@@ -505,7 +543,10 @@ module Aws
       #   (KMS) master encryption key to use for the object.
       #   @param [String] value
       #   @return [self]
-      define_field(:server_side_encryption_aws_kms_key_id, 'x-amz-server-side-encryption-aws-kms-key-id')
+      define_field(
+        :server_side_encryption_aws_kms_key_id,
+        'x-amz-server-side-encryption-aws-kms-key-id'
+      )
 
       # @!endgroup
 
@@ -518,7 +559,10 @@ module Aws
       #   @param [String] value
       #   @see #server_side_encryption_customer_key
       #   @return [self]
-      define_field(:server_side_encryption_customer_algorithm, 'x-amz-server-side-encryption-customer-algorithm')
+      define_field(
+        :server_side_encryption_customer_algorithm,
+        'x-amz-server-side-encryption-customer-algorithm'
+      )
 
       # Specifies the customer-provided encryption key for Amazon S3 to use
       # in encrypting data. This value is used to store the object and then
@@ -529,7 +573,7 @@ module Aws
       # @param [String] value
       # @see #server_side_encryption_customer_algorithm
       # @return [self]
-      def server_side_encryption_customer_key(value)
+      define_field(:server_side_encryption_customer_key) do |value|
         field_name = 'x-amz-server-side-encryption-customer-key'
         with(field_name, base64(value))
         with(field_name + '-MD5', base64(OpenSSL::Digest::MD5.digest(value)))
@@ -538,7 +582,7 @@ module Aws
       # @param [String] prefix
       # @see #server_side_encryption_customer_key
       # @return [self]
-      def server_side_encryption_customer_key_starts_with(prefix)
+      define_field(:server_side_encryption_customer_key_starts_with) do |prefix|
         field_name = 'x-amz-server-side-encryption-customer-key'
         starts_with(field_name, prefix)
       end
@@ -571,8 +615,8 @@ module Aws
 
       def check_required_values!
         unless @key_set
-          msg = "key required; you must provide a key via :key, "
-          msg << ":key_starts_with, or :allow_any => ['key']"
+          msg = 'key required; you must provide a key via :key, '\
+                ":key_starts_with, or :allow_any => ['key']"
           raise msg
         end
       end
@@ -580,10 +624,18 @@ module Aws
       def bucket_url
         url = Aws::Partitions::EndpointProvider.resolve(@bucket_region, 's3')
         url = URI.parse(url)
-        if Plugins::BucketDns.dns_compatible?(@bucket_name, true)
-          url.host = @bucket_name + '.' + url.host
+        if Plugins::BucketDns.dns_compatible?(@bucket_name, _ssl = true)
+          if @accelerate
+            url.host = "#{@bucket_name}.s3-accelerate.amazonaws.com"
+          else
+            url.host = "#{@bucket_name}.#{url.host}"
+          end
         else
-          url.path = '/' + @bucket_name
+          url.path = "/#{@bucket_name}"
+        end
+        if @bucket_region == 'us-east-1'
+          # keep legacy behavior by default
+          url.host = Plugins::IADRegionalEndpoint.legacy_host(url.host)
         end
         url.to_s
       end
@@ -613,7 +665,7 @@ module Aws
 
       def signature(datetime, string_to_sign)
         k_secret = @credentials.secret_access_key
-        k_date = hmac("AWS4" + k_secret, datetime[0,8])
+        k_date = hmac('AWS4' + k_secret, datetime[0,8])
         k_region = hmac(k_date, @bucket_region)
         k_service = hmac(k_region, 's3')
         k_credentials = hmac(k_service, 'aws4_request')

data/lib/aws-sdk-s3/presigner.rb

@@ -1,27 +1,94 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
-
-    # Allows you to create presigned URLs for S3 operations.
-    #
-    # Example Use:
-    #
-    #      signer = Aws::S3::Presigner.new
-    #      url = signer.presigned_url(:get_object, bucket: "bucket", key: "key")
-    #
     class Presigner
-
       # @api private
       ONE_WEEK = 60 * 60 * 24 * 7
 
       # @api private
       FIFTEEN_MINUTES = 60 * 15
 
+      # @api private
+      BLACKLISTED_HEADERS = [
+        'accept',
+        'amz-sdk-request',
+        'cache-control',
+        'content-length', # due to a ELB bug
+        'expect',
+        'from',
+        'if-match',
+        'if-none-match',
+        'if-modified-since',
+        'if-unmodified-since',
+        'if-range',
+        'max-forwards',
+        'pragma',
+        'proxy-authorization',
+        'referer',
+        'te',
+        'user-agent'
+      ].freeze
+
       # @option options [Client] :client Optionally provide an existing
       #   S3 client
       def initialize(options = {})
         @client = options[:client] || Aws::S3::Client.new
       end
 
+      # Create presigned URLs for S3 operations.
+      #
+      # @example
+      #  signer = Aws::S3::Presigner.new
+      #  url = signer.presigned_url(:get_object, bucket: "bucket", key: "key")
+      #
+      # @param [Symbol] method Symbolized method name of the operation you want
+      #   to presign.
+      #
+      # @option params [Integer] :expires_in (900) The number of seconds
+      #   before the presigned URL expires. Defaults to 15 minutes. As signature
+      #   version 4 has a maximum expiry time of one week for presigned URLs,
+      #   attempts to set this value to greater than one week (604800) will
+      #   raise an exception.
+      #
+      # @option params [Time] :time (Time.now) The starting time for when the
+      #   presigned url becomes active.
+      #
+      # @option params [Boolean] :secure (true) When `false`, a HTTP URL
+      #   is returned instead of the default HTTPS URL.
+      #
+      # @option params [Boolean] :virtual_host (false) When `true`, the
+      #   bucket name will be used as the hostname.
+      #
+      # @option params [Boolean] :use_accelerate_endpoint (false) When `true`,
+      #   Presigner will attempt to use accelerated endpoint.
+      #
+      # @option params [Array<String>] :whitelist_headers ([]) Additional
+      #   headers to be included for the signed request. Certain headers beyond
+      #   the authorization header could, in theory, be changed for various
+      #   reasons (including but not limited to proxies) while in transit and
+      #   after signing. This would lead to signature errors being returned,
+      #   despite no actual problems with signing. (see BLACKLISTED_HEADERS)
+      #
+      # @raise [ArgumentError] Raises an ArgumentError if `:expires_in`
+      #   exceeds one week.
+      #
+      # @return [String] a presigned url
+      def presigned_url(method, params = {})
+        url, _headers = _presigned_request(method, params)
+        url
+      end
+
+      # Allows you to create presigned URL requests for S3 operations. This
+      # method returns a tuple containing the URL and the signed X-amz-* headers
+      # to be used with the presigned url.
+      #
+      # @example
+      #  signer = Aws::S3::Presigner.new
+      #  url, headers = signer.presigned_request(
+      #    :get_object, bucket: "bucket", key: "key"
+      #  )
+      #
       # @param [Symbol] method Symbolized method name of the operation you want
       #   to presign.
       #
@@ -31,6 +98,9 @@ module Aws
       #   attempts to set this value to greater than one week (604800) will
       #   raise an exception.
       #
+      # @option params [Time] :time (Time.now) The starting time for when the
+      #   presigned url becomes active.
+      #
       # @option params [Boolean] :secure (true) When `false`, a HTTP URL
       #   is returned instead of the default HTTPS URL.
       #
@@ -38,29 +108,51 @@ module Aws
       #   bucket name will be used as the hostname. This will cause
       #   the returned URL to be 'http' and not 'https'.
       #
-      # @option params [Boolean] :use_accelerate_endpoint (false) When `true`, Presigner
-      #   will attempt to use accelerated endpoint
+      # @option params [Boolean] :use_accelerate_endpoint (false) When `true`,
+      #   Presigner will attempt to use accelerated endpoint.
+      #
+      # @option params [Array<String>] :whitelist_headers ([]) Additional
+      #   headers to be included for the signed request. Certain headers beyond
+      #   the authorization header could, in theory, be changed for various
+      #   reasons (including but not limited to proxies) while in transit and
+      #   after signing. This would lead to signature errors being returned,
+      #   despite no actual problems with signing. (see BLACKLISTED_HEADERS)
       #
       # @raise [ArgumentError] Raises an ArgumentError if `:expires_in`
       #   exceeds one week.
       #
-      def presigned_url(method, params = {})
-        if params[:key].nil? or params[:key] == ''
-          raise ArgumentError, ":key must not be blank"
-        end
-        virtual_host = !!params.delete(:virtual_host)
-        scheme = http_scheme(params, virtual_host)
+      # @return [String, Hash] A tuple with a presigned URL and headers that
+      #   should be included with the request.
+      def presigned_request(method, params = {})
+        _presigned_request(method, params, false)
+      end
+
+      private
+
+      def _presigned_request(method, params, hoist = true)
+        virtual_host = params.delete(:virtual_host)
+        time = params.delete(:time)
+        unsigned_headers = unsigned_headers(params)
+        scheme = http_scheme(params)
+        expires_in = expires_in(params)
 
         req = @client.build_request(method, params)
         use_bucket_as_hostname(req) if virtual_host
-        sign_but_dont_send(req, expires_in(params), scheme)
-        req.send_request.data
+        handle_presigned_url_context(req)
+
+        x_amz_headers = sign_but_dont_send(
+          req, expires_in, scheme, time, unsigned_headers, hoist
+        )
+        [req.send_request.data, x_amz_headers]
       end
 
-      private
+      def unsigned_headers(params)
+        whitelist_headers = params.delete(:whitelist_headers) || []
+        BLACKLISTED_HEADERS - whitelist_headers
+      end
 
-      def http_scheme(params, virtual_host)
-        if params.delete(:secure) == false || virtual_host
+      def http_scheme(params)
+        if params.delete(:secure) == false
           'http'
         else
           @client.config.endpoint.scheme
@@ -68,10 +160,13 @@ module Aws
       end
 
       def expires_in(params)
-        if expires_in = params.delete(:expires_in)
+        if (expires_in = params.delete(:expires_in))
           if expires_in > ONE_WEEK
-            msg = "expires_in value of #{expires_in} exceeds one-week maximum"
-            raise ArgumentError, msg
+            raise ArgumentError,
+                  "expires_in value of #{expires_in} exceeds one-week maximum."
+          elsif expires_in <= 0
+            raise ArgumentError,
+                  "expires_in value of #{expires_in} cannot be 0 or less."
           end
           expires_in
         else
@@ -85,14 +180,26 @@ module Aws
           uri = context.http_request.endpoint
           uri.host = context.params[:bucket]
           uri.path.sub!("/#{context.params[:bucket]}", '')
-          uri.scheme = 'http'
-          uri.port = 80
+          @handler.call(context)
+        end
+      end
+
+      # Used for excluding presigned_urls from API request count.
+      #
+      # Store context information as early as possible, to allow
+      # handlers to perform decisions based on this flag if need.
+      def handle_presigned_url_context(req)
+        req.handle(step: :initialize, priority: 98) do |context|
+          context[:presigned_url] = true
           @handler.call(context)
         end
       end
 
       # @param [Seahorse::Client::Request] req
-      def sign_but_dont_send(req, expires_in, scheme)
+      def sign_but_dont_send(
+        req, expires_in, scheme, time, unsigned_headers, hoist = true
+      )
+        x_amz_headers = {}
 
         http_req = req.context.http_request
 
@@ -100,11 +207,7 @@ module Aws
         req.handlers.remove(Aws::S3::Plugins::S3Signer::V4Handler)
         req.handlers.remove(Seahorse::Client::Plugins::ContentLength::Handler)
 
-        signer = build_signer(req.context.config)
-        req.context[:presigned_url] = true
-
         req.handle(step: :send) do |context|
-
           if scheme != http_req.endpoint.scheme
             endpoint = http_req.endpoint.dup
             endpoint.scheme = scheme
@@ -112,57 +215,56 @@ module Aws
             http_req.endpoint = URI.parse(endpoint.to_s)
           end
 
-          # hoist x-amz-* headers to the querystring
           query = http_req.endpoint.query ? http_req.endpoint.query.split('&') : []
-          http_req.headers.keys.each do |key|
-            if key.match(/^x-amz/i)
-              value = Aws::Sigv4::Signer.uri_escape(http_req.headers.delete(key))
+          http_req.headers.each do |key, value|
+            next unless key =~ /^x-amz/i
+
+            if hoist
+              value = Aws::Sigv4::Signer.uri_escape(value)
               key = Aws::Sigv4::Signer.uri_escape(key)
+              # hoist x-amz-* headers to the querystring
+              http_req.headers.delete(key)
               query << "#{key}=#{value}"
+            else
+              x_amz_headers[key] = value
             end
           end
           http_req.endpoint.query = query.join('&') unless query.empty?
 
+          signing_algorithm = :sigv4
+
+          # If it's an ARN, get the resolved region and service
+          if (arn = context.metadata[:s3_arn])
+            region = arn[:resolved_region]
+            service = arn[:arn].service
+            region = arn[:arn].is_a?(MultiRegionAccessPointARN) ? '*': arn[:resolved_region]
+            signing_algorithm = arn[:arn].is_a?(MultiRegionAccessPointARN) ? :sigv4a : :sigv4
+          end
+
+          signer = Aws::Sigv4::Signer.new(
+            service: service || 's3',
+            region: region || context.config.region,
+            signing_algorithm: signing_algorithm,
+            credentials_provider: context.config.credentials,
+            unsigned_headers: unsigned_headers,
+            apply_checksum_header: false,
+            uri_escape_path: false
+          )
+
           url = signer.presign_url(
             http_method: http_req.http_method,
             url: http_req.endpoint,
             headers: http_req.headers,
             body_digest: 'UNSIGNED-PAYLOAD',
-            expires_in: expires_in
+            expires_in: expires_in,
+            time: time
           ).to_s
 
           Seahorse::Client::Response.new(context: context, data: url)
         end
+        # Return the headers
+        x_amz_headers
       end
-
-      def build_signer(cfg)
-        Aws::Sigv4::Signer.new(
-          service: 's3',
-          region: cfg.region,
-          credentials_provider: cfg.credentials,
-          unsigned_headers: [
-            'cache-control',
-            'content-length', # due to a ELB bug
-            'expect',
-            'max-forwards',
-            'pragma',
-            'te',
-            'if-match',
-            'if-none-match',
-            'if-modified-since',
-            'if-unmodified-since',
-            'if-range',
-            'accept',
-            'proxy-authorization',
-            'from',
-            'referer',
-            'user-agent',
-            'x-amzn-trace-id'
-          ],
-          uri_escape_path: false
-        )
-      end
-
     end
   end
 end