aws-sdk-s3 1.36.0 → 1.95.1

data/lib/aws-sdk-s3/plugins/s3_signer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'aws-sigv4'
 
 module Aws
@@ -6,17 +8,20 @@ module Aws
       # This plugin is an implementation detail and may be modified.
       # @api private
       class S3Signer < Seahorse::Client::Plugin
-
         option(:signature_version, 'v4')
 
         option(:sigv4_signer) do |cfg|
           S3Signer.build_v4_signer(
+            service: 's3',
             region: cfg.sigv4_region,
             credentials: cfg.credentials
           )
         end
 
         option(:sigv4_region) do |cfg|
+          # S3 removes core's signature_v4 plugin that checks for this
+          raise Aws::Errors::MissingRegionError if cfg.region.nil?
+
           Aws::Partitions::EndpointProvider.signing_region(cfg.region, 's3')
         end
 
@@ -48,7 +53,6 @@ module Aws
         end
 
         class V4Handler < Seahorse::Client::Handler
-
           def call(context)
             Aws::Plugins::SignatureV4.apply_signature(
               context: context,
@@ -62,25 +66,34 @@ module Aws
           def sigv4_signer(context)
             # If the client was configured with the wrong region,
             # we have to build a new signer.
-            if
-              context[:cached_sigv4_region] &&
-              context[:cached_sigv4_region] != context.config.sigv4_signer.region
-            then
+            if context[:cached_sigv4_region] &&
+               context[:cached_sigv4_region] != context.config.sigv4_signer.region
               S3Signer.build_v4_signer(
+                service: 's3',
                 region: context[:cached_sigv4_region],
                 credentials: context.config.credentials
               )
+            elsif (arn = context.metadata[:s3_arn])
+              S3Signer.build_v4_signer(
+                service: arn[:arn].service,
+                region: arn[:resolved_region],
+                credentials: context.config.credentials
+              )
+            elsif context.operation.name == 'WriteGetObjectResponse'
+              S3Signer.build_v4_signer(
+                service: 's3-object-lambda',
+                region: context.config.sigv4_region,
+                credentials: context.config.credentials
+              )
             else
               context.config.sigv4_signer
             end
           end
-
         end
 
         # This handler will update the http endpoint when the bucket region
         # is known/cached.
         class CachedBucketRegionHandler < Seahorse::Client::Handler
-
           def call(context)
             bucket = context.params[:bucket]
             check_for_cached_region(context, bucket) if bucket
@@ -92,11 +105,12 @@ module Aws
           def check_for_cached_region(context, bucket)
             cached_region = S3::BUCKET_REGIONS[bucket]
             if cached_region && cached_region != context.config.region
-              context.http_request.endpoint.host = S3Signer.new_hostname(context, cached_region)
+              context.http_request.endpoint.host = S3Signer.new_hostname(
+                context, cached_region
+              )
               context[:cached_sigv4_region] = cached_region
             end
           end
-
         end
 
         # This handler detects when a request fails because of a mismatched bucket
@@ -104,7 +118,6 @@ module Aws
         # region, then finally a version 4 signed request against the correct
         # regional endpoint.
         class BucketRegionErrorHandler < Seahorse::Client::Handler
-
           def call(context)
             response = @handler.call(context)
             handle_region_errors(response)
@@ -113,7 +126,9 @@ module Aws
           private
 
           def handle_region_errors(response)
-            if wrong_sigv4_region?(response) && !fips_region?(response)
+            if wrong_sigv4_region?(response) &&
+               !fips_region?(response) &&
+               !custom_endpoint?(response)
               get_region_and_retry(response.context)
             else
               response
@@ -137,21 +152,33 @@ module Aws
             resp.context.http_request.endpoint.host.include?('fips')
           end
 
+          def custom_endpoint?(resp)
+            resolved_suffix = Aws::Partitions::EndpointProvider.dns_suffix_for(
+              resp.context.config.region
+            )
+            !resp.context.http_request.endpoint.hostname.include?(resolved_suffix)
+          end
+
           def wrong_sigv4_region?(resp)
             resp.context.http_response.status_code == 400 &&
-            (
-              resp.context.http_response.headers['x-amz-bucket-region'] ||
-              resp.context.http_response.body_contents.match(/<Region>.+?<\/Region>/)
-            )
+              (resp.context.http_response.headers['x-amz-bucket-region'] ||
+               resp.context.http_response.body_contents.match(/<Region>.+?<\/Region>/))
           end
 
           def resign_with_new_region(context, actual_region)
             context.http_response.body.truncate(0)
-            context.http_request.endpoint.host = S3Signer.new_hostname(context, actual_region)
+            context.http_request.endpoint.host = S3Signer.new_hostname(
+              context, actual_region
+            )
             context.metadata[:redirect_region] = actual_region
+            # if it's an ARN, use the service in the ARN
+            if (arn = context.metadata[:s3_arn])
+              service = arn[:arn].service
+            end
             Aws::Plugins::SignatureV4.apply_signature(
               context: context,
               signer: S3Signer.build_v4_signer(
+                service: service || 's3',
                 region: actual_region,
                 credentials: context.config.credentials
               )
@@ -160,7 +187,7 @@ module Aws
 
           def region_from_body(body)
             region = body.match(/<Region>(.+?)<\/Region>/)[1]
-            if region.nil? || region == ""
+            if region.nil? || region == ''
               raise "couldn't get region from body: #{body}"
             else
               region
@@ -168,44 +195,46 @@ module Aws
           end
 
           def log_warning(context, actual_region)
-            msg = "S3 client configured for #{context.config.region.inspect} " +
-              "but the bucket #{context.params[:bucket].inspect} is in " +
-              "#{actual_region.inspect}; Please configure the proper region " +
-              "to avoid multiple unnecessary redirects and signing attempts\n"
-            if logger = context.config.logger
+            msg = "S3 client configured for #{context.config.region.inspect} " \
+                  "but the bucket #{context.params[:bucket].inspect} is in " \
+                  "#{actual_region.inspect}; Please configure the proper region " \
+                  "to avoid multiple unnecessary redirects and signing attempts\n"
+            if (logger = context.config.logger)
               logger.warn(msg)
             else
               warn(msg)
             end
           end
-
         end
 
         class << self
-
           # @option options [required, String] :region
           # @option options [required, #credentials] :credentials
           # @api private
           def build_v4_signer(options = {})
-            Aws::Sigv4::Signer.new({
-              service: 's3',
+            Aws::Sigv4::Signer.new(
+              service: options[:service],
               region: options[:region],
               credentials_provider: options[:credentials],
               uri_escape_path: false,
-              unsigned_headers: ['content-length', 'x-amzn-trace-id'],
-            })
+              unsigned_headers: ['content-length', 'x-amzn-trace-id']
+            )
           end
 
+          # Check to see if the bucket is actually an ARN
+          # Otherwise it will retry with the ARN as the bucket name.
           def new_hostname(context, region)
-            bucket = context.params[:bucket]
-            if region == 'us-east-1'
-              "#{bucket}.s3.amazonaws.com"
+            uri = URI.parse(
+              Aws::Partitions::EndpointProvider.resolve(region, 's3')
+            )
+
+            if (arn = context.metadata[:s3_arn])
+              # Retry with the response region and not the ARN resolved one
+              ARN.resolve_url!(uri, arn[:arn], region).host
             else
-              endpoint = Aws::Partitions::EndpointProvider.resolve(region, 's3')
-              bucket + '.' + URI.parse(endpoint).host
+              "#{context.params[:bucket]}.#{uri.host}"
             end
           end
-
         end
       end
     end

data/lib/aws-sdk-s3/plugins/sse_cpk.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 require 'openssl'
 
@@ -18,7 +20,7 @@ This should only be disabled for local testing.
         class Handler < Seahorse::Client::Handler
 
           def call(context)
-            compute_key_md5(context)
+            compute_key_md5(context) if context.params.is_a?(Hash)
             @handler.call(context)
           end
 

data/lib/aws-sdk-s3/plugins/streaming_retry.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Aws
+  module S3
+    module Plugins
+
+      # A wrapper around BlockIO that adds no-ops for truncate and rewind
+      # @api private
+      class RetryableBlockIO
+        extend Forwardable
+        def_delegators :@block_io, :write, :read, :size
+
+        def initialize(block_io)
+          @block_io = block_io
+        end
+
+        def truncate(_integer); end
+
+        def rewind; end
+      end
+
+      # A wrapper around ManagedFile that adds no-ops for truncate and rewind
+      # @api private
+      class RetryableManagedFile
+        extend Forwardable
+        def_delegators :@file, :write, :read, :size, :open?, :close
+
+        def initialize(managed_file)
+          @file = managed_file
+        end
+
+        def truncate(_integer); end
+
+        def rewind; end
+      end
+
+      # This handler works with the ResponseTarget plugin to provide smart
+      # retries of S3 streaming operations that support the range parameter
+      # (currently only: get_object).  When a 200 OK with a TruncatedBodyError
+      # is received this handler will add a range header that excludes the
+      # data that has already been processed (written to file or sent to
+      # the target Proc).
+      # It is important to not write data to the custom target in the case of
+      # a non-success response. We do not want to write an XML error
+      # message to someone's file or pass it to a user's Proc.
+      # @api private
+      class StreamingRetry < Seahorse::Client::Plugin
+
+        class Handler < Seahorse::Client::Handler
+
+          def call(context)
+            target = context.params[:response_target] || context[:response_target]
+
+            # retry is only supported when range is NOT set on the initial request
+            if supported_target?(target) && !context.params[:range]
+              add_event_listeners(context, target)
+            end
+            @handler.call(context)
+          end
+
+          private
+
+          def add_event_listeners(context, target)
+            context.http_response.on_headers(200..299) do
+              case context.http_response.body
+              when Seahorse::Client::BlockIO then
+                context.http_response.body = RetryableBlockIO.new(context.http_response.body)
+              when Seahorse::Client::ManagedFile then
+                context.http_response.body = RetryableManagedFile.new(context.http_response.body)
+              end
+            end
+
+            context.http_response.on_headers(400..599) do
+              context.http_response.body = StringIO.new # something to write the error to
+            end
+
+            context.http_response.on_success(200..299) do
+              body = context.http_response.body
+              if body.is_a?(RetryableManagedFile) && body.open?
+                body.close
+              end
+            end
+
+            context.http_response.on_error do |error|
+              if retryable_body?(context) && truncated_body?(error)
+                context.http_request.headers[:range] = "bytes=#{context.http_response.body.size}-"
+              end
+            end
+          end
+
+          def truncated_body?(error)
+            error.is_a?(Seahorse::Client::NetworkingError) &&
+              error.original_error.is_a?(
+                Seahorse::Client::NetHttp::Handler::TruncatedBodyError
+              )
+          end
+
+          def retryable_body?(context)
+            context.http_response.body.is_a?(RetryableBlockIO) ||
+              context.http_response.body.is_a?(RetryableManagedFile)
+          end
+
+          def supported_target?(target)
+            case target
+            when Proc, String, Pathname then true
+            else false
+            end
+          end
+        end
+
+        handler(Handler, step: :sign, operations: [:get_object], priority: 10)
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/url_encoded_keys.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 require 'cgi'
 

data/lib/aws-sdk-s3/presigned_post.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'openssl'
 require 'base64'
 
@@ -183,35 +185,59 @@ module Aws
       #   the post will expire. Defaults to one hour from creation of the
       #   presigned post. May not exceed one week from creation time.
       # @option options [String] :key See {PresignedPost#key}.
-      # @option options [String] :key_starts_with See {PresignedPost#key_starts_with}.
+      # @option options [String] :key_starts_with
+      #   See {PresignedPost#key_starts_with}.
       # @option options [String] :acl See {PresignedPost#acl}.
-      # @option options [String] :acl_starts_with See {PresignedPost#acl_starts_with}.
-      # @option options [String] :cache_control See {PresignedPost#cache_control}.
-      # @option options [String] :cache_control_starts_with See {PresignedPost#cache_control_starts_with}.
+      # @option options [String] :acl_starts_with
+      #   See {PresignedPost#acl_starts_with}.
+      # @option options [String] :cache_control
+      #   See {PresignedPost#cache_control}.
+      # @option options [String] :cache_control_starts_with
+      #   See {PresignedPost#cache_control_starts_with}.
       # @option options [String] :content_type See {PresignedPost#content_type}.
-      # @option options [String] :content_type_starts_with See {PresignedPost#content_type_starts_with}.
-      # @option options [String] :content_disposition See {PresignedPost#content_disposition}.
-      # @option options [String] :content_disposition_starts_with See {PresignedPost#content_disposition_starts_with}.
-      # @option options [String] :content_encoding See {PresignedPost#content_encoding}.
-      # @option options [String] :content_encoding_starts_with See {PresignedPost#content_encoding_starts_with}.
+      # @option options [String] :content_type_starts_with
+      #   See {PresignedPost#content_type_starts_with}.
+      # @option options [String] :content_disposition
+      #   See {PresignedPost#content_disposition}.
+      # @option options [String] :content_disposition_starts_with
+      #   See {PresignedPost#content_disposition_starts_with}.
+      # @option options [String] :content_encoding
+      #   See {PresignedPost#content_encoding}.
+      # @option options [String] :content_encoding_starts_with
+      #   See {PresignedPost#content_encoding_starts_with}.
       # @option options [String] :expires See {PresignedPost#expires}.
-      # @option options [String] :expires_starts_with See {PresignedPost#expires_starts_with}.
-      # @option options [Range<Integer>] :content_length_range See {PresignedPost#content_length_range}.
-      # @option options [String] :success_action_redirect See {PresignedPost#success_action_redirect}.
-      # @option options [String] :success_action_redirect_starts_with See {PresignedPost#success_action_redirect_starts_with}.
-      # @option options [String] :success_action_status See {PresignedPost#success_action_status}.
-      # @option options [String] :storage_class See {PresignedPost#storage_class}.
-      # @option options [String] :website_redirect_location See {PresignedPost#website_redirect_location}.
-      # @option options [Hash<String,String>] :metadata See {PresignedPost#metadata}.
-      # @option options [Hash<String,String>] :metadata_starts_with See {PresignedPost#metadata_starts_with}.
-      # @option options [String] :server_side_encryption See {PresignedPost#server_side_encryption}.
-      # @option options [String] :server_side_encryption_aws_kms_key_id See {PresignedPost#server_side_encryption_aws_kms_key_id}.
-      # @option options [String] :server_side_encryption_customer_algorithm See {PresignedPost#server_side_encryption_customer_algorithm}.
-      # @option options [String] :server_side_encryption_customer_key See {PresignedPost#server_side_encryption_customer_key}.
+      # @option options [String] :expires_starts_with
+      #   See {PresignedPost#expires_starts_with}.
+      # @option options [Range<Integer>] :content_length_range
+      #   See {PresignedPost#content_length_range}.
+      # @option options [String] :success_action_redirect
+      #   See {PresignedPost#success_action_redirect}.
+      # @option options [String] :success_action_redirect_starts_with
+      #   See {PresignedPost#success_action_redirect_starts_with}.
+      # @option options [String] :success_action_status
+      #   See {PresignedPost#success_action_status}.
+      # @option options [String] :storage_class
+      #   See {PresignedPost#storage_class}.
+      # @option options [String] :website_redirect_location
+      #   See {PresignedPost#website_redirect_location}.
+      # @option options [Hash<String,String>] :metadata
+      #   See {PresignedPost#metadata}.
+      # @option options [Hash<String,String>] :metadata_starts_with
+      #   See {PresignedPost#metadata_starts_with}.
+      # @option options [String] :server_side_encryption
+      #   See {PresignedPost#server_side_encryption}.
+      # @option options [String] :server_side_encryption_aws_kms_key_id
+      #   See {PresignedPost#server_side_encryption_aws_kms_key_id}.
+      # @option options [String] :server_side_encryption_customer_algorithm
+      #   See {PresignedPost#server_side_encryption_customer_algorithm}.
+      # @option options [String] :server_side_encryption_customer_key
+      #   See {PresignedPost#server_side_encryption_customer_key}.
       def initialize(credentials, bucket_region, bucket_name, options = {})
         @credentials = credentials.credentials
         @bucket_region = bucket_region
         @bucket_name = bucket_name
+        @accelerate = !!options.delete(:use_accelerate_endpoint)
+        options.delete(:url) if @accelerate # resource methods pass url
         @url = options.delete(:url) || bucket_url
         @fields = {}
         @key_set = false
@@ -234,7 +260,7 @@ module Aws
       #   as hidden input fields.
       def fields
         check_required_values!
-        datetime = Time.now.utc.strftime("%Y%m%dT%H%M%SZ")
+        datetime = Time.now.utc.strftime('%Y%m%dT%H%M%SZ')
         fields = @fields.dup
         fields.update('policy' => policy(datetime))
         fields.update(signature_fields(datetime))
@@ -270,7 +296,7 @@ module Aws
 
       # @!group Fields
 
-      # The key to use for the uploaded object. Use can use `${filename}`
+      # The key to use for the uploaded object. You can use `${filename}`
       # as a variable in the key. This will be replaced with the name
       # of the file as provided by the user.
       #
@@ -505,7 +531,10 @@ module Aws
       #   (KMS) master encryption key to use for the object.
       #   @param [String] value
       #   @return [self]
-      define_field(:server_side_encryption_aws_kms_key_id, 'x-amz-server-side-encryption-aws-kms-key-id')
+      define_field(
+        :server_side_encryption_aws_kms_key_id,
+        'x-amz-server-side-encryption-aws-kms-key-id'
+      )
 
       # @!endgroup
 
@@ -518,7 +547,10 @@ module Aws
       #   @param [String] value
       #   @see #server_side_encryption_customer_key
       #   @return [self]
-      define_field(:server_side_encryption_customer_algorithm, 'x-amz-server-side-encryption-customer-algorithm')
+      define_field(
+        :server_side_encryption_customer_algorithm,
+        'x-amz-server-side-encryption-customer-algorithm'
+      )
 
       # Specifies the customer-provided encryption key for Amazon S3 to use
       # in encrypting data. This value is used to store the object and then
@@ -571,8 +603,8 @@ module Aws
 
       def check_required_values!
         unless @key_set
-          msg = "key required; you must provide a key via :key, "
-          msg << ":key_starts_with, or :allow_any => ['key']"
+          msg = 'key required; you must provide a key via :key, '\
+                ":key_starts_with, or :allow_any => ['key']"
           raise msg
         end
       end
@@ -580,10 +612,18 @@ module Aws
       def bucket_url
         url = Aws::Partitions::EndpointProvider.resolve(@bucket_region, 's3')
         url = URI.parse(url)
-        if Plugins::BucketDns.dns_compatible?(@bucket_name, true)
-          url.host = @bucket_name + '.' + url.host
+        if Plugins::BucketDns.dns_compatible?(@bucket_name, _ssl = true)
+          if @accelerate
+            url.host = "#{@bucket_name}.s3-accelerate.amazonaws.com"
+          else
+            url.host = "#{@bucket_name}.#{url.host}"
+          end
         else
-          url.path = '/' + @bucket_name
+          url.path = "/#{@bucket_name}"
+        end
+        if @bucket_region == 'us-east-1'
+          # keep legacy behavior by default
+          url.host = Plugins::IADRegionalEndpoint.legacy_host(url.host)
         end
         url.to_s
       end
@@ -613,7 +653,7 @@ module Aws
 
       def signature(datetime, string_to_sign)
         k_secret = @credentials.secret_access_key
-        k_date = hmac("AWS4" + k_secret, datetime[0,8])
+        k_date = hmac('AWS4' + k_secret, datetime[0,8])
         k_region = hmac(k_date, @bucket_region)
         k_service = hmac(k_region, 's3')
         k_credentials = hmac(k_service, 'aws4_request')