aws-sdk-s3 1.21.0 → 1.117.2

data/lib/aws-sdk-s3/plugins/s3_signer.rb

@@ -1,25 +1,17 @@
+# frozen_string_literal: true
+
 require 'aws-sigv4'
 
 module Aws
   module S3
     module Plugins
-      # This plugin is an implementation detail and may be modified.
+      # This plugin used to have a V4 signer but it was removed in favor of
+      # generic Sign plugin that uses endpoint auth scheme.
+      #
       # @api private
       class S3Signer < Seahorse::Client::Plugin
-
         option(:signature_version, 'v4')
 
-        option(:sigv4_signer) do |cfg|
-          S3Signer.build_v4_signer(
-            region: cfg.sigv4_region,
-            credentials: cfg.credentials
-          )
-        end
-
-        option(:sigv4_region) do |cfg|
-          Aws::Partitions::EndpointProvider.signing_region(cfg.region, 's3')
-        end
-
         def add_handlers(handlers, cfg)
           case cfg.signature_version
           when 'v4' then add_v4_handlers(handlers)
@@ -32,11 +24,11 @@ module Aws
 
         def add_v4_handlers(handlers)
           handlers.add(CachedBucketRegionHandler, step: :sign, priority: 60)
-          handlers.add(V4Handler, step: :sign)
           handlers.add(BucketRegionErrorHandler, step: :sign, priority: 40)
         end
 
         def add_legacy_handler(handlers)
+          # generic Sign plugin will be skipped if it sees sigv2
           handlers.add(LegacyHandler, step: :sign)
         end
 
@@ -47,40 +39,9 @@ module Aws
           end
         end
 
-        class V4Handler < Seahorse::Client::Handler
-
-          def call(context)
-            Aws::Plugins::SignatureV4.apply_signature(
-              context: context,
-              signer: sigv4_signer(context)
-            )
-            @handler.call(context)
-          end
-
-          private
-
-          def sigv4_signer(context)
-            # If the client was configured with the wrong region,
-            # we have to build a new signer.
-            if
-              context[:cached_sigv4_region] &&
-              context[:cached_sigv4_region] != context.config.sigv4_signer.region
-            then
-              S3Signer.build_v4_signer(
-                region: context[:cached_sigv4_region],
-                credentials: context.config.credentials
-              )
-            else
-              context.config.sigv4_signer
-            end
-          end
-
-        end
-
         # This handler will update the http endpoint when the bucket region
         # is known/cached.
         class CachedBucketRegionHandler < Seahorse::Client::Handler
-
           def call(context)
             bucket = context.params[:bucket]
             check_for_cached_region(context, bucket) if bucket
@@ -92,19 +53,20 @@ module Aws
           def check_for_cached_region(context, bucket)
             cached_region = S3::BUCKET_REGIONS[bucket]
             if cached_region && cached_region != context.config.region
-              context.http_request.endpoint.host = S3Signer.new_hostname(context, cached_region)
-              context[:cached_sigv4_region] = cached_region
+              context.http_request.endpoint.host = S3Signer.new_hostname(
+                context, cached_region
+              )
+              context[:sigv4_region] = cached_region # Sign plugin will use this
             end
           end
-
         end
 
         # This handler detects when a request fails because of a mismatched bucket
         # region. It follows up by making a request to determine the correct
         # region, then finally a version 4 signed request against the correct
-        # regional endpoint.
+        # regional endpoint. This is intended for s3's global endpoint which
+        # will return 400 if the bucket is not in region.
         class BucketRegionErrorHandler < Seahorse::Client::Handler
-
           def call(context)
             response = @handler.call(context)
             handle_region_errors(response)
@@ -113,7 +75,10 @@ module Aws
           private
 
           def handle_region_errors(response)
-            if wrong_sigv4_region?(response) && !fips_region?(response)
+            if wrong_sigv4_region?(response) &&
+               !fips_region?(response) &&
+               !custom_endpoint?(response) &&
+               !expired_credentials?(response)
               get_region_and_retry(response.context)
             else
               response
@@ -134,32 +99,47 @@ module Aws
           end
 
           def fips_region?(resp)
-            resp.context.http_request.endpoint.host.include?('fips')
+            resp.context.http_request.endpoint.host.include?('s3-fips.')
+          end
+
+          def expired_credentials?(resp)
+            resp.context.http_response.body_contents.match(/<Code>ExpiredToken<\/Code>/)
+          end
+
+          def custom_endpoint?(resp)
+            region = resp.context.config.region
+            partition = Aws::Endpoints::Matchers.aws_partition(region)
+            endpoint = resp.context.http_request.endpoint
+
+            !endpoint.hostname.include?(partition['dnsSuffix']) &&
+              !endpoint.hostname.include?(partition['dualStackDnsSuffix'])
           end
 
           def wrong_sigv4_region?(resp)
             resp.context.http_response.status_code == 400 &&
-            (
-              resp.context.http_response.headers['x-amz-bucket-region'] ||
-              resp.context.http_response.body_contents.match(/<Region>.+?<\/Region>/)
-            )
+              (resp.context.http_response.headers['x-amz-bucket-region'] ||
+               resp.context.http_response.body_contents.match(/<Region>.+?<\/Region>/))
           end
 
           def resign_with_new_region(context, actual_region)
             context.http_response.body.truncate(0)
-            context.http_request.endpoint.host = S3Signer.new_hostname(context, actual_region)
-            Aws::Plugins::SignatureV4.apply_signature(
-              context: context,
-              signer: S3Signer.build_v4_signer(
-                region: actual_region,
-                credentials: context.config.credentials
-              )
+            context.http_request.endpoint.host = S3Signer.new_hostname(
+              context, actual_region
             )
+            context.metadata[:redirect_region] = actual_region
+
+            signer = Aws::Plugins::Sign.signer_for(
+              context[:auth_scheme],
+              context.config,
+              actual_region
+            )
+
+            signer.sign(context)
           end
 
           def region_from_body(body)
             region = body.match(/<Region>(.+?)<\/Region>/)[1]
-            if region.nil? || region == ""
+            if region.nil? || region == ''
               raise "couldn't get region from body: #{body}"
             else
               region
@@ -167,44 +147,27 @@ module Aws
           end
 
           def log_warning(context, actual_region)
-            msg = "S3 client configured for #{context.config.region.inspect} " +
-              "but the bucket #{context.params[:bucket].inspect} is in " +
-              "#{actual_region.inspect}; Please configure the proper region " +
-              "to avoid multiple unnecessary redirects and signing attempts\n"
-            if logger = context.config.logger
+            msg = "S3 client configured for #{context.config.region.inspect} " \
+                  "but the bucket #{context.params[:bucket].inspect} is in " \
+                  "#{actual_region.inspect}; Please configure the proper region " \
+                  "to avoid multiple unnecessary redirects and signing attempts\n"
+            if (logger = context.config.logger)
               logger.warn(msg)
             else
               warn(msg)
             end
           end
-
         end
 
         class << self
-
-          # @option options [required, String] :region
-          # @option options [required, #credentials] :credentials
-          # @api private
-          def build_v4_signer(options = {})
-            Aws::Sigv4::Signer.new({
-              service: 's3',
-              region: options[:region],
-              credentials_provider: options[:credentials],
-              uri_escape_path: false,
-              unsigned_headers: ['content-length', 'x-amzn-trace-id'],
-            })
-          end
-
           def new_hostname(context, region)
-            bucket = context.params[:bucket]
-            if region == 'us-east-1'
-              "#{bucket}.s3.amazonaws.com"
-            else
-              endpoint = Aws::Partitions::EndpointProvider.resolve(region, 's3')
-              bucket + '.' + URI.parse(endpoint).host
-            end
+            endpoint_params = context[:endpoint_params].dup
+            endpoint_params.region = region
+            endpoint_params.endpoint = nil
+            endpoint =
+              context.config.endpoint_provider.resolve_endpoint(endpoint_params)
+            URI(endpoint.url).host
           end
-
         end
       end
     end

data/lib/aws-sdk-s3/plugins/skip_whole_multipart_get_checksums.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Plugins
+
+      # S3 GetObject results for whole Multipart Objects contain a checksum
+      # that cannot be validated.  These should be skipped by the
+      # ChecksumAlgorithm plugin.
+      class SkipWholeMultipartGetChecksums < Seahorse::Client::Plugin
+
+        class Handler < Seahorse::Client::Handler
+
+          def call(context)
+            context[:http_checksum] ||= {}
+            context[:http_checksum][:skip_on_suffix] = true
+
+            @handler.call(context)
+          end
+
+        end
+
+        handler(
+          Handler,
+          step: :initialize,
+          operations: [:get_object]
+        )
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/sse_cpk.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 require 'openssl'
 
@@ -18,7 +20,7 @@ This should only be disabled for local testing.
         class Handler < Seahorse::Client::Handler
 
           def call(context)
-            compute_key_md5(context)
+            compute_key_md5(context) if context.params.is_a?(Hash)
             @handler.call(context)
           end
 

data/lib/aws-sdk-s3/plugins/streaming_retry.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Aws
+  module S3
+    module Plugins
+
+      # A wrapper around BlockIO that adds no-ops for truncate and rewind
+      # @api private
+      class RetryableBlockIO
+        extend Forwardable
+        def_delegators :@block_io, :write, :read, :size
+
+        def initialize(block_io)
+          @block_io = block_io
+        end
+
+        def truncate(_integer); end
+
+        def rewind; end
+      end
+
+      # A wrapper around ManagedFile that adds no-ops for truncate and rewind
+      # @api private
+      class RetryableManagedFile
+        extend Forwardable
+        def_delegators :@file, :write, :read, :size, :open?, :close
+
+        def initialize(managed_file)
+          @file = managed_file
+        end
+
+        def truncate(_integer); end
+
+        def rewind; end
+      end
+
+      class NonRetryableStreamingError < StandardError
+
+        def initialize(error)
+          super('Unable to retry request - retry could result in processing duplicated chunks.')
+          set_backtrace(error.backtrace)
+          @original_error = error
+        end
+
+        attr_reader :original_error
+      end
+
+      # This handler works with the ResponseTarget plugin to provide smart
+      # retries of S3 streaming operations that support the range parameter
+      # (currently only: get_object).  When a 200 OK with a TruncatedBodyError
+      # is received this handler will add a range header that excludes the
+      # data that has already been processed (written to file or sent to
+      # the target Proc).
+      # It is important to not write data to the custom target in the case of
+      # a non-success response. We do not want to write an XML error
+      # message to someone's file or pass it to a user's Proc.
+      # @api private
+      class StreamingRetry < Seahorse::Client::Plugin
+
+        class Handler < Seahorse::Client::Handler
+
+          def call(context)
+            target = context.params[:response_target] || context[:response_target]
+
+            # retry is only supported when range is NOT set on the initial request
+            if supported_target?(target) && !context.params[:range]
+              add_event_listeners(context, target)
+            end
+            @handler.call(context)
+          end
+
+          private
+
+          def add_event_listeners(context, target)
+            context.http_response.on_headers(200..299) do
+              case context.http_response.body
+              when Seahorse::Client::BlockIO then
+                context.http_response.body = RetryableBlockIO.new(context.http_response.body)
+              when Seahorse::Client::ManagedFile then
+                context.http_response.body = RetryableManagedFile.new(context.http_response.body)
+              end
+            end
+
+            context.http_response.on_headers(400..599) do
+              context.http_response.body = StringIO.new # something to write the error to
+            end
+
+            context.http_response.on_success(200..299) do
+              body = context.http_response.body
+              if body.is_a?(RetryableManagedFile) && body.open?
+                body.close
+              end
+            end
+
+            context.http_response.on_error do |error|
+              if retryable_body?(context)
+                if truncated_body?(error)
+                  context.http_request.headers[:range] = "bytes=#{context.http_response.body.size}-"
+                else
+                  case context.http_response.body
+                  when RetryableManagedFile
+                    # call rewind on the underlying file
+                    context.http_response.body.instance_variable_get(:@file).rewind
+                  else
+                    raise NonRetryableStreamingError, error
+                  end
+                end
+              end
+            end
+          end
+
+          def truncated_body?(error)
+            error.is_a?(Seahorse::Client::NetworkingError) &&
+              error.original_error.is_a?(
+                Seahorse::Client::NetHttp::Handler::TruncatedBodyError
+              )
+          end
+
+          def retryable_body?(context)
+            context.http_response.body.is_a?(RetryableBlockIO) ||
+              context.http_response.body.is_a?(RetryableManagedFile)
+          end
+
+          def supported_target?(target)
+            case target
+            when Proc, String, Pathname then true
+            else false
+            end
+          end
+        end
+
+        handler(Handler, step: :sign, operations: [:get_object], priority: 10)
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/url_encoded_keys.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 require 'cgi'