aws-sdk-s3 1.114.0 → 1.119.1

data/lib/aws-sdk-s3/multipart_stream_uploader.rb

@@ -26,6 +26,10 @@ module Aws
       UPLOAD_PART_OPTIONS =
         Set.new(Client.api.operation(:upload_part).input.shape.member_names)
 
+      # @api private
+      COMPLETE_UPLOAD_OPTIONS =
+        Set.new(Client.api.operation(:complete_multipart_upload).input.shape.member_names)
+
       # @option options [Client] :client
       def initialize(options = {})
         @client = options[:client] || Client.new
@@ -39,7 +43,7 @@ module Aws
 
       # @option options [required,String] :bucket
       # @option options [required,String] :key
-      # @return [void]
+      # @return [Seahorse::Client::Response] - the CompleteMultipartUploadResponse
       def upload(options = {}, &block)
         upload_id = initiate_upload(options)
         parts = upload_parts(upload_id, options, &block)
@@ -54,17 +58,22 @@ module Aws
 
       def complete_upload(upload_id, parts, options)
         @client.complete_multipart_upload(
-          bucket: options[:bucket],
-          key: options[:key],
-          upload_id: upload_id,
-          multipart_upload: { parts: parts })
+          **complete_opts(options).merge(
+            upload_id: upload_id,
+            multipart_upload: { parts: parts }
+          )
+        )
       end
 
       def upload_parts(upload_id, options, &block)
         completed = Queue.new
+        thread_errors = []
         errors = begin
           IO.pipe do |read_pipe, write_pipe|
-            threads = upload_in_threads(read_pipe, completed, upload_part_opts(options).merge(upload_id: upload_id))
+            threads = upload_in_threads(
+              read_pipe, completed,
+              upload_part_opts(options).merge(upload_id: upload_id),
+              thread_errors)
             begin
               block.call(write_pipe)
             ensure
@@ -74,7 +83,7 @@ module Aws
             threads.map(&:value).compact
           end
         rescue => e
-          [e]
+          thread_errors + [e]
         end
 
         if errors.empty?
@@ -113,6 +122,13 @@ module Aws
         end
       end
 
+      def complete_opts(options)
+        COMPLETE_UPLOAD_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
       def read_to_part_body(read_pipe)
         return if read_pipe.closed?
         temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new(String.new)
@@ -130,7 +146,7 @@ module Aws
         end
       end
 
-      def upload_in_threads(read_pipe, completed, options)
+      def upload_in_threads(read_pipe, completed, options, thread_errors)
         mutex = Mutex.new
         part_number = 0
         @thread_count.times.map do
@@ -147,7 +163,14 @@ module Aws
                     part_number: thread_part_number,
                   )
                   resp = @client.upload_part(part)
-                  completed << {etag: resp.etag, part_number: part[:part_number]}
+                  completed_part = {etag: resp.etag, part_number: part[:part_number]}
+
+                  # get the requested checksum from the response
+                  if part[:checksum_algorithm]
+                    k = "checksum_#{part[:checksum_algorithm].downcase}".to_sym
+                    completed_part[k] = resp[k]
+                  end
+                  completed.push(completed_part)
                 ensure
                   if Tempfile === body
                     body.close
@@ -160,7 +183,10 @@ module Aws
               nil
             rescue => error
               # keep other threads from uploading other parts
-              mutex.synchronize { read_pipe.close_read unless read_pipe.closed? }
+              mutex.synchronize do
+                thread_errors.push(error)
+                read_pipe.close_read unless read_pipe.closed?
+              end
               error
             end
           end

data/lib/aws-sdk-s3/object_multipart_copier.rb

@@ -24,6 +24,9 @@ module Aws
         @thread_count = options.delete(:thread_count) || 10
         @min_part_size = options.delete(:min_part_size) || (FIVE_MB * 10)
         @client = options[:client] || Client.new
+        if options[:checksum_algorithm]
+          raise ArgumentError, 'Multipart Copy does not support setting :checksum_algorithm'
+        end
       end
 
       # @return [Client]
@@ -31,8 +34,9 @@ module Aws
 
       # @option (see S3::Client#copy_object)
       def copy(options = {})
-        size = source_size(options)
-        options[:upload_id] = initiate_upload(options)
+        metadata = source_metadata(options)
+        size = metadata[:content_length]
+        options[:upload_id] = initiate_upload(metadata.merge(options))
         begin
           parts = copy_parts(size, default_part_size(size), options)
           complete_upload(parts, options)
@@ -118,8 +122,10 @@ module Aws
         end
       end
 
-      def source_size(options)
-        return options.delete(:content_length) if options[:content_length]
+      def source_metadata(options)
+        if options[:content_length]
+          return { content_length: options.delete(:content_length) }
+        end
 
         client = options[:copy_source_client] || @client
 
@@ -132,7 +138,7 @@ module Aws
         key = CGI.unescape(key)
         opts = { bucket: bucket, key: key }
         opts[:version_id] = version_id if version_id
-        client.head_object(opts).content_length
+        client.head_object(opts).to_h
       end
 
       def default_part_size(source_size)

data/lib/aws-sdk-s3/plugins/accelerate.rb

@@ -23,73 +23,26 @@ each bucket. [Go here for more information](http://docs.aws.amazon.com/AmazonS3/
           operations = config.api.operation_names - [
             :create_bucket, :list_buckets, :delete_bucket
           ]
-          # Need 2 handlers so that the context can be set for other plugins
-          # and to remove :use_accelerate_endpoint from the params.
           handlers.add(
             OptionHandler, step: :initialize, operations: operations
           )
-          handlers.add(
-            AccelerateHandler, step: :build, priority: 11, operations: operations
-          )
         end
 
         # @api private
         class OptionHandler < Seahorse::Client::Handler
           def call(context)
             # Support client configuration and per-operation configuration
+            # TODO: move this to an options hash and warn here.
             if context.params.is_a?(Hash)
               accelerate = context.params.delete(:use_accelerate_endpoint)
             end
-            accelerate = context.config.use_accelerate_endpoint if accelerate.nil?
-            # Raise if :endpoint and accelerate are both provided
-            if accelerate && !context.config.regional_endpoint
-              raise ArgumentError,
-                    'Cannot use both :use_accelerate_endpoint and :endpoint'
-            end
-            # Raise if :use_fips_endpoint and accelerate are both provided
-            if accelerate && context.config.use_fips_endpoint
-              raise ArgumentError,
-                    'Cannot use both :use_accelerate_endpoint and '\
-                    ':use_fips_endpoint'
+            if accelerate.nil?
+              accelerate = context.config.use_accelerate_endpoint
             end
             context[:use_accelerate_endpoint] = accelerate
             @handler.call(context)
           end
         end
-
-        # @api private
-        class AccelerateHandler < Seahorse::Client::Handler
-          def call(context)
-            if context.config.regional_endpoint && context[:use_accelerate_endpoint]
-              dualstack = !!context[:use_dualstack_endpoint]
-              use_accelerate_endpoint(context, dualstack)
-            end
-            @handler.call(context)
-          end
-
-          private
-
-          def use_accelerate_endpoint(context, dualstack)
-            bucket_name = context.params[:bucket]
-            validate_bucket_name!(bucket_name)
-            endpoint = URI.parse(context.http_request.endpoint.to_s)
-            endpoint.scheme = 'https'
-            endpoint.port = 443
-            endpoint.host = "#{bucket_name}.s3-accelerate"\
-                            "#{'.dualstack' if dualstack}.amazonaws.com"
-            context.http_request.endpoint = endpoint.to_s
-            # s3 accelerate endpoint doesn't work with 'expect' header
-            context.http_request.headers.delete('expect')
-          end
-
-          def validate_bucket_name!(bucket_name)
-            unless BucketDns.dns_compatible?(bucket_name, _ssl = true)
-              raise ArgumentError,
-                    'Unable to use `use_accelerate_endpoint: true` on buckets '\
-                    'with non-DNS compatible names.'
-            end
-          end
-        end
       end
     end
   end

data/lib/aws-sdk-s3/plugins/arn.rb

@@ -1,10 +1,5 @@
 # frozen_string_literal: true
 
-require_relative '../arn/access_point_arn'
-require_relative '../arn/object_lambda_arn'
-require_relative '../arn/outpost_access_point_arn'
-require_relative '../arn/multi_region_access_point_arn'
-
 module Aws
   module S3
     module Plugins
@@ -36,150 +31,9 @@ result in cross region requests.
           resolve_s3_disable_multiregion_access_points(cfg)
         end
 
-        # param validator is validate:50
-        # endpoint is build:90 (populates the URI for the first time)
-        # endpoint pattern is build:10
-        def add_handlers(handlers, _config)
-          handlers.add(ARNHandler, step: :validate, priority: 75)
-          handlers.add(UrlHandler)
-        end
-
-        # After extracting out any ARN input, resolve a new URL with it.
-        class UrlHandler < Seahorse::Client::Handler
-          def call(context)
-            if context.metadata[:s3_arn]
-              ARN.resolve_url!(
-                context.http_request.endpoint,
-                context.metadata[:s3_arn][:arn],
-                context.metadata[:s3_arn][:resolved_region],
-                context.metadata[:s3_arn][:fips],
-                context.metadata[:s3_arn][:dualstack],
-                # if regional_endpoint is false, a custom endpoint was provided
-                # in this case, we want to prefix the endpoint using the ARN
-                !context.config.regional_endpoint
-              )
-            end
-            @handler.call(context)
-          end
-        end
-
-        # This plugin will extract out any ARN input and set context for other
-        # plugins to use without having to translate the ARN again.
-        class ARNHandler < Seahorse::Client::Handler
-          def call(context)
-            bucket_member = _bucket_member(context.operation.input.shape)
-            if bucket_member && (bucket = context.params[bucket_member])
-              resolved_region, arn = ARN.resolve_arn!(
-                bucket,
-                context.config.region,
-                context.config.s3_use_arn_region
-              )
-              if arn
-                validate_config!(context, arn)
-
-                context.metadata[:s3_arn] = {
-                  arn: arn,
-                  resolved_region: resolved_region,
-                  fips: context.config.use_fips_endpoint,
-                  dualstack: extract_dualstack_config!(context)
-                }
-              end
-            end
-            @handler.call(context)
-          end
-
-          private
-
-          def _bucket_member(input)
-            input.members.each do |member, ref|
-              return member if ref.shape.name == 'BucketName'
-            end
-            nil
-          end
-
-          # other plugins use dualstack so disable it when we're done
-          def extract_dualstack_config!(context)
-            dualstack = context[:use_dualstack_endpoint]
-            context[:use_dualstack_endpoint] = false if dualstack
-            dualstack
-          end
-
-          def validate_config!(context, arn)
-            if context.config.force_path_style
-              raise ArgumentError,
-                    'Cannot provide an Access Point ARN when '\
-                    '`:force_path_style` is set to true.'
-            end
-
-            if context.config.use_accelerate_endpoint
-              raise ArgumentError,
-                    'Cannot provide an Access Point ARN when '\
-                    '`:use_accelerate_endpoint` is set to true.'
-            end
-
-            if !arn.support_dualstack? && context[:use_dualstack_endpoint]
-              raise ArgumentError,
-                    'Cannot provide an Outpost Access Point, Object Lambda, '\
-                    'or Multi-region Access Point ARN'\
-                    ' when `:use_dualstack_endpoint` is set to true.'
-            end
-
-            if arn.region.empty? && context.config.s3_disable_multiregion_access_points
-              raise ArgumentError,
-                    'Cannot provide a Multi-region Access Point ARN with '\
-                    '`:s3_disable_multiregion_access_points` set to true'
-            end
-
-            if context.config.use_fips_endpoint && !arn.support_fips?
-              raise ArgumentError,
-                    'FIPS client regions are not supported for this type '\
-                    'of ARN.'
-            end
-          end
-        end
-
         class << self
-          # @api private
-          def resolve_arn!(member_value, region, use_arn_region)
-            if Aws::ARNParser.arn?(member_value)
-              arn = Aws::ARNParser.parse(member_value)
-              s3_arn = resolve_arn_type!(arn)
-              s3_arn.validate_arn!
-              validate_region_config!(s3_arn, region, use_arn_region)
-              region = s3_arn.region if use_arn_region
-              [region, s3_arn]
-            else
-              [region]
-            end
-          end
-
-          # @api private
-          def resolve_url!(url, arn, region, fips = false, dualstack = false, has_custom_endpoint = false)
-            custom_endpoint = url.host if has_custom_endpoint
-            url.host = arn.host_url(region, fips, dualstack, custom_endpoint)
-            url.path = url_path(url.path, arn)
-            url
-          end
-
           private
 
-          def resolve_arn_type!(arn)
-            case arn.service
-            when 's3'
-              arn.region.empty? ?
-                Aws::S3::MultiRegionAccessPointARN.new(arn.to_h) :
-                Aws::S3::AccessPointARN.new(arn.to_h)
-            when 's3-outposts'
-              Aws::S3::OutpostAccessPointARN.new(arn.to_h)
-            when 's3-object-lambda'
-              Aws::S3::ObjectLambdaARN.new(arn.to_h)
-            else
-              raise ArgumentError,
-                    'Only Access Point, Outposts, and Object Lambdas ARNs '\
-                    'are currently supported.'
-            end
-          end
-
           def resolve_s3_use_arn_region(cfg)
             value = ENV['AWS_S3_USE_ARN_REGION'] ||
                     Aws.shared_config.s3_use_arn_region(profile: cfg.profile) ||
@@ -209,44 +63,6 @@ result in cross region requests.
             end
             value
           end
-
-          # Remove ARN from the path because we've already set the new host
-          def url_path(path, arn)
-            path = path.sub("/#{Seahorse::Util.uri_escape(arn.to_s)}", '')
-                       .sub("/#{arn}", '')
-            "/#{path}" unless path =~ /^\//
-            path
-          end
-
-          def validate_region_config!(arn, region, use_arn_region)
-            if ['s3-external-1', 'aws-global'].include?(region)
-              # These "regions" are not regional endpoints
-              unless use_arn_region
-                raise Aws::Errors::InvalidARNRegionError,
-                      'Configured client region is not a regional endpoint.'
-              end
-              # These "regions" are in the AWS partition
-              # Cannot use ARN region unless it's the same partition
-              unless arn.partition == 'aws'
-                raise Aws::Errors::InvalidARNPartitionError
-              end
-            else
-              # use_arn_region does not apply to MRAP (global) arns
-              unless arn.region.empty?
-                # Raise if the ARN and client regions are in different partitions
-                if use_arn_region &&
-                   !Aws::Partitions.partition(arn.partition).region?(region)
-                  raise Aws::Errors::InvalidARNPartitionError
-                end
-
-                # Raise if regions mismatch
-                # Either when it's a fips client or not using the ARN region
-                if !use_arn_region && region != arn.region
-                  raise Aws::Errors::InvalidARNRegionError
-                end
-              end
-            end
-          end
         end
       end
     end

data/lib/aws-sdk-s3/plugins/bucket_dns.rb

@@ -23,45 +23,10 @@ When set to `true`, the bucket name is always left in the
 request URI and never moved to the host as a sub-domain.
           DOCS
 
-        def add_handlers(handlers, config)
-          handlers.add(Handler, priority: 48) unless config.force_path_style
-        end
-
-        # @api private
-        class Handler < Seahorse::Client::Handler
-
-          def call(context)
-            move_dns_compat_bucket_to_subdomain(context)
-            @handler.call(context)
-          end
-
-          private
-
-          def move_dns_compat_bucket_to_subdomain(context)
-            bucket_name = context.params[:bucket]
-            endpoint = context.http_request.endpoint
-            if bucket_name &&
-               BucketDns.dns_compatible?(bucket_name, https?(endpoint)) &&
-               context.operation_name.to_s != 'get_bucket_location'
-              move_bucket_to_subdomain(bucket_name, endpoint)
-            end
-          end
-
-          def move_bucket_to_subdomain(bucket_name, endpoint)
-            endpoint.host = "#{bucket_name}.#{endpoint.host}"
-            path = endpoint.path.sub("/#{bucket_name}", '')
-            path = "/#{path}" unless path.match(/^\//)
-            endpoint.path = path
-          end
-
-          def https?(uri)
-            uri.scheme == 'https'
-          end
-
-        end
-
+        # These class methods were originally used in a handler in this plugin.
+        # SigV2 legacy signer needs this logic so we keep it here as utility.
+        # New endpoint resolution will check this as a matcher.
         class << self
-
           # @param [String] bucket_name
           # @param [Boolean] ssl
           # @return [Boolean]
@@ -81,7 +46,6 @@ request URI and never moved to the host as a sub-domain.
             bucket_name !~ /(\d+\.){3}\d+/ &&
             bucket_name !~ /[.-]{2}/
           end
-
         end
       end
     end

data/lib/aws-sdk-s3/plugins/bucket_name_restrictions.rb

@@ -13,12 +13,7 @@ module Aws
           def call(context)
             bucket_member = _bucket_member(context.operation.input.shape)
             if bucket_member && (bucket = context.params[bucket_member])
-              _resolved_region, arn = ARN.resolve_arn!(
-                bucket,
-                context.config.region,
-                context.config.s3_use_arn_region
-              )
-              if !arn && bucket.include?('/')
+              if !Aws::ARNParser.arn?(bucket) && bucket.include?('/')
                 raise ArgumentError,
                       'bucket name must not contain a forward-slash (/)'
               end

data/lib/aws-sdk-s3/plugins/dualstack.rb

@@ -5,9 +5,8 @@ module Aws
     module Plugins
       # @api private
       class Dualstack < Seahorse::Client::Plugin
-        def add_handlers(handlers, config)
+        def add_handlers(handlers, _config)
           handlers.add(OptionHandler, step: :initialize)
-          handlers.add(DualstackHandler, step: :build, priority: 49)
         end
 
         # @api private
@@ -18,57 +17,10 @@ module Aws
               dualstack = context.params.delete(:use_dualstack_endpoint)
             end
             dualstack = context.config.use_dualstack_endpoint if dualstack.nil?
-            # Raise if :endpoint and dualstack are both provided
-            if dualstack && !context.config.regional_endpoint
-              raise ArgumentError,
-                    'Cannot use both :use_dualstack_endpoint and :endpoint'
-            end
             context[:use_dualstack_endpoint] = dualstack
             @handler.call(context)
           end
         end
-
-        # @api private
-        class DualstackHandler < Seahorse::Client::Handler
-          def call(context)
-            # only rewrite the endpoint if it's not a custom endpoint
-            # accelerate/ARN already handle dualstack cases, so ignore these
-            # check to see if dualstack is on but configured off via operation
-            if context.config.regional_endpoint &&
-               use_dualstack_endpoint?(context)
-              apply_dualstack_endpoint(context)
-            end
-            @handler.call(context)
-          end
-
-          private
-
-          def apply_dualstack_endpoint(context)
-            new_endpoint = Aws::Partitions::EndpointProvider.resolve(
-              context.config.region,
-              's3',
-              'regional',
-              {
-                dualstack: context[:use_dualstack_endpoint],
-                fips: context.config.use_fips_endpoint
-              }
-            )
-            endpoint = URI.parse(context.http_request.endpoint.to_s)
-            endpoint.host = URI.parse(new_endpoint).host
-            context.http_request.endpoint = endpoint
-          end
-
-          def use_dualstack_endpoint?(context)
-            # case when dualstack is turned off via operation
-            (context[:use_dualstack_endpoint] ||
-              context.config.use_dualstack_endpoint) &&
-              # accelerate plugin already applies dualstack
-              !context[:use_accelerate_endpoint] &&
-              # arns handle dualstack
-              !context.metadata[:s3_arn]
-          end
-        end
-
       end
     end
   end