aws-sdk-s3 1.48.0 → 1.183.0

data/lib/aws-sdk-s3/plugins/streaming_retry.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Aws
+  module S3
+    module Plugins
+
+      # A wrapper around BlockIO that adds no-ops for truncate and rewind
+      # @api private
+      class RetryableBlockIO
+        extend Forwardable
+        def_delegators :@block_io, :write, :read, :size
+
+        def initialize(block_io)
+          @block_io = block_io
+        end
+
+        def truncate(_integer); end
+
+        def rewind; end
+      end
+
+      # A wrapper around ManagedFile that adds no-ops for truncate and rewind
+      # @api private
+      class RetryableManagedFile
+        extend Forwardable
+        def_delegators :@file, :write, :read, :size, :open?, :close
+
+        def initialize(managed_file)
+          @file = managed_file
+        end
+
+        def truncate(_integer); end
+
+        def rewind; end
+      end
+
+      class NonRetryableStreamingError < StandardError
+
+        def initialize(error)
+          super('Unable to retry request - retry could result in processing duplicated chunks.')
+          set_backtrace(error.backtrace)
+          @original_error = error
+        end
+
+        attr_reader :original_error
+      end
+
+      # This handler works with the ResponseTarget plugin to provide smart
+      # retries of S3 streaming operations that support the range parameter
+      # (currently only: get_object).  When a 200 OK with a TruncatedBodyError
+      # is received this handler will add a range header that excludes the
+      # data that has already been processed (written to file or sent to
+      # the target Proc).
+      # It is important to not write data to the custom target in the case of
+      # a non-success response. We do not want to write an XML error
+      # message to someone's file or pass it to a user's Proc.
+      # @api private
+      class StreamingRetry < Seahorse::Client::Plugin
+
+        class Handler < Seahorse::Client::Handler
+
+          def call(context)
+            target = context.params[:response_target] || context[:response_target]
+
+            # retry is only supported when range is NOT set on the initial request
+            if supported_target?(target) && !context.params[:range]
+              add_event_listeners(context, target)
+            end
+            @handler.call(context)
+          end
+
+          private
+
+          def add_event_listeners(context, target)
+            context.http_response.on_headers(200..299) do
+              case context.http_response.body
+              when Seahorse::Client::BlockIO then
+                context.http_response.body = RetryableBlockIO.new(context.http_response.body)
+              when Seahorse::Client::ManagedFile then
+                context.http_response.body = RetryableManagedFile.new(context.http_response.body)
+              end
+            end
+
+            context.http_response.on_headers(400..599) do
+              context.http_response.body = StringIO.new # something to write the error to
+            end
+
+            context.http_response.on_success(200..299) do
+              body = context.http_response.body
+              if body.is_a?(RetryableManagedFile) && body.open?
+                body.close
+              end
+            end
+
+            context.http_response.on_error do |error|
+              if retryable_body?(context)
+                if truncated_body?(error)
+                  context.http_request.headers[:range] = "bytes=#{context.http_response.body.size}-"
+                else
+                  case context.http_response.body
+                  when RetryableManagedFile
+                    # call rewind on the underlying file
+                    context.http_response.body.instance_variable_get(:@file).rewind
+                  else
+                    raise NonRetryableStreamingError, error
+                  end
+                end
+              end
+            end
+          end
+
+          def truncated_body?(error)
+            error.is_a?(Seahorse::Client::NetworkingError) &&
+              error.original_error.is_a?(
+                Seahorse::Client::NetHttp::Handler::TruncatedBodyError
+              )
+          end
+
+          def retryable_body?(context)
+            context.http_response.body.is_a?(RetryableBlockIO) ||
+              context.http_response.body.is_a?(RetryableManagedFile)
+          end
+
+          def supported_target?(target)
+            case target
+            when Proc, String, Pathname then true
+            else false
+            end
+          end
+        end
+
+        handler(Handler, step: :sign, operations: [:get_object], priority: 10)
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/plugins/url_encoded_keys.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 require 'cgi'
 

data/lib/aws-sdk-s3/presigned_post.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'openssl'
 require 'base64'
 
@@ -96,7 +98,7 @@ module Aws
     # or call the associated method.
     #
     # ```ruby
-    # post = Aws::S3::PresignedPost.new(creds, region, bucket).
+    # post = Aws::S3::PresignedPost.new(creds, region, bucket)
     # post.content_type('text/plain')
     # ```
     #
@@ -174,44 +176,76 @@ module Aws
     # ```
     #
     class PresignedPost
+      @@allowed_fields = []
 
       # @param [Credentials] credentials Security credentials for signing
       #   the post policy.
       # @param [String] bucket_region Region of the target bucket.
       # @param [String] bucket_name Name of the target bucket.
+      # @option options [Boolean] :use_accelerate_endpoint (false) When `true`,
+      #   PresignedPost will attempt to use accelerated endpoint.
+      # @option options [String] :url See {PresignedPost#url}.
+      # @option options [Sting, Array<String>] :allow_any
+      #   See {PresignedPost#allow_any}.
       # @option options [Time] :signature_expiration Specify when the signature on
       #   the post will expire. Defaults to one hour from creation of the
       #   presigned post. May not exceed one week from creation time.
       # @option options [String] :key See {PresignedPost#key}.
-      # @option options [String] :key_starts_with See {PresignedPost#key_starts_with}.
+      # @option options [String] :key_starts_with
+      #   See {PresignedPost#key_starts_with}.
       # @option options [String] :acl See {PresignedPost#acl}.
-      # @option options [String] :acl_starts_with See {PresignedPost#acl_starts_with}.
-      # @option options [String] :cache_control See {PresignedPost#cache_control}.
-      # @option options [String] :cache_control_starts_with See {PresignedPost#cache_control_starts_with}.
+      # @option options [String] :acl_starts_with
+      #   See {PresignedPost#acl_starts_with}.
+      # @option options [String] :cache_control
+      #   See {PresignedPost#cache_control}.
+      # @option options [String] :cache_control_starts_with
+      #   See {PresignedPost#cache_control_starts_with}.
       # @option options [String] :content_type See {PresignedPost#content_type}.
-      # @option options [String] :content_type_starts_with See {PresignedPost#content_type_starts_with}.
-      # @option options [String] :content_disposition See {PresignedPost#content_disposition}.
-      # @option options [String] :content_disposition_starts_with See {PresignedPost#content_disposition_starts_with}.
-      # @option options [String] :content_encoding See {PresignedPost#content_encoding}.
-      # @option options [String] :content_encoding_starts_with See {PresignedPost#content_encoding_starts_with}.
-      # @option options [String] :expires See {PresignedPost#expires}.
-      # @option options [String] :expires_starts_with See {PresignedPost#expires_starts_with}.
-      # @option options [Range<Integer>] :content_length_range See {PresignedPost#content_length_range}.
-      # @option options [String] :success_action_redirect See {PresignedPost#success_action_redirect}.
-      # @option options [String] :success_action_redirect_starts_with See {PresignedPost#success_action_redirect_starts_with}.
-      # @option options [String] :success_action_status See {PresignedPost#success_action_status}.
-      # @option options [String] :storage_class See {PresignedPost#storage_class}.
-      # @option options [String] :website_redirect_location See {PresignedPost#website_redirect_location}.
-      # @option options [Hash<String,String>] :metadata See {PresignedPost#metadata}.
-      # @option options [Hash<String,String>] :metadata_starts_with See {PresignedPost#metadata_starts_with}.
-      # @option options [String] :server_side_encryption See {PresignedPost#server_side_encryption}.
-      # @option options [String] :server_side_encryption_aws_kms_key_id See {PresignedPost#server_side_encryption_aws_kms_key_id}.
-      # @option options [String] :server_side_encryption_customer_algorithm See {PresignedPost#server_side_encryption_customer_algorithm}.
-      # @option options [String] :server_side_encryption_customer_key See {PresignedPost#server_side_encryption_customer_key}.
+      # @option options [String] :content_type_starts_with
+      #   See {PresignedPost#content_type_starts_with}.
+      # @option options [String] :content_disposition
+      #   See {PresignedPost#content_disposition}.
+      # @option options [String] :content_disposition_starts_with
+      #   See {PresignedPost#content_disposition_starts_with}.
+      # @option options [String] :content_encoding
+      #   See {PresignedPost#content_encoding}.
+      # @option options [String] :content_encoding_starts_with
+      #   See {PresignedPost#content_encoding_starts_with}.
+      # @option options [Time] :expires See {PresignedPost#expires}.
+      # @option options [String] :expires_starts_with
+      #   See {PresignedPost#expires_starts_with}.
+      # @option options [Range<Integer>] :content_length_range
+      #   See {PresignedPost#content_length_range}.
+      # @option options [String] :success_action_redirect
+      #   See {PresignedPost#success_action_redirect}.
+      # @option options [String] :success_action_redirect_starts_with
+      #   See {PresignedPost#success_action_redirect_starts_with}.
+      # @option options [String] :success_action_status
+      #   See {PresignedPost#success_action_status}.
+      # @option options [String] :storage_class
+      #   See {PresignedPost#storage_class}.
+      # @option options [String] :website_redirect_location
+      #   See {PresignedPost#website_redirect_location}.
+      # @option options [Hash<String,String>] :metadata
+      #   See {PresignedPost#metadata}.
+      # @option options [Hash<String,String>] :metadata_starts_with
+      #   See {PresignedPost#metadata_starts_with}.
+      # @option options [String] :server_side_encryption
+      #   See {PresignedPost#server_side_encryption}.
+      # @option options [String] :server_side_encryption_aws_kms_key_id
+      #   See {PresignedPost#server_side_encryption_aws_kms_key_id}.
+      # @option options [String] :server_side_encryption_customer_algorithm
+      #   See {PresignedPost#server_side_encryption_customer_algorithm}.
+      # @option options [String] :server_side_encryption_customer_key
+      #   See {PresignedPost#server_side_encryption_customer_key}.
+      # @option options [String] :server_side_encryption_customer_key_starts_with
+      #   See {PresignedPost#server_side_encryption_customer_key_starts_with}.
       def initialize(credentials, bucket_region, bucket_name, options = {})
         @credentials = credentials.credentials
         @bucket_region = bucket_region
         @bucket_name = bucket_name
+        @accelerate = !!options.delete(:use_accelerate_endpoint)
+        options.delete(:url) if @accelerate # resource methods pass url
         @url = options.delete(:url) || bucket_url
         @fields = {}
         @key_set = false
@@ -221,7 +255,12 @@ module Aws
           case option_name
           when :allow_any then allow_any(option_value)
           when :signature_expiration then @signature_expiration = option_value
-          else send("#{option_name}", option_value)
+          else
+            if @@allowed_fields.include?(option_name)
+              send("#{option_name}", option_value)
+            else
+              raise ArgumentError, "Unsupported option: #{option_name}"
+            end
           end
         end
       end
@@ -234,7 +273,7 @@ module Aws
       #   as hidden input fields.
       def fields
         check_required_values!
-        datetime = Time.now.utc.strftime("%Y%m%dT%H%M%SZ")
+        datetime = Time.now.utc.strftime('%Y%m%dT%H%M%SZ')
         fields = @fields.dup
         fields.update('policy' => policy(datetime))
         fields.update(signature_fields(datetime))
@@ -253,44 +292,52 @@ module Aws
       end
 
       # @api private
-      def self.define_field(field, *args)
+      def self.define_field(field, *args, &block)
+        @@allowed_fields << field
         options = args.last.is_a?(Hash) ? args.pop : {}
         field_name = args.last || field.to_s
 
-        define_method("#{field}") do |value|
-          with(field_name, value)
-        end
+        if block_given?
+          define_method("#{field}", block)
+        else
+          define_method("#{field}") do |value|
+            with(field_name, value)
+          end
 
-        if options[:starts_with]
-          define_method("#{field}_starts_with") do |value|
-            starts_with(field_name, value)
+          if options[:starts_with]
+            @@allowed_fields << "#{field}_starts_with".to_sym
+            define_method("#{field}_starts_with") do |value|
+              starts_with(field_name, value)
+            end
           end
         end
       end
 
       # @!group Fields
 
-      # The key to use for the uploaded object. Use can use `${filename}`
-      # as a variable in the key. This will be replaced with the name
-      # of the file as provided by the user.
+      # @!method key(key)
+      #   The key to use for the uploaded object. You can use `${filename}`
+      #   as a variable in the key. This will be replaced with the name
+      #   of the file as provided by the user.
       #
-      # For example, if the key is given as `/user/betty/${filename}` and
-      # the file uploaded is named `lolcatz.jpg`, the resultant key will
-      # be `/user/betty/lolcatz.jpg`.
+      #   For example, if the key is given as `/user/betty/${filename}` and
+      #   the file uploaded is named `lolcatz.jpg`, the resultant key will
+      #   be `/user/betty/lolcatz.jpg`.
       #
-      # @param [String] key
-      # @see http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html)
-      # @return [self]
-      def key(key)
+      #   @param [String] key
+      #   @see http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html)
+      #   @return [self]
+      define_field(:key) do |key|
         @key_set = true
         with('key', key)
       end
 
-      # Specify a prefix the uploaded
-      # @param [String] prefix
-      # @see #key
-      # @return [self]
-      def key_starts_with(prefix)
+      # @!method key_starts_with(prefix)
+      #   Specify a prefix the uploaded
+      #   @param [String] prefix
+      #   @see #key
+      #   @return [self]
+      define_field(:key_starts_with) do |prefix|
         @key_set = true
         starts_with('key', prefix)
       end
@@ -367,27 +414,30 @@ module Aws
       #   @return [self]
       define_field(:content_encoding, 'Content-Encoding', starts_with: true)
 
-      # The date and time at which the object is no longer cacheable.
-      # @note This does not affect the expiration of the presigned post
-      #   signature.
-      # @param [Time] time
-      # @see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21
-      # @return [self]
-      def expires(time)
+      # @!method expires(time)
+      #   The date and time at which the object is no longer cacheable.
+      #   @note This does not affect the expiration of the presigned post
+      #     signature.
+      #   @param [Time] time
+      #   @see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21
+      #   @return [self]
+      define_field(:expires) do |time|
         with('Expires', time.httpdate)
       end
 
-      # @param [String] prefix
-      # @see #expires
-      # @return [self]
-      def expires_starts_with(prefix)
+      # @!method expires_starts_with(prefix)
+      #   @param [String] prefix
+      #   @see #expires
+      #   @return [self]
+      define_field(:expires_starts_with) do |prefix|
         starts_with('Expires', prefix)
       end
 
-      # The minimum and maximum allowable size for the uploaded content.
-      # @param [Range<Integer>] byte_range
-      # @return [self]
-      def content_length_range(byte_range)
+      # @!method content_length_range(byte_range)
+      #   The minimum and maximum allowable size for the uploaded content.
+      #   @param [Range<Integer>] byte_range
+      #   @return [self]
+      define_field(:content_length_range) do |byte_range|
         min = byte_range.begin
         max = byte_range.end
         max -= 1 if byte_range.exclude_end?
@@ -462,22 +512,24 @@ module Aws
       #   @return [self]
       define_field(:website_redirect_location, 'x-amz-website-redirect-location')
 
-      # Metadata hash to store with the uploaded object. Hash keys will be
-      # prefixed with "x-amz-meta-".
-      # @param [Hash<String,String>] hash
-      # @return [self]
-      def metadata(hash)
+      # @!method metadata(hash)
+      #   Metadata hash to store with the uploaded object. Hash keys will be
+      #   prefixed with "x-amz-meta-".
+      #   @param [Hash<String,String>] hash
+      #   @return [self]
+      define_field(:metadata) do |hash|
         hash.each do |key, value|
           with("x-amz-meta-#{key}", value)
         end
         self
       end
 
-      # Specify allowable prefix for each key in the metadata hash.
-      # @param [Hash<String,String>] hash
-      # @see #metadata
-      # @return [self]
-      def metadata_starts_with(hash)
+      # @!method metadata_starts_with(hash)
+      #   Specify allowable prefix for each key in the metadata hash.
+      #   @param [Hash<String,String>] hash
+      #   @see #metadata
+      #   @return [self]
+      define_field(:metadata_starts_with) do |hash|
         hash.each do |key, value|
           starts_with("x-amz-meta-#{key}", value)
         end
@@ -505,7 +557,10 @@ module Aws
       #   (KMS) master encryption key to use for the object.
       #   @param [String] value
       #   @return [self]
-      define_field(:server_side_encryption_aws_kms_key_id, 'x-amz-server-side-encryption-aws-kms-key-id')
+      define_field(
+        :server_side_encryption_aws_kms_key_id,
+        'x-amz-server-side-encryption-aws-kms-key-id'
+      )
 
       # @!endgroup
 
@@ -518,27 +573,32 @@ module Aws
       #   @param [String] value
       #   @see #server_side_encryption_customer_key
       #   @return [self]
-      define_field(:server_side_encryption_customer_algorithm, 'x-amz-server-side-encryption-customer-algorithm')
-
-      # Specifies the customer-provided encryption key for Amazon S3 to use
-      # in encrypting data. This value is used to store the object and then
-      # it is discarded; Amazon does not store the encryption key.
+      define_field(
+        :server_side_encryption_customer_algorithm,
+        'x-amz-server-side-encryption-customer-algorithm'
+      )
+
+      # @!method server_side_encryption_customer_key(value)
+      #   Specifies the customer-provided encryption key for Amazon S3 to use
+      #   in encrypting data. This value is used to store the object and then
+      #   it is discarded; Amazon does not store the encryption key.
       #
-      # You must also call {#server_side_encryption_customer_algorithm}.
+      #   You must also call {#server_side_encryption_customer_algorithm}.
       #
-      # @param [String] value
-      # @see #server_side_encryption_customer_algorithm
-      # @return [self]
-      def server_side_encryption_customer_key(value)
+      #   @param [String] value
+      #   @see #server_side_encryption_customer_algorithm
+      #   @return [self]
+      define_field(:server_side_encryption_customer_key) do |value|
         field_name = 'x-amz-server-side-encryption-customer-key'
         with(field_name, base64(value))
         with(field_name + '-MD5', base64(OpenSSL::Digest::MD5.digest(value)))
       end
 
-      # @param [String] prefix
-      # @see #server_side_encryption_customer_key
-      # @return [self]
-      def server_side_encryption_customer_key_starts_with(prefix)
+      # @!method server_side_encryption_customer_key_starts_with(prefix)
+      #   @param [String] prefix
+      #   @see #server_side_encryption_customer_key
+      #   @return [self]
+      define_field(:server_side_encryption_customer_key_starts_with) do |prefix|
         field_name = 'x-amz-server-side-encryption-customer-key'
         starts_with(field_name, prefix)
       end
@@ -571,21 +631,22 @@ module Aws
 
       def check_required_values!
         unless @key_set
-          msg = "key required; you must provide a key via :key, "
-          msg << ":key_starts_with, or :allow_any => ['key']"
+          msg = 'key required; you must provide a key via :key, '\
+                ":key_starts_with, or :allow_any => ['key']"
           raise msg
         end
       end
 
       def bucket_url
-        url = Aws::Partitions::EndpointProvider.resolve(@bucket_region, 's3')
-        url = URI.parse(url)
-        if Plugins::BucketDns.dns_compatible?(@bucket_name, true)
-          url.host = @bucket_name + '.' + url.host
-        else
-          url.path = '/' + @bucket_name
-        end
-        url.to_s
+        # Taken from Aws::S3::Endpoints module
+        params = Aws::S3::EndpointParameters.new(
+          bucket: @bucket_name,
+          region: @bucket_region,
+          accelerate: @accelerate,
+          use_global_endpoint: true
+        )
+        endpoint = Aws::S3::EndpointProvider.new.resolve_endpoint(params)
+        endpoint.url
       end
 
       # @return [Hash]
@@ -613,7 +674,7 @@ module Aws
 
       def signature(datetime, string_to_sign)
         k_secret = @credentials.secret_access_key
-        k_date = hmac("AWS4" + k_secret, datetime[0,8])
+        k_date = hmac('AWS4' + k_secret, datetime[0,8])
         k_region = hmac(k_date, @bucket_region)
         k_service = hmac(k_region, 's3')
         k_credentials = hmac(k_service, 'aws4_request')