aws-sdk-s3 1.68.1 → 1.93.1

data/lib/aws-sdk-s3/customizations.rb

@@ -1,6 +1,9 @@
+# frozen_string_literal: true
+
 # utility classes
 require 'aws-sdk-s3/bucket_region_cache'
 require 'aws-sdk-s3/encryption'
+require 'aws-sdk-s3/encryption_v2'
 require 'aws-sdk-s3/file_part'
 require 'aws-sdk-s3/file_uploader'
 require 'aws-sdk-s3/file_downloader'
@@ -26,6 +29,6 @@ require 'aws-sdk-s3/customizations/types/list_object_versions_output'
   Aws::S3::ObjectVersion::Collection,
 ].each do |klass|
   klass.send(:alias_method, :delete, :batch_delete!)
-  klass.send(:extend, Aws::Deprecations)
+  klass.extend Aws::Deprecations
   klass.send(:deprecated, :delete, use: :batch_delete!)
 end

data/lib/aws-sdk-s3/customizations/bucket.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 
 module Aws
@@ -10,13 +12,12 @@ module Aws
       # Define a new initialize method that extracts out a bucket ARN.
       define_method(:initialize) do |*args|
         old_initialize.bind(self).call(*args)
-        bucket_name, region, arn = Plugins::BucketARN.resolve_arn!(
+        resolved_region, arn = Plugins::ARN.resolve_arn!(
           name,
           client.config.region,
           client.config.s3_use_arn_region
         )
-        @name = bucket_name
-        @client.config.region = region
+        @resolved_region = resolved_region
         @arn = arn
       end
 
@@ -87,20 +88,29 @@ module Aws
       # You can pass `virtual_host: true` to use the bucket name as the
       # host name.
       #
-      #     bucket = s3.bucket('my.bucket.com')
+      #     bucket = s3.bucket('my-bucket.com')
       #     bucket.url(virtual_host: true)
-      #     #=> "http://my.bucket.com"
+      #     #=> "http://my-bucket.com"
       #
       # @option options [Boolean] :virtual_host (false) When `true`,
       #   the bucket name will be used as the host name. This is useful
       #   when you have a CNAME configured for this bucket.
       #
+      # @option options [Boolean] :secure (true) When `false`, http
+      #   will be used with virtual_host.  This is required when
+      #   the bucket name has a dot (.) in it.
+      #
       # @return [String] the URL for this bucket.
       def url(options = {})
         if options[:virtual_host]
-          "http://#{name}"
+          scheme = options.fetch(:secure, true) ? 'https' : 'http'
+          "#{scheme}://#{name}"
         elsif @arn
-          Plugins::BucketARN.resolve_url!(URI.parse(s3_bucket_url), @arn).to_s
+          Plugins::ARN.resolve_url!(
+            client.config.endpoint.dup,
+            @arn,
+            @resolved_region
+          ).to_s
         else
           s3_bucket_url
         end

data/lib/aws-sdk-s3/customizations/multipart_upload.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     class MultipartUpload

data/lib/aws-sdk-s3/customizations/object.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     class Object
@@ -199,16 +201,22 @@ module Aws
       #     s3.bucket('bucket-name').object('obj-key').public_url
       #     #=> "https://bucket-name.s3.amazonaws.com/obj-key"
       #
-      # To use virtual hosted bucket url (disables https):
+      # To use virtual hosted bucket url.
+      # Uses https unless secure: false is set.  If the bucket
+      # name contains dots (.) then you will need to set secure: false.
       #
-      #     s3.bucket('my.bucket.com').object('key')
+      #     s3.bucket('my-bucket.com').object('key')
       #       .public_url(virtual_host: true)
-      #     #=> "http://my.bucket.com/key"
+      #     #=> "https://my-bucket.com/key"
       #
       # @option options [Boolean] :virtual_host (false) When `true`, the bucket
       #   name will be used as the host name. This is useful when you have
       #   a CNAME configured for the bucket.
       #
+      # @option options [Boolean] :secure (true) When `false`, http
+      #   will be used with virtual_host.  This is required when
+      #   the bucket name has a dot (.) in it.
+      #
       # @return [String]
       def public_url(options = {})
         url = URI.parse(bucket.url(options))
@@ -280,8 +288,8 @@ module Aws
       #     # small files are uploaded in a single API call
       #     obj.upload_file('/path/to/file')
       #
-      # Files larger than `:multipart_threshold` are uploaded using the
-      # Amazon S3 multipart upload APIs.
+      # Files larger than or equal to `:multipart_threshold` are uploaded
+      # using the Amazon S3 multipart upload APIs.
       #
       #     # large files are automatically split into parts
       #     # and the parts are uploaded in parallel
@@ -294,6 +302,14 @@ module Aws
       #       etag = response.etag
       #     end
       #
+      # You can provide a callback to monitor progress of the upload:
+      #
+      #     # bytes and totals are each an array with 1 entry per part
+      #     progress = Proc.new do |bytes, totals|
+      #       puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{totals[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / totals.sum }%" }
+      #     end
+      #     obj.upload_file('/path/to/file')
+      #
       # @param [String, Pathname, File, Tempfile] source A file on the local
       #   file system that will be uploaded as this object. This can either be
       #   a String or Pathname to the file, an open File object, or an open
@@ -303,13 +319,18 @@ module Aws
       #   will be empty.
       #
       # @option options [Integer] :multipart_threshold (15728640) Files larger
-      #   than `:multipart_threshold` are uploaded using the S3 multipart APIs.
+      #   than or equal to `:multipart_threshold` are uploaded using the S3
+      #   multipart APIs.
       #   Default threshold is 15MB.
       #
       # @option options [Integer] :thread_count (10) The number of parallel
       #   multipart uploads. This option is not used if the file is smaller than
       #   `:multipart_threshold`.
       #
+      # @option options [Proc] :progress_callback
+      #   A Proc that will be called when each chunk of the upload is sent.
+      #   It will be invoked with [bytes_read], [total_sizes]
+      #
       # @raise [MultipartUploadError] If an object is being uploaded in
       #   parts, and the upload can not be completed, then the upload is
       #   aborted and this error is raised.  The raised error has a `#errors`
@@ -318,7 +339,6 @@ module Aws
       #
       # @return [Boolean] Returns `true` when the object is uploaded
       #   without any errors.
-      #
       def upload_file(source, options = {})
         uploading_options = options.dup
         uploader = FileUploader.new(

data/lib/aws-sdk-s3/customizations/object_summary.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     class ObjectSummary

data/lib/aws-sdk-s3/customizations/types/list_object_versions_output.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class Aws::S3::Types::ListObjectVersionsOutput
 
   # TODO : Remove this customization once the resource code

data/lib/aws-sdk-s3/encryption.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'aws-sdk-s3/encryption/client'
 require 'aws-sdk-s3/encryption/decrypt_handler'
 require 'aws-sdk-s3/encryption/default_cipher_provider'
@@ -15,5 +17,7 @@ require 'aws-sdk-s3/encryption/default_key_provider'
 module Aws
   module S3
     module Encryption; end
+    AES_GCM_TAG_LEN_BYTES = 16
+    EC_USER_AGENT = 'S3CryptoV1n'
   end
 end

data/lib/aws-sdk-s3/encryption/client.rb

@@ -1,8 +1,14 @@
+# frozen_string_literal: true
+
 require 'forwardable'
 
 module Aws
   module S3
 
+    # [MAINTENANCE MODE] There is a new version of the Encryption Client.
+    # AWS strongly recommends upgrading to the {Aws::S3::EncryptionV2::Client},
+    # which provides updated data security best practices.
+    # See documentation for {Aws::S3::EncryptionV2::Client}.
     # Provides an encryption client that encrypts and decrypts data client-side,
     # storing the encrypted data in Amazon S3.
     #
@@ -227,6 +233,13 @@ module Aws
           @envelope_location = extract_location(options)
           @instruction_file_suffix = extract_suffix(options)
         end
+        deprecated :initialize,
+                   message:
+                     '[MAINTENANCE MODE] This version of the S3 Encryption client is currently in maintenance mode. ' \
+	                     'AWS strongly recommends upgrading to the Aws::S3::EncryptionV2::Client, ' \
+	                     'which provides updated data security best practices. ' \
+	                     'See documentation for Aws::S3::EncryptionV2::Client.'
+
 
         # @return [S3::Client]
         attr_reader :client

data/lib/aws-sdk-s3/encryption/decrypt_handler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 
 module Aws
@@ -5,6 +7,7 @@ module Aws
     module Encryption
       # @api private
       class DecryptHandler < Seahorse::Client::Handler
+        @@warned_response_target_proc = false
 
         V1_ENVELOPE_KEYS = %w(
           x-amz-key
@@ -20,7 +23,17 @@ module Aws
           x-amz-matdesc
         )
 
-        POSSIBLE_ENVELOPE_KEYS = (V1_ENVELOPE_KEYS + V2_ENVELOPE_KEYS).uniq
+        V2_OPTIONAL_KEYS = %w(x-amz-tag-len)
+
+        POSSIBLE_ENVELOPE_KEYS = (V1_ENVELOPE_KEYS +
+          V2_ENVELOPE_KEYS + V2_OPTIONAL_KEYS).uniq
+
+        POSSIBLE_WRAPPING_FORMATS = %w(
+          AES/GCM
+          kms
+          kms+context
+          RSA-OAEP-SHA1
+        )
 
         POSSIBLE_ENCRYPTION_FORMATS = %w(
           AES/GCM/NoPadding
@@ -28,8 +41,21 @@ module Aws
           AES/CBC/PKCS7Padding
         )
 
+        AUTH_REQUIRED_CEK_ALGS = %w(AES/GCM/NoPadding)
+
         def call(context)
           attach_http_event_listeners(context)
+          apply_cse_user_agent(context)
+
+          if context[:response_target].is_a?(Proc) && !@@warned_response_target_proc
+            @@warned_response_target_proc = true
+            warn(':response_target is a Proc, or a block was provided. ' \
+              'Read the entire object to the ' \
+              'end before you start using the decrypted data. This is to ' \
+              'verify that the object has not been modified since it ' \
+              'was encrypted.')
+          end
+
           @handler.call(context)
         end
 
@@ -38,9 +64,9 @@ module Aws
         def attach_http_event_listeners(context)
 
           context.http_response.on_headers(200) do
-            cipher = decryption_cipher(context)
-            decrypter = body_contains_auth_tag?(context) ?
-              authenticated_decrypter(context, cipher) :
+            cipher, envelope = decryption_cipher(context)
+            decrypter = body_contains_auth_tag?(envelope) ?
+              authenticated_decrypter(context, cipher, envelope) :
               IODecrypter.new(cipher, context.http_response.body)
             context.http_response.body = decrypter
           end
@@ -60,8 +86,13 @@ module Aws
         end
 
         def decryption_cipher(context)
-          if envelope = get_encryption_envelope(context)
-            context[:encryption][:cipher_provider].decryption_cipher(envelope)
+          if (envelope = get_encryption_envelope(context))
+            cipher = context[:encryption][:cipher_provider]
+                     .decryption_cipher(
+                       envelope,
+                       context[:encryption]
+                     )
+            [cipher, envelope]
           else
             raise Errors::DecryptionError, "unable to locate encryption envelope"
           end
@@ -97,13 +128,12 @@ module Aws
         end
 
         def extract_envelope(hash)
+          return nil unless hash
           return v1_envelope(hash) if hash.key?('x-amz-key')
           return v2_envelope(hash) if hash.key?('x-amz-key-v2')
           if hash.keys.any? { |key| key.match(/^x-amz-key-(.+)$/) }
             msg = "unsupported envelope encryption version #{$1}"
             raise Errors::DecryptionError, msg
-          else
-            nil # no envelope found
           end
         end
 
@@ -117,39 +147,31 @@ module Aws
             msg = "unsupported content encrypting key (cek) format: #{alg}"
             raise Errors::DecryptionError, msg
           end
-          unless envelope['x-amz-wrap-alg'] == 'kms'
-            # possible to support
-            #   RSA/ECB/OAEPWithSHA-256AndMGF1Padding
+          unless POSSIBLE_WRAPPING_FORMATS.include? envelope['x-amz-wrap-alg']
             alg = envelope['x-amz-wrap-alg'].inspect
             msg = "unsupported key wrapping algorithm: #{alg}"
             raise Errors::DecryptionError, msg
           end
-          unless V2_ENVELOPE_KEYS.sort == envelope.keys.sort
+          unless (missing_keys = V2_ENVELOPE_KEYS - envelope.keys).empty?
             msg = "incomplete v2 encryption envelope:\n"
-            msg += "  expected: #{V2_ENVELOPE_KEYS.join(',')}\n"
-            msg += "  got: #{envelope_keys.join(', ')}"
+            msg += "  missing: #{missing_keys.join(',')}\n"
             raise Errors::DecryptionError, msg
           end
           envelope
         end
 
-        # When the x-amz-meta-x-amz-tag-len header is present, it indicates
-        # that the body of this object has a trailing auth tag. The header
-        # indicates the length of that tag.
-        #
         # This method fetches the tag from the end of the object by
         # making a GET Object w/range request. This auth tag is used
         # to initialize the cipher, and the decrypter truncates the
         # auth tag from the body when writing the final bytes.
-        def authenticated_decrypter(context, cipher)
+        def authenticated_decrypter(context, cipher, envelope)
           if RUBY_VERSION.match(/1.9/)
-            raise "authenticated decryption not supported by OpeenSSL in Ruby version ~> 1.9"
+            raise "authenticated decryption not supported by OpenSSL in Ruby version ~> 1.9"
             raise Aws::Errors::NonSupportedRubyVersionError, msg
           end
           http_resp = context.http_response
           content_length = http_resp.headers['content-length'].to_i
-          auth_tag_length = http_resp.headers['x-amz-meta-x-amz-tag-len']
-          auth_tag_length = auth_tag_length.to_i / 8
+          auth_tag_length = auth_tag_length(envelope)
 
           auth_tag = context.client.get_object(
             bucket: context.params[:bucket],
@@ -161,16 +183,40 @@ module Aws
           cipher.auth_data = ''
 
           # The encrypted object contains both the cipher text
-          # plus a trailing auth tag. This decrypter will the body
-          # expect for the trailing auth tag.
+          # plus a trailing auth tag.
           IOAuthDecrypter.new(
             io: http_resp.body,
             encrypted_content_length: content_length - auth_tag_length,
             cipher: cipher)
         end
 
-        def body_contains_auth_tag?(context)
-          context.http_response.headers['x-amz-meta-x-amz-tag-len']
+        def body_contains_auth_tag?(envelope)
+          AUTH_REQUIRED_CEK_ALGS.include?(envelope['x-amz-cek-alg'])
+        end
+
+        # Determine the auth tag length from the algorithm
+        # Validate it against the value provided in the x-amz-tag-len
+        # Return the tag length in bytes
+        def auth_tag_length(envelope)
+          tag_length =
+            case envelope['x-amz-cek-alg']
+            when 'AES/GCM/NoPadding' then AES_GCM_TAG_LEN_BYTES
+            else
+              raise ArgumentError, 'Unsupported cek-alg: ' \
+                "#{envelope['x-amz-cek-alg']}"
+            end
+          if (tag_length * 8) != envelope['x-amz-tag-len'].to_i
+            raise Errors::DecryptionError, 'x-amz-tag-len does not match expected'
+          end
+          tag_length
+        end
+
+        def apply_cse_user_agent(context)
+          if context.config.user_agent_suffix.nil?
+            context.config.user_agent_suffix = EC_USER_AGENT
+          elsif !context.config.user_agent_suffix.include? EC_USER_AGENT
+            context.config.user_agent_suffix += " #{EC_USER_AGENT}"
+          end
         end
 
       end

data/lib/aws-sdk-s3/encryption/default_cipher_provider.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 
 module Aws
@@ -24,11 +26,48 @@ module Aws
 
         # @return [Cipher] Given an encryption envelope, returns a
         #   decryption cipher.
-        def decryption_cipher(envelope)
+        def decryption_cipher(envelope, options = {})
           master_key = @key_provider.key_for(envelope['x-amz-matdesc'])
-          key = Utils.decrypt(master_key, decode64(envelope['x-amz-key']))
-          iv = decode64(envelope['x-amz-iv'])
-          Utils.aes_decryption_cipher(:CBC, key, iv)
+          if envelope.key? 'x-amz-key'
+            # Support for decryption of legacy objects
+            key = Utils.decrypt(master_key, decode64(envelope['x-amz-key']))
+            iv = decode64(envelope['x-amz-iv'])
+            Utils.aes_decryption_cipher(:CBC, key, iv)
+          else
+            if envelope['x-amz-cek-alg'] != 'AES/GCM/NoPadding'
+              raise ArgumentError, 'Unsupported cek-alg: ' \
+                "#{envelope['x-amz-cek-alg']}"
+            end
+            key =
+              case envelope['x-amz-wrap-alg']
+              when 'AES/GCM'
+                if master_key.is_a? OpenSSL::PKey::RSA
+                  raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with an RSA key and the x-amz-wrap-alg is AES/GCM.'
+                end
+                Utils.decrypt_aes_gcm(master_key,
+                                      decode64(envelope['x-amz-key-v2']),
+                                      envelope['x-amz-cek-alg'])
+              when 'RSA-OAEP-SHA1'
+                unless master_key.is_a? OpenSSL::PKey::RSA
+                  raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with an AES key and the x-amz-wrap-alg is RSA-OAEP-SHA1.'
+                end
+                key, cek_alg = Utils.decrypt_rsa(master_key, decode64(envelope['x-amz-key-v2']))
+                raise Errors::DecryptionError unless cek_alg == envelope['x-amz-cek-alg']
+                key
+              when 'kms+context'
+                raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with a user provided key and the x-amz-wrap-alg is' \
+                    ' kms+context.  Please configure the client with the' \
+                    ' required kms_key_id'
+              else
+                raise ArgumentError, 'Unsupported wrap-alg: ' \
+                "#{envelope['x-amz-wrap-alg']}"
+              end
+            iv = decode64(envelope['x-amz-iv'])
+            Utils.aes_decryption_cipher(:GCM, key, iv)
+          end
         end
 
         private
@@ -56,7 +95,6 @@ module Aws
         def decode64(str)
           Base64.decode64(str)
         end
-
       end
     end
   end