aws-sdk-s3 1.73.0 → 1.78.0

data/lib/aws-sdk-s3/customizations/object.rb

@@ -296,6 +296,14 @@ module Aws
       #       etag = response.etag
       #     end
       #
+      # You can provide a callback to monitor progress of the upload:
+      #
+      #     # bytes and totals are each an array with 1 entry per part
+      #     progress = Proc.new do |bytes, totals|
+      #       puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{totals[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / totals.sum }%" }
+      #     end
+      #     obj.upload_file('/path/to/file')
+      #
       # @param [String, Pathname, File, Tempfile] source A file on the local
       #   file system that will be uploaded as this object. This can either be
       #   a String or Pathname to the file, an open File object, or an open
@@ -312,6 +320,10 @@ module Aws
       #   multipart uploads. This option is not used if the file is smaller than
       #   `:multipart_threshold`.
       #
+      # @option options [Proc] :progress_callback
+      #   A Proc that will be called when each chunk of the upload is sent.
+      #   It will be invoked with [bytes_read], [total_sizes]
+      #
       # @raise [MultipartUploadError] If an object is being uploaded in
       #   parts, and the upload can not be completed, then the upload is
       #   aborted and this error is raised.  The raised error has a `#errors`
@@ -320,7 +332,6 @@ module Aws
       #
       # @return [Boolean] Returns `true` when the object is uploaded
       #   without any errors.
-      #
       def upload_file(source, options = {})
         uploading_options = options.dup
         uploader = FileUploader.new(

data/lib/aws-sdk-s3/encryption.rb

@@ -17,5 +17,7 @@ require 'aws-sdk-s3/encryption/default_key_provider'
 module Aws
   module S3
     module Encryption; end
+    AES_GCM_TAG_LEN_BYTES = 16
+    EC_USER_AGENT = 'S3CryptoV1n'
   end
 end

data/lib/aws-sdk-s3/encryption/client.rb

@@ -5,6 +5,10 @@ require 'forwardable'
 module Aws
   module S3
 
+    # [MAINTENANCE MODE] There is a new version of the Encryption Client.
+    # AWS strongly recommends upgrading to the {Aws::S3::EncryptionV2::Client},
+    # which provides updated data security best practices.
+    # See documentation for {Aws::S3::EncryptionV2::Client}.
     # Provides an encryption client that encrypts and decrypts data client-side,
     # storing the encrypted data in Amazon S3.
     #
@@ -229,6 +233,13 @@ module Aws
           @envelope_location = extract_location(options)
           @instruction_file_suffix = extract_suffix(options)
         end
+        deprecated :initialize,
+                   message:
+                     '[MAINTENANCE MODE] This version of the S3 Encryption client is currently in maintenance mode. ' \
+	                     'AWS strongly recommends upgrading to the Aws::S3::EncryptionV2::Client, ' \
+	                     'which provides updated data security best practices. ' \
+	                     'See documentation for Aws::S3::EncryptionV2::Client.'
+
 
         # @return [S3::Client]
         attr_reader :client

data/lib/aws-sdk-s3/encryption/decrypt_handler.rb

@@ -7,6 +7,7 @@ module Aws
     module Encryption
       # @api private
       class DecryptHandler < Seahorse::Client::Handler
+        @@warned_response_target_proc = false
 
         V1_ENVELOPE_KEYS = %w(
           x-amz-key
@@ -22,7 +23,17 @@ module Aws
           x-amz-matdesc
         )
 
-        POSSIBLE_ENVELOPE_KEYS = (V1_ENVELOPE_KEYS + V2_ENVELOPE_KEYS).uniq
+        V2_OPTIONAL_KEYS = %w(x-amz-tag-len)
+
+        POSSIBLE_ENVELOPE_KEYS = (V1_ENVELOPE_KEYS +
+          V2_ENVELOPE_KEYS + V2_OPTIONAL_KEYS).uniq
+
+        POSSIBLE_WRAPPING_FORMATS = %w(
+          AES/GCM
+          kms
+          kms+context
+          RSA-OAEP-SHA1
+        )
 
         POSSIBLE_ENCRYPTION_FORMATS = %w(
           AES/GCM/NoPadding
@@ -30,9 +41,21 @@ module Aws
           AES/CBC/PKCS7Padding
         )
 
+        AUTH_REQUIRED_CEK_ALGS = %w(AES/GCM/NoPadding)
+
         def call(context)
           attach_http_event_listeners(context)
           apply_cse_user_agent(context)
+
+          if context[:response_target].is_a?(Proc) && !@@warned_response_target_proc
+            @@warned_response_target_proc = true
+            warn(':response_target is a Proc, or a block was provided. ' \
+              'Read the entire object to the ' \
+              'end before you start using the decrypted data. This is to ' \
+              'verify that the object has not been modified since it ' \
+              'was encrypted.')
+          end
+
           @handler.call(context)
         end
 
@@ -41,9 +64,9 @@ module Aws
         def attach_http_event_listeners(context)
 
           context.http_response.on_headers(200) do
-            cipher = decryption_cipher(context)
-            decrypter = body_contains_auth_tag?(context) ?
-              authenticated_decrypter(context, cipher) :
+            cipher, envelope = decryption_cipher(context)
+            decrypter = body_contains_auth_tag?(envelope) ?
+              authenticated_decrypter(context, cipher, envelope) :
               IODecrypter.new(cipher, context.http_response.body)
             context.http_response.body = decrypter
           end
@@ -63,8 +86,13 @@ module Aws
         end
 
         def decryption_cipher(context)
-          if envelope = get_encryption_envelope(context)
-            context[:encryption][:cipher_provider].decryption_cipher(envelope)
+          if (envelope = get_encryption_envelope(context))
+            cipher = context[:encryption][:cipher_provider]
+                     .decryption_cipher(
+                       envelope,
+                       context[:encryption]
+                     )
+            [cipher, envelope]
           else
             raise Errors::DecryptionError, "unable to locate encryption envelope"
           end
@@ -100,13 +128,12 @@ module Aws
         end
 
         def extract_envelope(hash)
+          return nil unless hash
           return v1_envelope(hash) if hash.key?('x-amz-key')
           return v2_envelope(hash) if hash.key?('x-amz-key-v2')
           if hash.keys.any? { |key| key.match(/^x-amz-key-(.+)$/) }
             msg = "unsupported envelope encryption version #{$1}"
             raise Errors::DecryptionError, msg
-          else
-            nil # no envelope found
           end
         end
 
@@ -120,39 +147,31 @@ module Aws
             msg = "unsupported content encrypting key (cek) format: #{alg}"
             raise Errors::DecryptionError, msg
           end
-          unless envelope['x-amz-wrap-alg'] == 'kms'
-            # possible to support
-            #   RSA/ECB/OAEPWithSHA-256AndMGF1Padding
+          unless POSSIBLE_WRAPPING_FORMATS.include? envelope['x-amz-wrap-alg']
             alg = envelope['x-amz-wrap-alg'].inspect
             msg = "unsupported key wrapping algorithm: #{alg}"
             raise Errors::DecryptionError, msg
           end
-          unless V2_ENVELOPE_KEYS.sort == envelope.keys.sort
+          unless (missing_keys = V2_ENVELOPE_KEYS - envelope.keys).empty?
             msg = "incomplete v2 encryption envelope:\n"
-            msg += "  expected: #{V2_ENVELOPE_KEYS.join(',')}\n"
-            msg += "  got: #{envelope_keys.join(', ')}"
+            msg += "  missing: #{missing_keys.join(',')}\n"
             raise Errors::DecryptionError, msg
           end
           envelope
         end
 
-        # When the x-amz-meta-x-amz-tag-len header is present, it indicates
-        # that the body of this object has a trailing auth tag. The header
-        # indicates the length of that tag.
-        #
         # This method fetches the tag from the end of the object by
         # making a GET Object w/range request. This auth tag is used
         # to initialize the cipher, and the decrypter truncates the
         # auth tag from the body when writing the final bytes.
-        def authenticated_decrypter(context, cipher)
+        def authenticated_decrypter(context, cipher, envelope)
           if RUBY_VERSION.match(/1.9/)
-            raise "authenticated decryption not supported by OpeenSSL in Ruby version ~> 1.9"
+            raise "authenticated decryption not supported by OpenSSL in Ruby version ~> 1.9"
             raise Aws::Errors::NonSupportedRubyVersionError, msg
           end
           http_resp = context.http_response
           content_length = http_resp.headers['content-length'].to_i
-          auth_tag_length = http_resp.headers['x-amz-meta-x-amz-tag-len']
-          auth_tag_length = auth_tag_length.to_i / 8
+          auth_tag_length = auth_tag_length(envelope)
 
           auth_tag = context.client.get_object(
             bucket: context.params[:bucket],
@@ -164,23 +183,39 @@ module Aws
           cipher.auth_data = ''
 
           # The encrypted object contains both the cipher text
-          # plus a trailing auth tag. This decrypter will the body
-          # expect for the trailing auth tag.
+          # plus a trailing auth tag.
           IOAuthDecrypter.new(
             io: http_resp.body,
             encrypted_content_length: content_length - auth_tag_length,
             cipher: cipher)
         end
 
-        def body_contains_auth_tag?(context)
-          context.http_response.headers['x-amz-meta-x-amz-tag-len']
+        def body_contains_auth_tag?(envelope)
+          AUTH_REQUIRED_CEK_ALGS.include?(envelope['x-amz-cek-alg'])
+        end
+
+        # Determine the auth tag length from the algorithm
+        # Validate it against the value provided in the x-amz-tag-len
+        # Return the tag length in bytes
+        def auth_tag_length(envelope)
+          tag_length =
+            case envelope['x-amz-cek-alg']
+            when 'AES/GCM/NoPadding' then AES_GCM_TAG_LEN_BYTES
+            else
+              raise ArgumentError, 'Unsupported cek-alg: ' \
+                "#{envelope['x-amz-cek-alg']}"
+            end
+          if (tag_length * 8) != envelope['x-amz-tag-len'].to_i
+            raise Errors::DecryptionError, 'x-amz-tag-len does not match expected'
+          end
+          tag_length
         end
 
         def apply_cse_user_agent(context)
           if context.config.user_agent_suffix.nil?
-            context.config.user_agent_suffix = 'CSE_V1'
-          elsif !context.config.user_agent_suffix.include? 'CSE_V1'
-            context.config.user_agent_suffix += ' CSE_V1'
+            context.config.user_agent_suffix = EC_USER_AGENT
+          elsif !context.config.user_agent_suffix.include? EC_USER_AGENT
+            context.config.user_agent_suffix += " #{EC_USER_AGENT}"
           end
         end
 

data/lib/aws-sdk-s3/encryption/default_cipher_provider.rb

@@ -26,11 +26,48 @@ module Aws
 
         # @return [Cipher] Given an encryption envelope, returns a
         #   decryption cipher.
-        def decryption_cipher(envelope)
+        def decryption_cipher(envelope, options = {})
           master_key = @key_provider.key_for(envelope['x-amz-matdesc'])
-          key = Utils.decrypt(master_key, decode64(envelope['x-amz-key']))
-          iv = decode64(envelope['x-amz-iv'])
-          Utils.aes_decryption_cipher(:CBC, key, iv)
+          if envelope.key? 'x-amz-key'
+            # Support for decryption of legacy objects
+            key = Utils.decrypt(master_key, decode64(envelope['x-amz-key']))
+            iv = decode64(envelope['x-amz-iv'])
+            Utils.aes_decryption_cipher(:CBC, key, iv)
+          else
+            if envelope['x-amz-cek-alg'] != 'AES/GCM/NoPadding'
+              raise ArgumentError, 'Unsupported cek-alg: ' \
+                "#{envelope['x-amz-cek-alg']}"
+            end
+            key =
+              case envelope['x-amz-wrap-alg']
+              when 'AES/GCM'
+                if master_key.is_a? OpenSSL::PKey::RSA
+                  raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with an RSA key and the x-amz-wrap-alg is AES/GCM.'
+                end
+                Utils.decrypt_aes_gcm(master_key,
+                                      decode64(envelope['x-amz-key-v2']),
+                                      envelope['x-amz-cek-alg'])
+              when 'RSA-OAEP-SHA1'
+                unless master_key.is_a? OpenSSL::PKey::RSA
+                  raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with an AES key and the x-amz-wrap-alg is RSA-OAEP-SHA1.'
+                end
+                key, cek_alg = Utils.decrypt_rsa(master_key, decode64(envelope['x-amz-key-v2']))
+                raise Errors::DecryptionError unless cek_alg == envelope['x-amz-cek-alg']
+                key
+              when 'kms+context'
+                raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with a user provided key and the x-amz-wrap-alg is' \
+                    ' kms+context.  Please configure the client with the' \
+                    ' required kms_key_id'
+              else
+                raise ArgumentError, 'Unsupported wrap-alg: ' \
+                "#{envelope['x-amz-wrap-alg']}"
+              end
+            iv = decode64(envelope['x-amz-iv'])
+            Utils.aes_decryption_cipher(:GCM, key, iv)
+          end
         end
 
         private
@@ -58,7 +95,6 @@ module Aws
         def decode64(str)
           Base64.decode64(str)
         end
-
       end
     end
   end

data/lib/aws-sdk-s3/encryption/encrypt_handler.rb

@@ -39,8 +39,8 @@ module Aws
           context.params[:body] = IOEncrypter.new(cipher, io)
           context.params[:metadata] ||= {}
           context.params[:metadata]['x-amz-unencrypted-content-length'] = io.size
-          if md5 = context.params.delete(:content_md5)
-            context.params[:metadata]['x-amz-unencrypted-content-md5'] = md5
+          if context.params.delete(:content_md5)
+            warn('Setting content_md5 on client side encrypted objects is deprecated')
           end
           context.http_response.on_headers do
             context.params[:body].close
@@ -49,9 +49,9 @@ module Aws
 
         def apply_cse_user_agent(context)
           if context.config.user_agent_suffix.nil?
-            context.config.user_agent_suffix = 'CSE_V1'
-          elsif !context.config.user_agent_suffix.include? 'CSE_V1'
-            context.config.user_agent_suffix += ' CSE_V1'
+            context.config.user_agent_suffix = EC_USER_AGENT
+          elsif !context.config.user_agent_suffix.include? EC_USER_AGENT
+            context.config.user_agent_suffix += " #{EC_USER_AGENT}"
           end
         end
 

data/lib/aws-sdk-s3/encryption/io_decrypter.rb

@@ -9,9 +9,10 @@ module Aws
         # @param [OpenSSL::Cipher] cipher
         # @param [IO#write] io An IO-like object that responds to `#write`.
         def initialize(cipher, io)
-          @cipher = cipher.clone
+          @cipher = cipher
           # Ensure that IO is reset between retries
           @io = io.tap { |io| io.truncate(0) if io.respond_to?(:truncate) }
+          @cipher_buffer = String.new
         end
 
         # @return [#write]
@@ -19,17 +20,17 @@ module Aws
 
         def write(chunk)
           # decrypt and write
-          @io.write(@cipher.update(chunk))
+          if @cipher.method(:update).arity == 1
+            @io.write(@cipher.update(chunk))
+          else
+            @io.write(@cipher.update(chunk, @cipher_buffer))
+          end
         end
 
         def finalize
           @io.write(@cipher.final)
         end
 
-        def size
-          @io.size
-        end
-
       end
     end
   end

data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb

@@ -36,15 +36,36 @@ module Aws
 
         # @return [Cipher] Given an encryption envelope, returns a
         #   decryption cipher.
-        def decryption_cipher(envelope)
+        def decryption_cipher(envelope, options = {})
           encryption_context = Json.load(envelope['x-amz-matdesc'])
+          cek_alg = envelope['x-amz-cek-alg']
+
+          case envelope['x-amz-wrap-alg']
+          when 'kms'; # NO OP
+          when 'kms+context'
+            if cek_alg != encryption_context['aws:x-amz-cek-alg']
+              raise Errors::DecryptionError, 'Value of cek-alg from envelope'\
+              ' does not match the value in the encryption context'
+            end
+          when 'AES/GCM'
+            raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with a KMS key and the x-amz-wrap-alg is AES/GCM.'
+          when 'RSA-OAEP-SHA1'
+            raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with a KMS key and the x-amz-wrap-alg is RSA-OAEP-SHA1.'
+          else
+            raise ArgumentError, 'Unsupported wrap-alg: ' \
+                "#{envelope['x-amz-wrap-alg']}"
+          end
+
           key = @kms_client.decrypt(
             ciphertext_blob: decode64(envelope['x-amz-key-v2']),
-            encryption_context: encryption_context,
+            encryption_context: encryption_context
           ).plaintext
+
           iv = decode64(envelope['x-amz-iv'])
           block_mode =
-            case envelope['x-amz-cek-alg']
+            case cek_alg
             when 'AES/CBC/PKCS5Padding'
               :CBC
             when 'AES/CBC/PKCS7Padding'
@@ -61,6 +82,14 @@ module Aws
 
         private
 
+        def build_encryption_context(cek_alg, options = {})
+          kms_context = (options[:kms_encryption_context] || {})
+                        .each_with_object({}) { |(k, v), h| h[k.to_s] = v }
+          {
+            'aws:x-amz-cek-alg' => cek_alg
+          }.merge(kms_context)
+        end
+
         def encode64(str)
           Base64.encode64(str).split("\n") * ""
         end

data/lib/aws-sdk-s3/encryption/utils.rb

@@ -39,6 +39,29 @@ module Aws
             end
           end
 
+
+          def decrypt_aes_gcm(key, data, auth_data)
+            # data is iv (12B) + key + tag (16B)
+            buf = data.unpack('C*')
+            iv = buf[0,12].pack('C*') # iv will always be 12 bytes
+            tag = buf[-16, 16].pack('C*') # tag is 16 bytes
+            enc_key = buf[12, buf.size - (12+16)].pack('C*')
+            cipher = aes_cipher(:decrypt, :GCM, key, iv)
+            cipher.auth_tag = tag
+            cipher.auth_data = auth_data
+            cipher.update(enc_key) + cipher.final
+          end
+
+          # returns the decrypted data + auth_data
+          def decrypt_rsa(key, enc_data)
+            # Plaintext must be KeyLengthInBytes (1 Byte) + DataKey + AuthData
+            buf = key.private_decrypt(enc_data, OpenSSL::PKey::RSA::PKCS1_OAEP_PADDING).unpack('C*')
+            key_length = buf[0]
+            data = buf[1, key_length].pack('C*')
+            auth_data = buf[key_length+1, buf.length - key_length].pack('C*')
+            [data, auth_data]
+          end
+
           # @param [String] block_mode "CBC" or "ECB"
           # @param [OpenSSL::PKey::RSA, String, nil] key
           # @param [String, nil] iv The initialization vector