aws-sdk-s3 1.13.0 → 1.146.1

data/lib/aws-sdk-s3/encryption/client.rb

@@ -1,6 +1,14 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
 module Aws
   module S3
 
+    # [MAINTENANCE MODE] There is a new version of the Encryption Client.
+    # AWS strongly recommends upgrading to the {Aws::S3::EncryptionV2::Client},
+    # which provides updated data security best practices.
+    # See documentation for {Aws::S3::EncryptionV2::Client}.
     # Provides an encryption client that encrypts and decrypts data client-side,
     # storing the encrypted data in Amazon S3.
     #
@@ -112,7 +120,7 @@ module Aws
     #       attr_reader :encryption_materials
     #
     #       def key_for(matdesc)
-    #         key_name = JSON.load(matdesc)['key']
+    #         key_name = JSON.parse(matdesc)['key']
     #         if key = @keys[key_name]
     #           key
     #         else
@@ -178,8 +186,10 @@ module Aws
       class Client
 
         extend Deprecations
+        extend Forwardable
+        def_delegators :@client, :config, :delete_object, :head_object, :build_request
 
-        # Creates a new encryption client. You must provide on of the following
+        # Creates a new encryption client. You must provide one of the following
         # options:
         #
         # * `:encryption_key`
@@ -223,6 +233,13 @@ module Aws
           @envelope_location = extract_location(options)
           @instruction_file_suffix = extract_suffix(options)
         end
+        deprecated :initialize,
+                   message:
+                     '[MAINTENANCE MODE] This version of the S3 Encryption client is currently in maintenance mode. ' \
+	                     'AWS strongly recommends upgrading to the Aws::S3::EncryptionV2::Client, ' \
+	                     'which provides updated data security best practices. ' \
+	                     'See documentation for Aws::S3::EncryptionV2::Client.'
+
 
         # @return [S3::Client]
         attr_reader :client
@@ -253,7 +270,9 @@ module Aws
             envelope_location: @envelope_location,
             instruction_file_suffix: @instruction_file_suffix,
           }
-          req.send_request
+          Aws::Plugins::UserAgent.feature('S3CryptoV1n') do
+            req.send_request
+          end
         end
 
         # Gets an object from Amazon S3, decrypting  data locally.
@@ -281,7 +300,9 @@ module Aws
             envelope_location: envelope_location,
             instruction_file_suffix: instruction_file_suffix,
           }
-          req.send_request(target: block)
+          Aws::Plugins::UserAgent.feature('S3CryptoV1n') do
+            req.send_request(target: block)
+          end
         end
 
         private
@@ -327,7 +348,7 @@ module Aws
           elsif options[:encryption_key]
             DefaultKeyProvider.new(options)
           else
-            msg = "you must pass a :kms_key_id, :key_provider, or :encryption_key"
+            msg = 'you must pass a :kms_key_id, :key_provider, or :encryption_key'
             raise ArgumentError, msg
           end
         end
@@ -347,8 +368,8 @@ module Aws
           if [:metadata, :instruction_file].include?(location)
             location
           else
-            msg = ":envelope_location must be :metadata or :instruction_file "
-            msg << "got #{location.inspect}"
+            msg = ':envelope_location must be :metadata or :instruction_file '\
+                  "got #{location.inspect}"
             raise ArgumentError, msg
           end
         end
@@ -358,7 +379,7 @@ module Aws
           if String === suffix
             suffix
           else
-            msg = ":instruction_file_suffix must be a String"
+            msg = ':instruction_file_suffix must be a String'
             raise ArgumentError, msg
           end
         end

data/lib/aws-sdk-s3/encryption/decrypt_handler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 
 module Aws
@@ -5,6 +7,7 @@ module Aws
     module Encryption
       # @api private
       class DecryptHandler < Seahorse::Client::Handler
+        @@warned_response_target_proc = false
 
         V1_ENVELOPE_KEYS = %w(
           x-amz-key
@@ -20,7 +23,17 @@ module Aws
           x-amz-matdesc
         )
 
-        POSSIBLE_ENVELOPE_KEYS = (V1_ENVELOPE_KEYS + V2_ENVELOPE_KEYS).uniq
+        V2_OPTIONAL_KEYS = %w(x-amz-tag-len)
+
+        POSSIBLE_ENVELOPE_KEYS = (V1_ENVELOPE_KEYS +
+          V2_ENVELOPE_KEYS + V2_OPTIONAL_KEYS).uniq
+
+        POSSIBLE_WRAPPING_FORMATS = %w(
+          AES/GCM
+          kms
+          kms+context
+          RSA-OAEP-SHA1
+        )
 
         POSSIBLE_ENCRYPTION_FORMATS = %w(
           AES/GCM/NoPadding
@@ -28,8 +41,21 @@ module Aws
           AES/CBC/PKCS7Padding
         )
 
+        AUTH_REQUIRED_CEK_ALGS = %w(AES/GCM/NoPadding)
+
         def call(context)
           attach_http_event_listeners(context)
+          apply_cse_user_agent(context)
+
+          if context[:response_target].is_a?(Proc) && !@@warned_response_target_proc
+            @@warned_response_target_proc = true
+            warn(':response_target is a Proc, or a block was provided. ' \
+              'Read the entire object to the ' \
+              'end before you start using the decrypted data. This is to ' \
+              'verify that the object has not been modified since it ' \
+              'was encrypted.')
+          end
+
           @handler.call(context)
         end
 
@@ -38,9 +64,9 @@ module Aws
         def attach_http_event_listeners(context)
 
           context.http_response.on_headers(200) do
-            cipher = decryption_cipher(context)
-            decrypter = body_contains_auth_tag?(context) ?
-              authenticated_decrypter(context, cipher) :
+            cipher, envelope = decryption_cipher(context)
+            decrypter = body_contains_auth_tag?(envelope) ?
+              authenticated_decrypter(context, cipher, envelope) :
               IODecrypter.new(cipher, context.http_response.body)
             context.http_response.body = decrypter
           end
@@ -60,8 +86,13 @@ module Aws
         end
 
         def decryption_cipher(context)
-          if envelope = get_encryption_envelope(context)
-            context[:encryption][:cipher_provider].decryption_cipher(envelope)
+          if (envelope = get_encryption_envelope(context))
+            cipher = context[:encryption][:cipher_provider]
+                     .decryption_cipher(
+                       envelope,
+                       context[:encryption]
+                     )
+            [cipher, envelope]
           else
             raise Errors::DecryptionError, "unable to locate encryption envelope"
           end
@@ -97,13 +128,12 @@ module Aws
         end
 
         def extract_envelope(hash)
+          return nil unless hash
           return v1_envelope(hash) if hash.key?('x-amz-key')
           return v2_envelope(hash) if hash.key?('x-amz-key-v2')
           if hash.keys.any? { |key| key.match(/^x-amz-key-(.+)$/) }
             msg = "unsupported envelope encryption version #{$1}"
             raise Errors::DecryptionError, msg
-          else
-            nil # no envelope found
           end
         end
 
@@ -117,39 +147,27 @@ module Aws
             msg = "unsupported content encrypting key (cek) format: #{alg}"
             raise Errors::DecryptionError, msg
           end
-          unless envelope['x-amz-wrap-alg'] == 'kms'
-            # possible to support
-            #   RSA/ECB/OAEPWithSHA-256AndMGF1Padding
+          unless POSSIBLE_WRAPPING_FORMATS.include? envelope['x-amz-wrap-alg']
             alg = envelope['x-amz-wrap-alg'].inspect
             msg = "unsupported key wrapping algorithm: #{alg}"
             raise Errors::DecryptionError, msg
           end
-          unless V2_ENVELOPE_KEYS.sort == envelope.keys.sort
+          unless (missing_keys = V2_ENVELOPE_KEYS - envelope.keys).empty?
             msg = "incomplete v2 encryption envelope:\n"
-            msg += "  expected: #{V2_ENVELOPE_KEYS.join(',')}\n"
-            msg += "  got: #{envelope_keys.join(', ')}"
+            msg += "  missing: #{missing_keys.join(',')}\n"
             raise Errors::DecryptionError, msg
           end
           envelope
         end
 
-        # When the x-amz-meta-x-amz-tag-len header is present, it indicates
-        # that the body of this object has a trailing auth tag. The header
-        # indicates the length of that tag.
-        #
         # This method fetches the tag from the end of the object by
         # making a GET Object w/range request. This auth tag is used
         # to initialize the cipher, and the decrypter truncates the
         # auth tag from the body when writing the final bytes.
-        def authenticated_decrypter(context, cipher)
-          if RUBY_VERSION.match(/1.9/)
-            raise "authenticated decryption not supported by OpeenSSL in Ruby version ~> 1.9"
-            raise Aws::Errors::NonSupportedRubyVersionError, msg
-          end
+        def authenticated_decrypter(context, cipher, envelope)
           http_resp = context.http_response
           content_length = http_resp.headers['content-length'].to_i
-          auth_tag_length = http_resp.headers['x-amz-meta-x-amz-tag-len']
-          auth_tag_length = auth_tag_length.to_i / 8
+          auth_tag_length = auth_tag_length(envelope)
 
           auth_tag = context.client.get_object(
             bucket: context.params[:bucket],
@@ -161,16 +179,40 @@ module Aws
           cipher.auth_data = ''
 
           # The encrypted object contains both the cipher text
-          # plus a trailing auth tag. This decrypter will the body
-          # expect for the trailing auth tag.
+          # plus a trailing auth tag.
           IOAuthDecrypter.new(
             io: http_resp.body,
             encrypted_content_length: content_length - auth_tag_length,
             cipher: cipher)
         end
 
-        def body_contains_auth_tag?(context)
-          context.http_response.headers['x-amz-meta-x-amz-tag-len']
+        def body_contains_auth_tag?(envelope)
+          AUTH_REQUIRED_CEK_ALGS.include?(envelope['x-amz-cek-alg'])
+        end
+
+        # Determine the auth tag length from the algorithm
+        # Validate it against the value provided in the x-amz-tag-len
+        # Return the tag length in bytes
+        def auth_tag_length(envelope)
+          tag_length =
+            case envelope['x-amz-cek-alg']
+            when 'AES/GCM/NoPadding' then AES_GCM_TAG_LEN_BYTES
+            else
+              raise ArgumentError, 'Unsupported cek-alg: ' \
+                "#{envelope['x-amz-cek-alg']}"
+            end
+          if (tag_length * 8) != envelope['x-amz-tag-len'].to_i
+            raise Errors::DecryptionError, 'x-amz-tag-len does not match expected'
+          end
+          tag_length
+        end
+
+        def apply_cse_user_agent(context)
+          if context.config.user_agent_suffix.nil?
+            context.config.user_agent_suffix = EC_USER_AGENT
+          elsif !context.config.user_agent_suffix.include? EC_USER_AGENT
+            context.config.user_agent_suffix += " #{EC_USER_AGENT}"
+          end
         end
 
       end

data/lib/aws-sdk-s3/encryption/default_cipher_provider.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 
 module Aws
@@ -24,11 +26,48 @@ module Aws
 
         # @return [Cipher] Given an encryption envelope, returns a
         #   decryption cipher.
-        def decryption_cipher(envelope)
+        def decryption_cipher(envelope, options = {})
           master_key = @key_provider.key_for(envelope['x-amz-matdesc'])
-          key = Utils.decrypt(master_key, decode64(envelope['x-amz-key']))
-          iv = decode64(envelope['x-amz-iv'])
-          Utils.aes_decryption_cipher(:CBC, key, iv)
+          if envelope.key? 'x-amz-key'
+            # Support for decryption of legacy objects
+            key = Utils.decrypt(master_key, decode64(envelope['x-amz-key']))
+            iv = decode64(envelope['x-amz-iv'])
+            Utils.aes_decryption_cipher(:CBC, key, iv)
+          else
+            if envelope['x-amz-cek-alg'] != 'AES/GCM/NoPadding'
+              raise ArgumentError, 'Unsupported cek-alg: ' \
+                "#{envelope['x-amz-cek-alg']}"
+            end
+            key =
+              case envelope['x-amz-wrap-alg']
+              when 'AES/GCM'
+                if master_key.is_a? OpenSSL::PKey::RSA
+                  raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with an RSA key and the x-amz-wrap-alg is AES/GCM.'
+                end
+                Utils.decrypt_aes_gcm(master_key,
+                                      decode64(envelope['x-amz-key-v2']),
+                                      envelope['x-amz-cek-alg'])
+              when 'RSA-OAEP-SHA1'
+                unless master_key.is_a? OpenSSL::PKey::RSA
+                  raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with an AES key and the x-amz-wrap-alg is RSA-OAEP-SHA1.'
+                end
+                key, cek_alg = Utils.decrypt_rsa(master_key, decode64(envelope['x-amz-key-v2']))
+                raise Errors::DecryptionError unless cek_alg == envelope['x-amz-cek-alg']
+                key
+              when 'kms+context'
+                raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with a user provided key and the x-amz-wrap-alg is' \
+                    ' kms+context.  Please configure the client with the' \
+                    ' required kms_key_id'
+              else
+                raise ArgumentError, 'Unsupported wrap-alg: ' \
+                "#{envelope['x-amz-wrap-alg']}"
+              end
+            iv = decode64(envelope['x-amz-iv'])
+            Utils.aes_decryption_cipher(:GCM, key, iv)
+          end
         end
 
         private
@@ -56,7 +95,6 @@ module Aws
         def decode64(str)
           Base64.decode64(str)
         end
-
       end
     end
   end

data/lib/aws-sdk-s3/encryption/default_key_provider.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     module Encryption

data/lib/aws-sdk-s3/encryption/encrypt_handler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 
 module Aws
@@ -10,6 +12,7 @@ module Aws
           envelope, cipher = context[:encryption][:cipher_provider].encryption_cipher
           apply_encryption_envelope(context, envelope, cipher)
           apply_encryption_cipher(context, cipher)
+          apply_cse_user_agent(context)
           @handler.call(context)
         end
 
@@ -36,14 +39,22 @@ module Aws
           context.params[:body] = IOEncrypter.new(cipher, io)
           context.params[:metadata] ||= {}
           context.params[:metadata]['x-amz-unencrypted-content-length'] = io.size
-          if md5 = context.params.delete(:content_md5)
-            context.params[:metadata]['x-amz-unencrypted-content-md5'] = md5
+          if context.params.delete(:content_md5)
+            warn('Setting content_md5 on client side encrypted objects is deprecated')
           end
           context.http_response.on_headers do
             context.params[:body].close
           end
         end
 
+        def apply_cse_user_agent(context)
+          if context.config.user_agent_suffix.nil?
+            context.config.user_agent_suffix = EC_USER_AGENT
+          elsif !context.config.user_agent_suffix.include? EC_USER_AGENT
+            context.config.user_agent_suffix += " #{EC_USER_AGENT}"
+          end
+        end
+
       end
     end
   end

data/lib/aws-sdk-s3/encryption/errors.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     module Encryption

data/lib/aws-sdk-s3/encryption/io_auth_decrypter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     module Encryption
@@ -22,8 +24,10 @@ module Aws
 
         def write(chunk)
           chunk = truncate_chunk(chunk)
-          @bytes_written += chunk.bytesize
-          @decrypter.write(chunk)
+          if chunk.bytesize > 0
+            @bytes_written += chunk.bytesize
+            @decrypter.write(chunk)
+          end
         end
 
         def finalize
@@ -39,8 +43,12 @@ module Aws
         def truncate_chunk(chunk)
           if chunk.bytesize + @bytes_written <= @max_bytes
             chunk
-          else
+          elsif @bytes_written < @max_bytes
             chunk[0..(@max_bytes - @bytes_written - 1)]
+          else
+            # If the tag was sent over after the full body has been read,
+            # we don't want to accidentally append it.
+            ""
           end
         end
 

data/lib/aws-sdk-s3/encryption/io_decrypter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     module Encryption
@@ -7,8 +9,10 @@ module Aws
         # @param [OpenSSL::Cipher] cipher
         # @param [IO#write] io An IO-like object that responds to `#write`.
         def initialize(cipher, io)
-          @cipher = cipher.clone
-          @io = io
+          @cipher = cipher
+          # Ensure that IO is reset between retries
+          @io = io.tap { |io| io.truncate(0) if io.respond_to?(:truncate) }
+          @cipher_buffer = String.new
         end
 
         # @return [#write]
@@ -16,7 +20,11 @@ module Aws
 
         def write(chunk)
           # decrypt and write
-          @io.write(@cipher.update(chunk))
+          if @cipher.method(:update).arity == 1
+            @io.write(@cipher.update(chunk))
+          else
+            @io.write(@cipher.update(chunk, @cipher_buffer))
+          end
         end
 
         def finalize

data/lib/aws-sdk-s3/encryption/io_encrypter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'stringio'
 require 'tempfile'
 

data/lib/aws-sdk-s3/encryption/key_provider.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     module Encryption

data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 
 module Aws
@@ -15,11 +17,13 @@ module Aws
         #   envelope and encryption cipher.
         def encryption_cipher
           encryption_context = { "kms_cmk_id" => @kms_key_id }
-          key_data = @kms_client.generate_data_key(
-            key_id: @kms_key_id,
-            encryption_context: encryption_context,
-            key_spec: 'AES_256',
-          )
+          key_data = Aws::Plugins::UserAgent.feature('S3CryptoV1n') do
+            @kms_client.generate_data_key(
+              key_id: @kms_key_id,
+              encryption_context: encryption_context,
+              key_spec: 'AES_256'
+            )
+          end
           cipher = Utils.aes_encryption_cipher(:CBC)
           cipher.key = key_data.plaintext
           envelope = {
@@ -34,15 +38,38 @@ module Aws
 
         # @return [Cipher] Given an encryption envelope, returns a
         #   decryption cipher.
-        def decryption_cipher(envelope)
+        def decryption_cipher(envelope, options = {})
           encryption_context = Json.load(envelope['x-amz-matdesc'])
-          key = @kms_client.decrypt(
-            ciphertext_blob: decode64(envelope['x-amz-key-v2']),
-            encryption_context: encryption_context,
-          ).plaintext
+          cek_alg = envelope['x-amz-cek-alg']
+
+          case envelope['x-amz-wrap-alg']
+          when 'kms'; # NO OP
+          when 'kms+context'
+            if cek_alg != encryption_context['aws:x-amz-cek-alg']
+              raise Errors::DecryptionError, 'Value of cek-alg from envelope'\
+              ' does not match the value in the encryption context'
+            end
+          when 'AES/GCM'
+            raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with a KMS key and the x-amz-wrap-alg is AES/GCM.'
+          when 'RSA-OAEP-SHA1'
+            raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with a KMS key and the x-amz-wrap-alg is RSA-OAEP-SHA1.'
+          else
+            raise ArgumentError, 'Unsupported wrap-alg: ' \
+                "#{envelope['x-amz-wrap-alg']}"
+          end
+
+          key = Aws::Plugins::UserAgent.feature('S3CryptoV1n') do
+            @kms_client.decrypt(
+              ciphertext_blob: decode64(envelope['x-amz-key-v2']),
+              encryption_context: encryption_context
+            ).plaintext
+          end
+
           iv = decode64(envelope['x-amz-iv'])
           block_mode =
-            case envelope['x-amz-cek-alg']
+            case cek_alg
             when 'AES/CBC/PKCS5Padding'
               :CBC
             when 'AES/CBC/PKCS7Padding'
@@ -59,6 +86,14 @@ module Aws
 
         private
 
+        def build_encryption_context(cek_alg, options = {})
+          kms_context = (options[:kms_encryption_context] || {})
+                        .each_with_object({}) { |(k, v), h| h[k.to_s] = v }
+          {
+            'aws:x-amz-cek-alg' => cek_alg
+          }.merge(kms_context)
+        end
+
         def encode64(str)
           Base64.encode64(str).split("\n") * ""
         end

data/lib/aws-sdk-s3/encryption/materials.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'base64'
 
 module Aws
@@ -32,14 +34,14 @@ module Aws
             if [32, 24, 16].include?(key.bytesize)
               key
             else
-              msg = "invalid key, symmetric key required to be 16, 24, or "
-              msg << "32 bytes in length, saw length " + key.bytesize.to_s
+              msg = 'invalid key, symmetric key required to be 16, 24, or '\
+                    '32 bytes in length, saw length ' + key.bytesize.to_s
               raise ArgumentError, msg
             end
           else
-            msg = "invalid encryption key, expected an OpenSSL::PKey::RSA key "
-            msg << "(for asymmetric encryption) or a String (for symmetric "
-            msg << "encryption)."
+            msg = 'invalid encryption key, expected an OpenSSL::PKey::RSA key '\
+                  '(for asymmetric encryption) or a String (for symmetric '\
+                  'encryption).'
             raise ArgumentError, msg
           end
         end
@@ -48,7 +50,7 @@ module Aws
           Json.load(description)
           description
         rescue Json::ParseError, EncodingError
-          msg = "expected description to be a valid JSON document string"
+          msg = 'expected description to be a valid JSON document string'
           raise ArgumentError, msg
         end
 

data/lib/aws-sdk-s3/encryption/utils.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'openssl'
 
 module Aws
@@ -37,6 +39,29 @@ module Aws
             end
           end
 
+
+          def decrypt_aes_gcm(key, data, auth_data)
+            # data is iv (12B) + key + tag (16B)
+            buf = data.unpack('C*')
+            iv = buf[0,12].pack('C*') # iv will always be 12 bytes
+            tag = buf[-16, 16].pack('C*') # tag is 16 bytes
+            enc_key = buf[12, buf.size - (12+16)].pack('C*')
+            cipher = aes_cipher(:decrypt, :GCM, key, iv)
+            cipher.auth_tag = tag
+            cipher.auth_data = auth_data
+            cipher.update(enc_key) + cipher.final
+          end
+
+          # returns the decrypted data + auth_data
+          def decrypt_rsa(key, enc_data)
+            # Plaintext must be KeyLengthInBytes (1 Byte) + DataKey + AuthData
+            buf = key.private_decrypt(enc_data, OpenSSL::PKey::RSA::PKCS1_OAEP_PADDING).unpack('C*')
+            key_length = buf[0]
+            data = buf[1, key_length].pack('C*')
+            auth_data = buf[key_length+1, buf.length - key_length].pack('C*')
+            [data, auth_data]
+          end
+
           # @param [String] block_mode "CBC" or "ECB"
           # @param [OpenSSL::PKey::RSA, String, nil] key
           # @param [String, nil] iv The initialization vector

data/lib/aws-sdk-s3/encryption.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'aws-sdk-s3/encryption/client'
 require 'aws-sdk-s3/encryption/decrypt_handler'
 require 'aws-sdk-s3/encryption/default_cipher_provider'
@@ -15,5 +17,7 @@ require 'aws-sdk-s3/encryption/default_key_provider'
 module Aws
   module S3
     module Encryption; end
+    AES_GCM_TAG_LEN_BYTES = 16
+    EC_USER_AGENT = 'S3CryptoV1n'
   end
 end