aws-sdk-s3 1.31.0 → 1.111.1

data/lib/aws-sdk-s3/encryptionV2/decrypt_handler.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module Aws
+  module S3
+    module EncryptionV2
+      # @api private
+      class DecryptHandler < Seahorse::Client::Handler
+        @@warned_response_target_proc = false
+
+        V1_ENVELOPE_KEYS = %w(
+          x-amz-key
+          x-amz-iv
+          x-amz-matdesc
+        )
+
+        V2_ENVELOPE_KEYS = %w(
+          x-amz-key-v2
+          x-amz-iv
+          x-amz-cek-alg
+          x-amz-wrap-alg
+          x-amz-matdesc
+        )
+
+        V2_OPTIONAL_KEYS = %w(x-amz-tag-len)
+
+        POSSIBLE_ENVELOPE_KEYS = (V1_ENVELOPE_KEYS +
+          V2_ENVELOPE_KEYS + V2_OPTIONAL_KEYS).uniq
+
+        POSSIBLE_WRAPPING_FORMATS = %w(
+          AES/GCM
+          kms
+          kms+context
+          RSA-OAEP-SHA1
+        )
+
+        POSSIBLE_ENCRYPTION_FORMATS = %w(
+          AES/GCM/NoPadding
+          AES/CBC/PKCS5Padding
+          AES/CBC/PKCS7Padding
+        )
+
+        AUTH_REQUIRED_CEK_ALGS = %w(AES/GCM/NoPadding)
+
+        def call(context)
+          attach_http_event_listeners(context)
+          apply_cse_user_agent(context)
+
+          if context[:response_target].is_a?(Proc) && !@@warned_response_target_proc
+            @@warned_response_target_proc = true
+            warn(':response_target is a Proc, or a block was provided. ' \
+              'Read the entire object to the ' \
+              'end before you start using the decrypted data. This is to ' \
+              'verify that the object has not been modified since it ' \
+              'was encrypted.')
+
+          end
+
+          @handler.call(context)
+        end
+
+        private
+
+        def attach_http_event_listeners(context)
+
+          context.http_response.on_headers(200) do
+            cipher, envelope = decryption_cipher(context)
+            decrypter = body_contains_auth_tag?(envelope) ?
+              authenticated_decrypter(context, cipher, envelope) :
+              IODecrypter.new(cipher, context.http_response.body)
+            context.http_response.body = decrypter
+          end
+
+          context.http_response.on_success(200) do
+            decrypter = context.http_response.body
+            decrypter.finalize
+            decrypter.io.rewind if decrypter.io.respond_to?(:rewind)
+            context.http_response.body = decrypter.io
+          end
+
+          context.http_response.on_error do
+            if context.http_response.body.respond_to?(:io)
+              context.http_response.body = context.http_response.body.io
+            end
+          end
+        end
+
+        def decryption_cipher(context)
+          if (envelope = get_encryption_envelope(context))
+            cipher = context[:encryption][:cipher_provider]
+             .decryption_cipher(
+               envelope,
+               context[:encryption]
+             )
+            [cipher, envelope]
+          else
+            raise Errors::DecryptionError, "unable to locate encryption envelope"
+          end
+        end
+
+        def get_encryption_envelope(context)
+          if context[:encryption][:envelope_location] == :metadata
+            envelope_from_metadata(context) || envelope_from_instr_file(context)
+          else
+            envelope_from_instr_file(context) || envelope_from_metadata(context)
+          end
+        end
+
+        def envelope_from_metadata(context)
+          possible_envelope = {}
+          POSSIBLE_ENVELOPE_KEYS.each do |suffix|
+            if value = context.http_response.headers["x-amz-meta-#{suffix}"]
+              possible_envelope[suffix] = value
+            end
+          end
+          extract_envelope(possible_envelope)
+        end
+
+        def envelope_from_instr_file(context)
+          suffix = context[:encryption][:instruction_file_suffix]
+          possible_envelope = Json.load(context.client.get_object(
+            bucket: context.params[:bucket],
+            key: context.params[:key] + suffix
+          ).body.read)
+          extract_envelope(possible_envelope)
+        rescue S3::Errors::ServiceError, Json::ParseError
+          nil
+        end
+
+        def extract_envelope(hash)
+          return nil unless hash
+          return v1_envelope(hash) if hash.key?('x-amz-key')
+          return v2_envelope(hash) if hash.key?('x-amz-key-v2')
+          if hash.keys.any? { |key| key.match(/^x-amz-key-(.+)$/) }
+            msg = "unsupported envelope encryption version #{$1}"
+            raise Errors::DecryptionError, msg
+          end
+        end
+
+        def v1_envelope(envelope)
+          envelope
+        end
+
+        def v2_envelope(envelope)
+          unless POSSIBLE_ENCRYPTION_FORMATS.include? envelope['x-amz-cek-alg']
+            alg = envelope['x-amz-cek-alg'].inspect
+            msg = "unsupported content encrypting key (cek) format: #{alg}"
+            raise Errors::DecryptionError, msg
+          end
+          unless POSSIBLE_WRAPPING_FORMATS.include? envelope['x-amz-wrap-alg']
+            alg = envelope['x-amz-wrap-alg'].inspect
+            msg = "unsupported key wrapping algorithm: #{alg}"
+            raise Errors::DecryptionError, msg
+          end
+          unless (missing_keys = V2_ENVELOPE_KEYS - envelope.keys).empty?
+            msg = "incomplete v2 encryption envelope:\n"
+            msg += "  missing: #{missing_keys.join(',')}\n"
+            raise Errors::DecryptionError, msg
+          end
+          envelope
+        end
+
+        # This method fetches the tag from the end of the object by
+        # making a GET Object w/range request. This auth tag is used
+        # to initialize the cipher, and the decrypter truncates the
+        # auth tag from the body when writing the final bytes.
+        def authenticated_decrypter(context, cipher, envelope)
+          http_resp = context.http_response
+          content_length = http_resp.headers['content-length'].to_i
+          auth_tag_length = auth_tag_length(envelope)
+
+          auth_tag = context.client.get_object(
+            bucket: context.params[:bucket],
+            key: context.params[:key],
+            range: "bytes=-#{auth_tag_length}"
+          ).body.read
+
+          cipher.auth_tag = auth_tag
+          cipher.auth_data = ''
+
+          # The encrypted object contains both the cipher text
+          # plus a trailing auth tag.
+          IOAuthDecrypter.new(
+            io: http_resp.body,
+            encrypted_content_length: content_length - auth_tag_length,
+            cipher: cipher)
+        end
+
+        def body_contains_auth_tag?(envelope)
+          AUTH_REQUIRED_CEK_ALGS.include?(envelope['x-amz-cek-alg'])
+        end
+
+        # Determine the auth tag length from the algorithm
+        # Validate it against the value provided in the x-amz-tag-len
+        # Return the tag length in bytes
+        def auth_tag_length(envelope)
+          tag_length =
+            case envelope['x-amz-cek-alg']
+            when 'AES/GCM/NoPadding' then AES_GCM_TAG_LEN_BYTES
+            else
+              raise ArgumentError, 'Unsupported cek-alg: ' \
+                "#{envelope['x-amz-cek-alg']}"
+            end
+          if (tag_length * 8) != envelope['x-amz-tag-len'].to_i
+            raise Errors::DecryptionError, 'x-amz-tag-len does not match expected'
+          end
+          tag_length
+        end
+
+        def apply_cse_user_agent(context)
+          if context.config.user_agent_suffix.nil?
+            context.config.user_agent_suffix = EC_USER_AGENT
+          elsif !context.config.user_agent_suffix.include? EC_USER_AGENT
+            context.config.user_agent_suffix += " #{EC_USER_AGENT}"
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryptionV2/default_cipher_provider.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module Aws
+  module S3
+    module EncryptionV2
+      # @api private
+      class DefaultCipherProvider
+
+        def initialize(options = {})
+          @key_provider = options[:key_provider]
+          @key_wrap_schema = validate_key_wrap(
+            options[:key_wrap_schema],
+            @key_provider.encryption_materials.key
+          )
+          @content_encryption_schema = validate_cek(
+            options[:content_encryption_schema]
+          )
+        end
+
+        # @return [Array<Hash,Cipher>] Creates an returns a new encryption
+        #   envelope and encryption cipher.
+        def encryption_cipher(options = {})
+          validate_options(options)
+          cipher = Utils.aes_encryption_cipher(:GCM)
+          if @key_provider.encryption_materials.key.is_a? OpenSSL::PKey::RSA
+            enc_key = encode64(
+              encrypt_rsa(envelope_key(cipher), @content_encryption_schema)
+            )
+          else
+            enc_key = encode64(
+              encrypt_aes_gcm(envelope_key(cipher), @content_encryption_schema)
+            )
+          end
+          envelope = {
+            'x-amz-key-v2' => enc_key,
+            'x-amz-cek-alg' => @content_encryption_schema,
+            'x-amz-tag-len' => (AES_GCM_TAG_LEN_BYTES * 8).to_s,
+            'x-amz-wrap-alg' => @key_wrap_schema,
+            'x-amz-iv' => encode64(envelope_iv(cipher)),
+            'x-amz-matdesc' => materials_description
+          }
+          cipher.auth_data = '' # auth_data must be set after key and iv
+          [envelope, cipher]
+        end
+
+        # @return [Cipher] Given an encryption envelope, returns a
+        #   decryption cipher.
+        def decryption_cipher(envelope, options = {})
+          validate_options(options)
+          master_key = @key_provider.key_for(envelope['x-amz-matdesc'])
+          if envelope.key? 'x-amz-key'
+            unless options[:security_profile] == :v2_and_legacy
+              raise Errors::LegacyDecryptionError
+            end
+            # Support for decryption of legacy objects
+            key = Utils.decrypt(master_key, decode64(envelope['x-amz-key']))
+            iv = decode64(envelope['x-amz-iv'])
+            Utils.aes_decryption_cipher(:CBC, key, iv)
+          else
+            if envelope['x-amz-cek-alg'] != 'AES/GCM/NoPadding'
+              raise ArgumentError, 'Unsupported cek-alg: ' \
+                "#{envelope['x-amz-cek-alg']}"
+            end
+            key =
+              case envelope['x-amz-wrap-alg']
+              when 'AES/GCM'
+                if master_key.is_a? OpenSSL::PKey::RSA
+                  raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with an RSA key and the x-amz-wrap-alg is AES/GCM.'
+                end
+                Utils.decrypt_aes_gcm(master_key,
+                                    decode64(envelope['x-amz-key-v2']),
+                                    envelope['x-amz-cek-alg'])
+              when 'RSA-OAEP-SHA1'
+                unless master_key.is_a? OpenSSL::PKey::RSA
+                  raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with an AES key and the x-amz-wrap-alg is RSA-OAEP-SHA1.'
+                end
+                key, cek_alg = Utils.decrypt_rsa(master_key, decode64(envelope['x-amz-key-v2']))
+                raise Errors::CEKAlgMismatchError unless cek_alg == envelope['x-amz-cek-alg']
+                key
+              when 'kms+context'
+                raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with a user provided key and the x-amz-wrap-alg is' \
+                    ' kms+context.  Please configure the client with the' \
+                    ' required kms_key_id'
+              else
+                raise ArgumentError, 'Unsupported wrap-alg: ' \
+                      "#{envelope['x-amz-wrap-alg']}"
+              end
+            iv = decode64(envelope['x-amz-iv'])
+            Utils.aes_decryption_cipher(:GCM, key, iv)
+          end
+        end
+
+        private
+
+        # Validate that the key_wrap_schema
+        # is valid, supported and matches the provided key.
+        # Returns the string version for the x-amz-key-wrap-alg
+        def validate_key_wrap(key_wrap_schema, key)
+          if key.is_a? OpenSSL::PKey::RSA
+            unless key_wrap_schema == :rsa_oaep_sha1
+              raise ArgumentError, ':key_wrap_schema must be set to :rsa_oaep_sha1 for RSA keys.'
+            end
+          else
+            unless key_wrap_schema == :aes_gcm
+              raise ArgumentError, ':key_wrap_schema must be set to :aes_gcm for AES keys.'
+            end
+          end
+
+          case key_wrap_schema
+          when :rsa_oaep_sha1 then 'RSA-OAEP-SHA1'
+          when :aes_gcm then 'AES/GCM'
+          when :kms_context
+            raise ArgumentError, 'A kms_key_id is required when using :kms_context.'
+          else
+            raise ArgumentError, "Unsupported key_wrap_schema: #{key_wrap_schema}"
+          end
+        end
+
+        def validate_cek(content_encryption_schema)
+          case content_encryption_schema
+          when :aes_gcm_no_padding
+            "AES/GCM/NoPadding"
+          else
+            raise ArgumentError, "Unsupported content_encryption_schema: #{content_encryption_schema}"
+          end
+        end
+
+        def envelope_key(cipher)
+          cipher.key = cipher.random_key
+        end
+
+        def envelope_iv(cipher)
+          cipher.iv = cipher.random_iv
+        end
+
+        def encrypt_aes_gcm(data, auth_data)
+          Utils.encrypt_aes_gcm(@key_provider.encryption_materials.key, data, auth_data)
+        end
+
+        def encrypt_rsa(data, auth_data)
+          Utils.encrypt_rsa(@key_provider.encryption_materials.key, data, auth_data)
+        end
+
+        def materials_description
+          @key_provider.encryption_materials.description
+        end
+
+        def encode64(str)
+          Base64.encode64(str).split("\n") * ''
+        end
+
+        def decode64(str)
+          Base64.decode64(str)
+        end
+
+        def validate_options(options)
+          if !options[:kms_encryption_context].nil?
+            raise ArgumentError, 'Cannot provide :kms_encryption_context ' \
+            'with non KMS client.'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryptionV2/default_key_provider.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module EncryptionV2
+
+      # The default key provider is constructed with a single key
+      # that is used for both encryption and decryption, ignoring
+      # the possible per-object envelope encryption materials description.
+      # @api private
+      class DefaultKeyProvider
+
+        include KeyProvider
+
+        # @option options [required, OpenSSL::PKey::RSA, String] :encryption_key
+        #   The master key to use for encrypting objects.
+        # @option options [String<JSON>] :materials_description ('{}')
+        #   A description of the encryption key.
+        def initialize(options = {})
+          @encryption_materials = Materials.new(
+            key: options[:encryption_key],
+            description: options[:materials_description] || '{}'
+          )
+        end
+
+        # @return [Materials]
+        def encryption_materials
+          @encryption_materials
+        end
+
+        # @param [String<JSON>] materials_description
+        # @return Returns the key given in the constructor.
+        def key_for(materials_description)
+          @encryption_materials.key
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryptionV2/encrypt_handler.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module Aws
+  module S3
+    module EncryptionV2
+      # @api private
+      class EncryptHandler < Seahorse::Client::Handler
+
+        def call(context)
+          envelope, cipher = context[:encryption][:cipher_provider]
+           .encryption_cipher(
+             kms_encryption_context: context[:encryption][:kms_encryption_context]
+           )
+          context[:encryption][:cipher] = cipher
+          apply_encryption_envelope(context, envelope)
+          apply_encryption_cipher(context, cipher)
+          apply_cse_user_agent(context)
+          @handler.call(context)
+        end
+
+        private
+
+        def apply_encryption_envelope(context, envelope)
+          if context[:encryption][:envelope_location] == :instruction_file
+            suffix = context[:encryption][:instruction_file_suffix]
+            context.client.put_object(
+              bucket: context.params[:bucket],
+              key: context.params[:key] + suffix,
+              body: Json.dump(envelope)
+            )
+          else # :metadata
+            context.params[:metadata] ||= {}
+            context.params[:metadata].update(envelope)
+          end
+        end
+
+        def apply_encryption_cipher(context, cipher)
+          io = context.params[:body] || ''
+          io = StringIO.new(io) if io.is_a? String
+          context.params[:body] = IOEncrypter.new(cipher, io)
+          context.params[:metadata] ||= {}
+          context.params[:metadata]['x-amz-unencrypted-content-length'] = io.size
+          if context.params.delete(:content_md5)
+            raise ArgumentError, 'Setting content_md5 on client side '\
+              'encrypted objects is deprecated.'
+          end
+          context.http_response.on_headers do
+            context.params[:body].close
+          end
+        end
+
+        def apply_cse_user_agent(context)
+          if context.config.user_agent_suffix.nil?
+            context.config.user_agent_suffix = EC_USER_AGENT
+          elsif !context.config.user_agent_suffix.include? EC_USER_AGENT
+            context.config.user_agent_suffix += " #{EC_USER_AGENT}"
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryptionV2/errors.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module EncryptionV2
+      module Errors
+
+        # Generic DecryptionError
+        class DecryptionError < RuntimeError; end
+
+        class EncryptionError < RuntimeError; end
+
+        # Raised when attempting to decrypt a legacy (V1) encrypted object
+        # when using a security_profile that does not support it.
+        class LegacyDecryptionError < DecryptionError
+          def initialize(*args)
+            msg = 'The requested object is ' \
+              'encrypted with V1 encryption schemas that have been disabled ' \
+              'by client configuration security_profile = :v2. Retry with ' \
+              ':v2_and_legacy or re-encrypt the object.'
+            super(msg)
+          end
+        end
+
+        class CEKAlgMismatchError < DecryptionError
+          def initialize(*args)
+            msg = 'The content encryption algorithm used at encryption time ' \
+              'does not match the algorithm stored for decryption time. ' \
+              'The object may be altered or corrupted.'
+            super(msg)
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryptionV2/io_auth_decrypter.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module EncryptionV2
+      # @api private
+      class IOAuthDecrypter
+
+        # @option options [required, IO#write] :io
+        #   An IO-like object that responds to {#write}.
+        # @option options [required, Integer] :encrypted_content_length
+        #   The number of bytes to decrypt from the `:io` object.
+        #   This should be the total size of `:io` minus the length of
+        #   the cipher auth tag.
+        # @option options [required, OpenSSL::Cipher] :cipher An initialized
+        #   cipher that can be used to decrypt the bytes as they are
+        #   written to the `:io` object. The cipher should already have
+        #   its `#auth_tag` set.
+        def initialize(options = {})
+          @decrypter = IODecrypter.new(options[:cipher], options[:io])
+          @max_bytes = options[:encrypted_content_length]
+          @bytes_written = 0
+        end
+
+        def write(chunk)
+          chunk = truncate_chunk(chunk)
+          if chunk.bytesize > 0
+            @bytes_written += chunk.bytesize
+            @decrypter.write(chunk)
+          end
+        end
+
+        def finalize
+          @decrypter.finalize
+        end
+
+        def io
+          @decrypter.io
+        end
+
+        private
+
+        def truncate_chunk(chunk)
+          if chunk.bytesize + @bytes_written <= @max_bytes
+            chunk
+          elsif @bytes_written < @max_bytes
+            chunk[0..(@max_bytes - @bytes_written - 1)]
+          else
+            # If the tag was sent over after the full body has been read,
+            # we don't want to accidentally append it.
+            ""
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryptionV2/io_decrypter.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module EncryptionV2
+      # @api private
+      class IODecrypter
+
+        # @param [OpenSSL::Cipher] cipher
+        # @param [IO#write] io An IO-like object that responds to `#write`.
+        def initialize(cipher, io)
+          @cipher = cipher
+          # Ensure that IO is reset between retries
+          @io = io.tap { |io| io.truncate(0) if io.respond_to?(:truncate) }
+          @cipher_buffer = String.new
+        end
+
+        # @return [#write]
+        attr_reader :io
+
+        def write(chunk)
+          # decrypt and write
+          if @cipher.method(:update).arity == 1
+            @io.write(@cipher.update(chunk))
+          else
+            @io.write(@cipher.update(chunk, @cipher_buffer))
+          end
+        end
+
+        def finalize
+          @io.write(@cipher.final)
+        end
+
+      end
+    end
+  end
+end