aws-sdk-s3 1.75.0

data/lib/aws-sdk-s3/encryption/io_encrypter.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require 'tempfile'
+
+module Aws
+  module S3
+    module Encryption
+
+      # Provides an IO wrapper encrpyting a stream of data.
+      # It is possible to use this same object for decrypting. You must
+      # initialize it with a decryptiion cipher in that case and the
+      # IO object must contain cipher text instead of plain text.
+      # @api private
+      class IOEncrypter
+
+        # @api private
+        ONE_MEGABYTE = 1024 * 1024
+
+        def initialize(cipher, io)
+          @encrypted = io.size <= ONE_MEGABYTE ?
+            encrypt_to_stringio(cipher, io.read) :
+            encrypt_to_tempfile(cipher, io)
+          @size = @encrypted.size
+        end
+
+        # @return [Integer]
+        attr_reader :size
+
+        def read(bytes =  nil, output_buffer = nil)
+          if Tempfile === @encrypted && @encrypted.closed?
+            @encrypted.open
+            @encrypted.binmode
+          end
+          @encrypted.read(bytes, output_buffer)
+        end
+
+        def rewind
+          @encrypted.rewind
+        end
+
+        # @api private
+        def close
+          @encrypted.close if Tempfile === @encrypted
+        end
+
+        private
+
+        def encrypt_to_stringio(cipher, plain_text)
+          if plain_text.empty?
+            StringIO.new(cipher.final)
+          else
+            StringIO.new(cipher.update(plain_text) + cipher.final)
+          end
+        end
+
+        def encrypt_to_tempfile(cipher, io)
+          encrypted = Tempfile.new(self.object_id.to_s)
+          encrypted.binmode
+          while chunk = io.read(ONE_MEGABYTE)
+            encrypted.write(cipher.update(chunk))
+          end
+          encrypted.write(cipher.final)
+          encrypted.rewind
+          encrypted
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryption/key_provider.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module Encryption
+
+      # This module defines the interface required for a {Client#key_provider}.
+      # A key provider is any object that:
+      #
+      # * Responds to {#encryption_materials} with an {Materials} object.
+      #
+      # * Responds to {#key_for}, receiving a JSON document String,
+      #   returning an encryption key. The returned encryption key
+      #   must be one of:
+      #
+      #   * `OpenSSL::PKey::RSA` - for asymmetric encryption
+      #   * `String` - 32, 24, or 16 bytes long, for symmetric encryption
+      #
+      module KeyProvider
+
+        # @return [Materials]
+        def encryption_materials; end
+
+        # @param [String<JSON>] materials_description
+        # @return [OpenSSL::PKey::RSA, String] encryption_key
+        def key_for(materials_description); end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module Aws
+  module S3
+    module Encryption
+      # @api private
+      class KmsCipherProvider
+
+        def initialize(options = {})
+          @kms_key_id = options[:kms_key_id]
+          @kms_client = options[:kms_client]
+        end
+
+        # @return [Array<Hash,Cipher>] Creates an returns a new encryption
+        #   envelope and encryption cipher.
+        def encryption_cipher
+          encryption_context = { "kms_cmk_id" => @kms_key_id }
+          key_data = @kms_client.generate_data_key(
+            key_id: @kms_key_id,
+            encryption_context: encryption_context,
+            key_spec: 'AES_256',
+          )
+          cipher = Utils.aes_encryption_cipher(:CBC)
+          cipher.key = key_data.plaintext
+          envelope = {
+            'x-amz-key-v2' => encode64(key_data.ciphertext_blob),
+            'x-amz-iv' => encode64(cipher.iv = cipher.random_iv),
+            'x-amz-cek-alg' => 'AES/CBC/PKCS5Padding',
+            'x-amz-wrap-alg' => 'kms',
+            'x-amz-matdesc' => Json.dump(encryption_context)
+          }
+          [envelope, cipher]
+        end
+
+        # @return [Cipher] Given an encryption envelope, returns a
+        #   decryption cipher.
+        def decryption_cipher(envelope)
+          encryption_context = Json.load(envelope['x-amz-matdesc'])
+          key = @kms_client.decrypt(
+            ciphertext_blob: decode64(envelope['x-amz-key-v2']),
+            encryption_context: encryption_context,
+          ).plaintext
+          iv = decode64(envelope['x-amz-iv'])
+          block_mode =
+            case envelope['x-amz-cek-alg']
+            when 'AES/CBC/PKCS5Padding'
+              :CBC
+            when 'AES/CBC/PKCS7Padding'
+              :CBC
+            when 'AES/GCM/NoPadding'
+              :GCM
+            else
+              type = envelope['x-amz-cek-alg'].inspect
+              msg = "unsupported content encrypting key (cek) format: #{type}"
+              raise Errors::DecryptionError, msg
+            end
+          Utils.aes_decryption_cipher(block_mode, key, iv)
+        end
+
+        private
+
+        def encode64(str)
+          Base64.encode64(str).split("\n") * ""
+        end
+
+        def decode64(str)
+          Base64.decode64(str)
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryption/materials.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module Aws
+  module S3
+    module Encryption
+      class Materials
+
+        # @option options [required, OpenSSL::PKey::RSA, String] :key
+        #   The master key to use for encrypting/decrypting all objects.
+        #
+        # @option options [String<JSON>] :description ('{}')
+        #   The encryption materials description. This is must be
+        #   a JSON document string.
+        #
+        def initialize(options = {})
+          @key = validate_key(options[:key])
+          @description = validate_desc(options[:description])
+        end
+
+        # @return [OpenSSL::PKey::RSA, String]
+        attr_reader :key
+
+        # @return [String<JSON>]
+        attr_reader :description
+
+        private
+
+        def validate_key(key)
+          case key
+          when OpenSSL::PKey::RSA then key
+          when String
+            if [32, 24, 16].include?(key.bytesize)
+              key
+            else
+              msg = 'invalid key, symmetric key required to be 16, 24, or '\
+                    '32 bytes in length, saw length ' + key.bytesize.to_s
+              raise ArgumentError, msg
+            end
+          else
+            msg = 'invalid encryption key, expected an OpenSSL::PKey::RSA key '\
+                  '(for asymmetric encryption) or a String (for symmetric '\
+                  'encryption).'
+            raise ArgumentError, msg
+          end
+        end
+
+        def validate_desc(description)
+          Json.load(description)
+          description
+        rescue Json::ParseError, EncodingError
+          msg = 'expected description to be a valid JSON document string'
+          raise ArgumentError, msg
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryption/utils.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'openssl'
+
+module Aws
+  module S3
+    module Encryption
+      # @api private
+      module Utils
+
+        UNSAFE_MSG = "unsafe encryption, data is longer than key length"
+
+        class << self
+
+          def encrypt(key, data)
+            case key
+            when OpenSSL::PKey::RSA # asymmetric encryption
+              warn(UNSAFE_MSG) if key.public_key.n.num_bits < cipher_size(data)
+              key.public_encrypt(data)
+            when String # symmetric encryption
+              warn(UNSAFE_MSG) if cipher_size(key) < cipher_size(data)
+              cipher = aes_encryption_cipher(:ECB, key)
+              cipher.update(data) + cipher.final
+            end
+          end
+
+          def decrypt(key, data)
+            begin
+              case key
+              when OpenSSL::PKey::RSA # asymmetric decryption
+                key.private_decrypt(data)
+              when String # symmetric Decryption
+                cipher = aes_cipher(:decrypt, :ECB, key, nil)
+                cipher.update(data) + cipher.final
+              end
+            rescue OpenSSL::Cipher::CipherError
+              msg = 'decryption failed, possible incorrect key'
+              raise Errors::DecryptionError, msg
+            end
+          end
+
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_encryption_cipher(block_mode, key = nil, iv = nil)
+            aes_cipher(:encrypt, block_mode, key, iv)
+          end
+
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_decryption_cipher(block_mode, key = nil, iv = nil)
+            aes_cipher(:decrypt, block_mode, key, iv)
+          end
+
+          # @param [String] mode "encrypt" or "decrypt"
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_cipher(mode, block_mode, key, iv)
+            cipher = key ?
+              OpenSSL::Cipher.new("aes-#{cipher_size(key)}-#{block_mode.downcase}") :
+              OpenSSL::Cipher.new("aes-256-#{block_mode.downcase}")
+            cipher.send(mode) # encrypt or decrypt
+            cipher.key = key if key
+            cipher.iv = iv if iv
+            cipher
+          end
+
+          # @param [String] key
+          # @return [Integer]
+          # @raise ArgumentError
+          def cipher_size(key)
+            key.bytesize * 8
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryptionV2/client.rb

@@ -0,0 +1,388 @@
+require 'forwardable'
+
+module Aws
+  module S3
+
+    # Provides an encryption client that encrypts and decrypts data client-side,
+    # storing the encrypted data in Amazon S3.  The EncryptionV2::Client
+    # provides improved security over the Encryption::Client by using more
+    # modern and secure algorithms.  The EncryptionV2::Client maintains
+    # backwards compatibility: Using the EncryptionV2::Client you can decrypt
+    # objects encrypted with the Encryption::Client.  However, objects you
+    # encrypt using the EncryptionV2::Client cannot be decrypted using the
+    # Encryption::Client.
+    #
+    # This client uses a process called "envelope encryption". Your private
+    # encryption keys and your data's plain-text are **never** sent to
+    # Amazon S3. **If you lose you encryption keys, you will not be able to
+    # decrypt your data.**
+    #
+    # ## Envelope Encryption Overview
+    #
+    # The goal of envelope encryption is to combine the performance of
+    # fast symmetric encryption while maintaining the secure key management
+    # that asymmetric keys provide.
+    #
+    # A one-time-use symmetric key (envelope key) is generated client-side.
+    # This is used to encrypt the data client-side. This key is then
+    # encrypted by your master key and stored alongside your data in Amazon
+    # S3.
+    #
+    # When accessing your encrypted data with the encryption client,
+    # the encrypted envelope key is retrieved and decrypted client-side
+    # with your master key. The envelope key is then used to decrypt the
+    # data client-side.
+    #
+    # One of the benefits of envelope encryption is that if your master key
+    # is compromised, you have the option of just re-encrypting the stored
+    # envelope symmetric keys, instead of re-encrypting all of the
+    # data in your account.
+    #
+    # ## Basic Usage
+    #
+    # The encryption client requires an {Aws::S3::Client}. If you do not
+    # provide a `:client`, then a client will be constructed for you.
+    #
+    #     require 'openssl'
+    #     key = OpenSSL::PKey::RSA.new(1024)
+    #
+    #     # encryption client
+    #     s3 = Aws::S3::EncryptionV2::Client.new(encryption_key: key)
+    #
+    #     # round-trip an object, encrypted/decrypted locally
+    #     s3.put_object(bucket:'aws-sdk', key:'secret', body:'handshake')
+    #     s3.get_object(bucket:'aws-sdk', key:'secret').body.read
+    #     #=> 'handshake'
+    #
+    #     # reading encrypted object without the encryption client
+    #     # results in the getting the cipher text
+    #     Aws::S3::Client.new.get_object(bucket:'aws-sdk', key:'secret').body.read
+    #     #=> "... cipher text ..."
+    #
+    # ## Keys
+    #
+    # For client-side encryption to work, you must provide one of the following:
+    #
+    # * An encryption key
+    # * A {KeyProvider}
+    # * A KMS encryption key id
+    #
+    # ### An Encryption Key
+    #
+    # You can pass a single encryption key. This is used as a master key
+    # encrypting and decrypting all object keys.
+    #
+    #     key = OpenSSL::Cipher.new("AES-256-ECB").random_key # symmetric key
+    #     key = OpenSSL::PKey::RSA.new(1024) # asymmetric key pair
+    #
+    #     s3 = Aws::S3::EncryptionV2::Client.new(encryption_key: key)
+    #
+    # ### Key Provider
+    #
+    # Alternatively, you can use a {KeyProvider}. A key provider makes
+    # it easy to work with multiple keys and simplifies key rotation.
+    #
+    # ### KMS Encryption Key Id
+    #
+    # If you pass the id to an AWS Key Management Service (KMS) key,
+    # then KMS will be used to generate, encrypt and decrypt object keys.
+    #
+    #     # keep track of the kms key id
+    #     kms = Aws::KMS::Client.new
+    #     key_id = kms.create_key.key_metadata.key_id
+    #
+    #     Aws::S3::EncryptionV2::Client.new(
+    #       kms_key_id: key_id,
+    #       kms_client: kms,
+    #     )
+    #
+    # ## Custom Key Providers
+    #
+    # A {KeyProvider} is any object that responds to:
+    #
+    # * `#encryption_materials`
+    # * `#key_for(materials_description)`
+    #
+    # Here is a trivial implementation of an in-memory key provider.
+    # This is provided as a demonstration of the key provider interface,
+    # and should not be used in production:
+    #
+    #     class KeyProvider
+    #
+    #       def initialize(default_key_name, keys)
+    #         @keys = keys
+    #         @encryption_materials = Aws::S3::EncryptionV2::Materials.new(
+    #           key: @keys[default_key_name],
+    #           description: JSON.dump(key: default_key_name),
+    #         )
+    #       end
+    #
+    #       attr_reader :encryption_materials
+    #
+    #       def key_for(matdesc)
+    #         key_name = JSON.load(matdesc)['key']
+    #         if key = @keys[key_name]
+    #           key
+    #         else
+    #           raise "encryption key not found for: #{matdesc.inspect}"
+    #         end
+    #       end
+    #     end
+    #
+    # Given the above key provider, you can create an encryption client that
+    # chooses the key to use based on the materials description stored with
+    # the encrypted object. This makes it possible to use multiple keys
+    # and simplifies key rotation.
+    #
+    #     # uses "new-key" for encrypting objects, uses either for decrypting
+    #     keys = KeyProvider.new('new-key', {
+    #       "old-key" => Base64.decode64("kM5UVbhE/4rtMZJfsadYEdm2vaKFsmV2f5+URSeUCV4="),
+    #       "new-key" => Base64.decode64("w1WLio3agRWRTSJK/Ouh8NHoqRQ6fn5WbSXDTHjXMSo="),
+    #     }),
+    #
+    #     # chooses the key based on the materials description stored
+    #     # with the encrypted object
+    #     s3 = Aws::S3::EncryptionV2::Client.new(key_provider: keys)
+    #
+    # ## Materials Description
+    #
+    # A materials description is JSON document string that is stored
+    # in the metadata (or instruction file) of an encrypted object.
+    # The {DefaultKeyProvider} uses the empty JSON document `"{}"`.
+    #
+    # When building a key provider, you are free to store whatever
+    # information you need to identify the master key that was used
+    # to encrypt the object.
+    #
+    # ## Envelope Location
+    #
+    # By default, the encryption client store the encryption envelope
+    # with the object, as metadata. You can choose to have the envelope
+    # stored in a separate "instruction file". An instruction file
+    # is an object, with the key of the encrypted object, suffixed with
+    # `".instruction"`.
+    #
+    # Specify the `:envelope_location` option as `:instruction_file` to
+    # use an instruction file for storing the envelope.
+    #
+    #     # default behavior
+    #     s3 = Aws::S3::EncryptionV2::Client.new(
+    #       key_provider: ...,
+    #       envelope_location: :metadata,
+    #     )
+    #
+    #     # store envelope in a separate object
+    #     s3 = Aws::S3::EncryptionV2::Client.new(
+    #       key_provider: ...,
+    #       envelope_location: :instruction_file,
+    #       instruction_file_suffix: '.instruction' # default
+    #     )
+    #
+    # When using an instruction file, multiple requests are made when
+    # putting and getting the object. **This may cause issues if you are
+    # issuing concurrent PUT and GET requests to an encrypted object.**
+    #
+    module EncryptionV2
+      class Client
+
+        extend Deprecations
+        extend Forwardable
+        def_delegators :@client, :config, :delete_object, :head_object, :build_request
+
+        # Creates a new encryption client. You must provide one of the following
+        # options:
+        #
+        # * `:encryption_key`
+        # * `:kms_key_id`
+        # * `:key_provider`
+        #
+        # You may also pass any other options accepted by `Client#initialize`.
+        #
+        # @option options [S3::Client] :client A basic S3 client that is used
+        #   to make api calls. If a `:client` is not provided, a new {S3::Client}
+        #   will be constructed.
+        #
+        # @option options [OpenSSL::PKey::RSA, String] :encryption_key The master
+        #   key to use for encrypting/decrypting all objects.
+        #
+        # @option options [String] :kms_key_id When you provide a `:kms_key_id`,
+        #   then AWS Key Management Service (KMS) will be used to manage the
+        #   object encryption keys. By default a {KMS::Client} will be
+        #   constructed for KMS API calls. Alternatively, you can provide
+        #   your own via `:kms_client`.
+        #
+        # @option options [#key_for] :key_provider Any object that responds
+        #   to `#key_for`. This method should accept a materials description
+        #   JSON document string and return return an encryption key.
+        #
+        # @option options [Symbol] :envelope_location (:metadata) Where to
+        #   store the envelope encryption keys. By default, the envelope is
+        #   stored with the encrypted object. If you pass `:instruction_file`,
+        #   then the envelope is stored in a separate object in Amazon S3.
+        #
+        # @option options [String] :instruction_file_suffix ('.instruction')
+        #   When `:envelope_location` is `:instruction_file` then the
+        #   instruction file uses the object key with this suffix appended.
+        #
+        # @option options [KMS::Client] :kms_client A default {KMS::Client}
+        #   is constructed when using KMS to manage encryption keys.
+        #
+        def initialize(options = {})
+          @client = extract_client(options)
+          @cipher_provider = cipher_provider(options)
+          @envelope_location = extract_location(options)
+          @instruction_file_suffix = extract_suffix(options)
+        end
+
+        # @return [S3::Client]
+        attr_reader :client
+
+        # @return [KeyProvider, nil] Returns `nil` if you are using
+        #   AWS Key Management Service (KMS).
+        attr_reader :key_provider
+
+        # @return [Symbol<:metadata, :instruction_file>]
+        attr_reader :envelope_location
+
+        # @return [String] When {#envelope_location} is `:instruction_file`,
+        #   the envelope is stored in the object with the object key suffixed
+        #   by this string.
+        attr_reader :instruction_file_suffix
+
+        # Uploads an object to Amazon S3, encrypting data client-side.
+        # See {S3::Client#put_object} for documentation on accepted
+        # request parameters.
+        # @option params [Hash] :kms_encryption_context Additional encryption
+        #   context to use with KMS.  Applies only when KMS is used. In order
+        #   to decrypt the object you will need to provide the identical
+        #   :kms_encryption_context to `get_object`.
+        # @option (see S3::Client#put_object)
+        # @return (see S3::Client#put_object)
+        # @see S3::Client#put_object
+        def put_object(params = {})
+          kms_encryption_context = params.delete(:kms_encryption_context)
+          req = @client.build_request(:put_object, params)
+          req.handlers.add(EncryptHandler, priority: 95)
+          req.context[:encryption] = {
+            cipher_provider: @cipher_provider,
+            envelope_location: @envelope_location,
+            instruction_file_suffix: @instruction_file_suffix,
+            kms_encryption_context: kms_encryption_context
+          }
+          req.send_request
+        end
+
+        # Gets an object from Amazon S3, decrypting  data locally.
+        # See {S3::Client#get_object} for documentation on accepted
+        # request parameters.
+        # @option params [String] :instruction_file_suffix The suffix
+        #   used to find the instruction file containing the encryption
+        #   envelope. You should not set this option when the envelope
+        #   is stored in the object metadata. Defaults to
+        #   {#instruction_file_suffix}.
+        # @option params [Hash] :kms_encryption_context Additional encryption
+        #   context to use with KMS.  Applies only when KMS is used.
+        # @option params [String] :instruction_file_suffix
+        # @option (see S3::Client#get_object)
+        # @return (see S3::Client#get_object)
+        # @see S3::Client#get_object
+        # @note The `:range` request parameter is not yet supported.
+        def get_object(params = {}, &block)
+          if params[:range]
+            raise NotImplementedError, '#get_object with :range not supported'
+          end
+          envelope_location, instruction_file_suffix = envelope_options(params)
+          kms_encryption_context = params.delete(:kms_encryption_context)
+          req = @client.build_request(:get_object, params)
+          req.handlers.add(DecryptHandler)
+          req.context[:encryption] = {
+            cipher_provider: @cipher_provider,
+            envelope_location: envelope_location,
+            instruction_file_suffix: instruction_file_suffix,
+            kms_encryption_context: kms_encryption_context
+          }
+          req.send_request(target: block)
+        end
+
+        private
+
+        def extract_client(options)
+          options[:client] || begin
+            options = options.dup
+            options.delete(:kms_key_id)
+            options.delete(:kms_client)
+            options.delete(:key_provider)
+            options.delete(:encryption_key)
+            options.delete(:envelope_location)
+            options.delete(:instruction_file_suffix)
+            S3::Client.new(options)
+          end
+        end
+
+        def kms_client(options)
+          options[:kms_client] || begin
+            KMS::Client.new(
+              region: @client.config.region,
+              credentials: @client.config.credentials,
+            )
+          end
+        end
+
+        def cipher_provider(options)
+          if options[:kms_key_id]
+            KmsCipherProvider.new(
+              kms_key_id: options[:kms_key_id],
+              kms_client: kms_client(options),
+            )
+          else
+            @key_provider = extract_key_provider(options)
+            DefaultCipherProvider.new(key_provider: @key_provider)
+          end
+        end
+
+        def extract_key_provider(options)
+          if options[:key_provider]
+            options[:key_provider]
+          elsif options[:encryption_key]
+            DefaultKeyProvider.new(options)
+          else
+            msg = 'you must pass a :kms_key_id, :key_provider, or :encryption_key'
+            raise ArgumentError, msg
+          end
+        end
+
+        def envelope_options(params)
+          location = params.delete(:envelope_location) || @envelope_location
+          suffix = params.delete(:instruction_file_suffix)
+          if suffix
+            [:instruction_file, suffix]
+          else
+            [location, @instruction_file_suffix]
+          end
+        end
+
+        def extract_location(options)
+          location = options[:envelope_location] || :metadata
+          if [:metadata, :instruction_file].include?(location)
+            location
+          else
+            msg = ':envelope_location must be :metadata or :instruction_file '\
+                  "got #{location.inspect}"
+            raise ArgumentError, msg
+          end
+        end
+
+        def extract_suffix(options)
+          suffix = options[:instruction_file_suffix] || '.instruction'
+          if suffix.is_a? String
+            suffix
+          else
+            msg = ':instruction_file_suffix must be a String'
+            raise ArgumentError, msg
+          end
+        end
+
+      end
+    end
+  end
+end