symmetric-encryption 3.9.1 → 4.0.0.beta3

data/lib/symmetric_encryption/header.rb

@@ -0,0 +1,260 @@
+module SymmetricEncryption
+  # Defines the Header Structure returned when parsing the header.
+  #
+  # Note:
+  # * Header only works against binary encrypted data that has not been decoded.
+  # * Decode data first before trying to extract its header.
+  # * Decoding is not required when encoding is set to `:none`.
+  class Header
+    # Encrypted data includes this header prior to encoding when
+    # `always_add_header` is true.
+    MAGIC_HEADER      = '@EnC'.force_encoding(SymmetricEncryption::BINARY_ENCODING)
+    MAGIC_HEADER_SIZE = MAGIC_HEADER.size
+
+    # [true|false] Whether to compress the data before encryption.
+    # If supplied in the header.
+    attr_accessor :compress
+
+    # [String] IV used to encrypt the data.
+    # If supplied in the header.
+    attr_accessor :iv
+
+    # [String] Key used to encrypt the data.
+    # If supplied in the header.
+    attr_accessor :key
+
+    # [String] Name of the cipher used.
+    attr_accessor :cipher_name
+
+    # [Integer] Version of the cipher used.
+    attr_reader :version
+
+    # [String] Binary auth tag used to encrypt the data.
+    # Usually 16 bytes.
+    # Present when using an authenticated encryption mode.
+    attr_reader :auth_tag
+
+    # Returns whether the supplied buffer starts with a symmetric_encryption header
+    # Note: The encoding of the supplied buffer is forced to binary if not already binary
+    def self.present?(buffer)
+      return false if buffer.nil? || (buffer == '')
+      buffer.force_encoding(SymmetricEncryption::BINARY_ENCODING)
+      buffer.start_with?(MAGIC_HEADER)
+    end
+
+    # Returns a magic header for this cipher instance that can be placed at
+    # the beginning of a file or stream to indicate how the data was encrypted
+    #
+    # Parameters
+    #   compress [true|false]
+    #     Whether the data should be compressed before encryption.
+    #     Default: false
+    #
+    #   iv [String]
+    #     The iv to to put in the header
+    #     Default: nil : Exclude from header
+    #
+    #   key [String]
+    #     The key to to put in the header.
+    #     The key is encrypted using the global encryption key
+    #     Default: nil : Exclude key from header
+    #
+    #   version: [Integer (0..255)]
+    #     Version of the global cipher used to encrypt the data,
+    #     or the encryption key if supplied.
+    #     default: The current global encryption cipher version.
+    #
+    #   cipher_name [String]
+    #     The cipher_name to be used for encrypting the data portion.
+    #     For example 'aes-256-cbc'
+    #     `key` if supplied is encrypted with the cipher name based on the cipher version in this header.
+    #     Intended for use when encrypting large files with a different cipher to the global one.
+    #     Default: nil : Exclude cipher_name name from header
+    def initialize(version: SymmetricEncryption.cipher.version,
+                   compress: false,
+                   iv: nil,
+                   key: nil,
+                   cipher_name: nil,
+                   auth_tag: nil)
+
+      @version     = version
+      @compress  = compress
+      @iv          = iv
+      @key         = key
+      @cipher_name = cipher_name
+      @auth_tag    = auth_tag
+    end
+
+    # Returns [SymmetricEncryption::Cipher] the cipher used to decrypt or encrypt the key
+    # specified in this header, if supplied.
+    def cipher
+      @cipher ||= SymmetricEncryption.cipher(version)
+    end
+
+    def version=(version)
+      @version = version
+      @cipher  = nil
+    end
+
+    def compressed?
+      @compress
+    end
+
+    # Returns [String] the encrypted data without header
+    # Returns nil if no header is present
+    #
+    # The supplied buffer will be updated directly and
+    # its header will be stripped if present.
+    #
+    # Parameters
+    #   buffer
+    #     String to extract the header from
+    def parse!(buffer)
+      offset = parse(buffer)
+      return if offset == 0
+      buffer.slice!(0..offset - 1)
+      buffer
+    end
+
+    # Returns [Integer] the offset within the buffer of the data after the header has been read.
+    #
+    # Returns 0 if no header is present
+    def parse(buffer, offset = 0)
+      return 0 if buffer.nil? || (buffer == '') || (buffer.length <= MAGIC_HEADER_SIZE + 2)
+
+      # Symmetric Encryption Header
+      #
+      # Consists of:
+      #    4 Bytes: Magic Header Prefix: @Enc
+      #    1 Byte:  The version of the cipher used to encrypt the header.
+      #    1 Byte:  Flags:
+      #       Bit 1: Whether the data is compressed
+      #       Bit 2: Whether the IV is included
+      #       Bit 3: Whether the Key is included
+      #       Bit 4: Whether the Cipher Name is included
+      #       Bit 5: Future use
+      #       Bit 6: Future use
+      #       Bit 7: Future use
+      #       Bit 8: Future use
+      #    2 Bytes: IV Length (little endian), if included.
+      #      IV in binary form.
+      #    2 Bytes: Key Length (little endian), if included.
+      #      Key in binary form
+      #    2 Bytes: Cipher Name Length (little endian), if included.
+      #      Cipher name it UTF8 text
+
+      buffer.force_encoding(SymmetricEncryption::BINARY_ENCODING)
+      header = buffer.byteslice(offset, MAGIC_HEADER_SIZE)
+      return 0 unless header == MAGIC_HEADER
+
+      offset += MAGIC_HEADER_SIZE
+
+      # Remove header and extract flags
+      self.version = buffer.getbyte(offset)
+      offset       += 1
+
+      raise(SymmetricEncryption::CipherError, "Cipher with version:#{version.inspect} not found in any of the configured SymmetricEncryption ciphers") unless cipher
+      flags  = buffer.getbyte(offset)
+      offset += 1
+
+      self.compress = (flags & FLAG_COMPRESSED) != 0
+
+      if (flags & FLAG_IV) != 0
+        self.iv, offset = read_string(buffer, offset)
+      else
+        self.iv = nil
+      end
+
+      if (flags & FLAG_KEY) != 0
+        encrypted_key, offset = read_string(buffer, offset)
+        self.key              = cipher.binary_decrypt(encrypted_key)
+      else
+        self.key = nil
+      end
+
+      if (flags & FLAG_CIPHER_NAME) != 0
+        self.cipher_name, offset = read_string(buffer, offset)
+      else
+        self.cipher_name = nil
+      end
+
+      if (flags & FLAG_AUTH_TAG) != 0
+        self.auth_tag, offset = read_string(buffer, offset)
+      else
+        self.auth_tag = nil
+      end
+
+      offset
+    end
+
+    # Returns [String] this header as a string
+    def to_s
+      flags = 0
+      flags |= FLAG_COMPRESSED if compressed?
+      flags |= FLAG_IV if iv
+      flags |= FLAG_KEY if key
+      flags |= FLAG_CIPHER_NAME if cipher_name
+      flags |= FLAG_AUTH_TAG if auth_tag
+
+      header = "#{MAGIC_HEADER}#{version.chr(SymmetricEncryption::BINARY_ENCODING)}#{flags.chr(SymmetricEncryption::BINARY_ENCODING)}"
+
+      if iv
+        header << [iv.length].pack('v')
+        header << iv
+      end
+
+      if key
+        encrypted = cipher.binary_encrypt(key, header: false)
+        header << [encrypted.length].pack('v')
+        header << encrypted
+      end
+
+      if cipher_name
+        header << [cipher_name.length].pack('v')
+        header << cipher_name
+      end
+
+      if auth_tag
+        header << [auth_tag.length].pack('v')
+        header << auth_tag
+      end
+
+      header
+    end
+
+    private
+
+    FLAG_COMPRESSED  = 0b1000_0000
+    FLAG_IV          = 0b0100_0000
+    FLAG_KEY         = 0b0010_0000
+    FLAG_CIPHER_NAME = 0b0001_0000
+    FLAG_AUTH_TAG    = 0b0000_1000
+
+    attr_writer :auth_tag
+
+    # Extracts a string from the supplied buffer.
+    # The buffer starts with a 2 byte length indicator in little endian format.
+    #
+    # Parameters
+    #   buffer [String]
+    #   offset [Integer]
+    #     Start position within the buffer.
+    #
+    # Returns [string, offset]
+    #   string [String]
+    #     The string copied from the buffer.
+    #   offset [Integer]
+    #     The new offset within the buffer.
+    def read_string(buffer, offset)
+      # TODO: Length check
+      #   Exception when
+      #   - offset exceeds length of buffer
+      #   byteslice truncates when too long, but returns nil when start is beyond end of buffer
+      len    = buffer.byteslice(offset, 2).unpack('v').first
+      offset += 2
+      out    = buffer.byteslice(offset, len)
+      [out, offset + len]
+    end
+
+  end
+end

data/lib/symmetric_encryption/key.rb

@@ -0,0 +1,106 @@
+# The key, iv and encrypted data are handled in their raw form, with no encoding.
+module SymmetricEncryption
+  class Key
+    attr_reader :key, :iv, :cipher_name
+
+    # Returns [Key] from cipher data usually extracted from the configuration file.
+    #
+    # Supports N level deep key encrypting keys.
+    #
+    # Configuration keys:
+    # * key
+    # * encrypted_key
+    # * key_filename
+    def self.from_config(key: nil, key_filename: nil, encrypted_key: nil, key_env_var: nil,
+      iv:, key_encrypting_key: nil, cipher_name: 'aes-256-cbc')
+
+      if key_encrypting_key.is_a?(Hash)
+        # Recurse up the chain returning the parent key_encrypting_key
+        key_encrypting_key = from_config(cipher_name: cipher_name, **key_encrypting_key)
+      end
+
+      key ||=
+        if encrypted_key
+          raise(ArgumentError, "Missing mandatory :key_encrypting_key when config includes :encrypted_key") unless key_encrypting_key
+          Keystore::Memory.new(encrypted_key: encrypted_key, key_encrypting_key: key_encrypting_key).read
+        elsif key_filename
+          Keystore::File.new(file_name: key_filename, key_encrypting_key: key_encrypting_key).read
+        elsif key_env_var
+          raise(ArgumentError, "Missing mandatory :key_encrypting_key when config includes :key_env_var") unless key_encrypting_key
+          Keystore::Environment.new(key_env_var: key_env_var, key_encrypting_key: key_encrypting_key).read
+        end
+
+      new(key: key, iv: iv, cipher_name: cipher_name)
+    end
+
+    # Migrate a prior config.
+    #
+    # Note:
+    # * The config cannot be saved back to the config file once
+    #   migrated, without generating new Key Encrypting Keys.
+    # * Only run this migration in the target environment so that the
+    #   current key encrypting files are present.
+    def self.migrate_config!(config)
+      # Backward compatibility - Deprecated
+      private_rsa_key = config.delete(:private_rsa_key)
+
+      # Migrate old encrypted_iv
+      if (encrypted_iv = config.delete(:encrypted_iv)) && private_rsa_key
+        encrypted_iv = RSAKey.new(private_rsa_key).decrypt(encrypted_iv)
+        config[:iv]  = ::Base64.decode64(encrypted_iv)
+      end
+
+      # Migrate old iv_filename
+      if (file_name = config.delete(:iv_filename)) && private_rsa_key
+        encrypted_iv = File.read(file_name)
+        config[:iv]  = RSAKey.new(private_rsa_key).decrypt(encrypted_iv)
+      end
+
+      # Backward compatibility - Deprecated
+      config[:key_encrypting_key] = RSAKey.new(private_rsa_key) if private_rsa_key
+
+      # Migrate old encrypted_key to new binary format
+      if (encrypted_key = config[:encrypted_key]) && private_rsa_key
+        config[:encrypted_key] = ::Base64.decode64(encrypted_key)
+      end
+    end
+
+    def initialize(key: :random, iv: :random, cipher_name: 'aes-256-cbc')
+      @key         = key == :random ? ::OpenSSL::Cipher.new(cipher_name).random_key : key
+      @iv          = iv == :random ? ::OpenSSL::Cipher.new(cipher_name).random_iv : iv
+      @cipher_name = cipher_name
+    end
+
+    def encrypt(string)
+      return if string.nil?
+      string = string.to_s
+      return string if string.empty?
+
+      # Creates a new OpenSSL::Cipher with every call so that this key instance is thread-safe.
+      openssl_cipher = ::OpenSSL::Cipher.new(cipher_name)
+      openssl_cipher.encrypt
+      openssl_cipher.key = key
+      openssl_cipher.iv  = iv
+
+      result = openssl_cipher.update(string)
+      result << openssl_cipher.final
+    end
+
+    def decrypt(encrypted_string)
+      return if encrypted_string.nil?
+      encrypted_string = encrypted_string.to_s
+      encrypted_string.force_encoding(SymmetricEncryption::BINARY_ENCODING)
+      return encrypted_string if encrypted_string.empty?
+
+      # Creates a new OpenSSL::Cipher with every call so that this key instance is thread-safe.
+      openssl_cipher = ::OpenSSL::Cipher.new(cipher_name)
+      openssl_cipher.decrypt
+      openssl_cipher.key = key
+      openssl_cipher.iv  = iv
+
+      result = openssl_cipher.update(encrypted_string)
+      result << openssl_cipher.final
+    end
+
+  end
+end

data/lib/symmetric_encryption/keystore/environment.rb

@@ -0,0 +1,90 @@
+module SymmetricEncryption
+  module Keystore
+    # Store the encrypted encryption key in an environment variable
+    class Environment < Memory
+      attr_accessor :key_env_var, :encoding
+
+      # Returns [Hash] initial configuration for heroku.
+      # Displays the keys that need to be added to the heroku environment.
+      def self.new_config(app_name: 'symmetric-encryption',
+        environments: %i(development test release production),
+        cipher_name: 'aes-256-cbc')
+
+        configs = {}
+        environments.each do |environment|
+          environment          = environment.to_sym
+          configs[environment] =
+            if %i(development test).include?(environment)
+              Keystore.dev_config
+            else
+              cfg = new_key_config(cipher_name: cipher_name, app_name: app_name, environment: environment)
+              {
+                ciphers: [cfg]
+              }
+            end
+        end
+        configs
+      end
+
+      # Returns [Hash] a new cipher, and writes its encrypted key file.
+      #
+      # Increments the supplied version number by 1.
+      def self.new_key_config(cipher_name:, app_name:, environment:, version: 0, dek: nil)
+        version >= 255 ? (version = 1) : (version += 1)
+
+        kek = SymmetricEncryption::Key.new(cipher_name: cipher_name)
+        dek ||= SymmetricEncryption::Key.new(cipher_name: cipher_name)
+
+        key_env_var = "#{app_name}_#{environment}_v#{version}".upcase.gsub('-', '_')
+        new(key_env_var: key_env_var, key_encrypting_key: kek).write(dek.key)
+
+        {
+          cipher_name:        dek.cipher_name,
+          version:            version,
+          key_env_var:        key_env_var,
+          iv:                 dek.iv,
+          key_encrypting_key: {
+            key: kek.key,
+            iv:  kek.iv,
+          }
+        }
+      end
+
+      # Stores the Encryption key in an environment var.
+      # Secures the Encryption key by encrypting it with a key encryption key.
+      def initialize(key_encrypting_key:, key_env_var:, encoding: :base64strict)
+        @key_env_var        = key_env_var
+        @key_encrypting_key = key_encrypting_key
+        @encoding           = encoding
+      end
+
+      # Returns the Encryption key in the clear.
+      def read
+        encrypted = ENV[key_env_var]
+        raise "The Environment Variable #{key_env_var} must be set with the encrypted encryption key." unless encrypted
+        binary = encoder.decode(encrypted)
+        key_encrypting_key.decrypt(binary)
+      end
+
+      # Write the encrypted Encryption key to `encrypted_key` attribute.
+      def write(key)
+        encrypted_key = key_encrypting_key.encrypt(key)
+        puts "\n\n********************************************************************************"
+        puts "Add the environment key to Heroku:\n\n"
+        puts "  heroku config:add #{key_env_var}=#{encoder.encode(encrypted_key)}"
+        puts
+        puts "Or, if using environment variables on another system set the environment variable as follows:\n\n"
+        puts "  export #{key_env_var}=\"#{encoder.encode(encrypted_key)}\"\n\n"
+        puts "********************************************************************************"
+      end
+
+      private
+
+      # Returns [SymmetricEncryption::Encoder] the encoder to use for the current encoding.
+      def encoder
+        @encoder ||= SymmetricEncryption::Encoder[encoding]
+      end
+
+    end
+  end
+end

data/lib/symmetric_encryption/keystore/file.rb

@@ -0,0 +1,102 @@
+module SymmetricEncryption
+  module Keystore
+    class File
+      attr_accessor :file_name, :key_encrypting_key
+
+      # Returns [Hash] initial configuration.
+      # Generates the encrypted key file for every environment except development and test.
+      def self.new_config(key_path: '/etc/symmetric-encryption',
+        app_name: 'symmetric-encryption',
+        environments: %i(development test release production),
+        cipher_name: 'aes-256-cbc')
+
+        configs = {}
+        environments.each do |environment|
+          environment          = environment.to_sym
+          configs[environment] =
+            if %i(development test).include?(environment)
+              Keystore.dev_config
+            else
+              cfg = new_key_config(key_path: key_path, cipher_name: cipher_name, app_name: app_name, environment: environment)
+              {
+                ciphers: [cfg]
+              }
+            end
+        end
+        configs
+      end
+
+      # Returns [Hash] a new cipher, and writes its encrypted key file.
+      #
+      # Increments the supplied version number by 1.
+      def self.new_key_config(key_path:, cipher_name:, app_name:, environment:, version: 0, dek: nil)
+        version >= 255 ? (version = 1) : (version += 1)
+
+        dek   ||= SymmetricEncryption::Key.new(cipher_name: cipher_name)
+        kek   = SymmetricEncryption::Key.new(cipher_name: cipher_name)
+        kekek = SymmetricEncryption::Key.new(cipher_name: cipher_name)
+
+        dek_file_name = ::File.join(key_path, "#{app_name}_#{environment}_v#{version}.encrypted_key")
+        new(file_name: dek_file_name, key_encrypting_key: kek).write(dek.key)
+
+        kekek_file_name = ::File.join(key_path, "#{app_name}_#{environment}_v#{version}.kekek")
+        new(file_name: kekek_file_name).write(kekek.key)
+
+        {
+          cipher_name:        dek.cipher_name,
+          version:            version,
+          key_filename:       dek_file_name,
+          iv:                 dek.iv,
+          key_encrypting_key: {
+            encrypted_key:      kekek.encrypt(kek.key),
+            iv:                 kek.iv,
+            key_encrypting_key: {
+              key_filename: kekek_file_name,
+              iv:           kekek.iv
+            }
+          }
+        }
+      end
+
+      # Stores the Encryption key in a file.
+      # Secures the Encryption key by encrypting it with a key encryption key.
+      def initialize(file_name:, key_encrypting_key: nil)
+        @file_name          = file_name
+        @key_encrypting_key = key_encrypting_key
+      end
+
+      # Returns the Encryption key in the clear.
+      def read
+        # TODO: Validate that file is not globally readable.
+        raise(SymmetricEncryption::ConfigError, "Symmetric Encryption key file: '#{file_name}' not found") unless ::File.exist?(file_name)
+
+        data = read_from_file
+        key_encrypting_key ? key_encrypting_key.decrypt(data) : data
+      end
+
+      # Encrypt and write the key to file.
+      def write(key)
+        data = key_encrypting_key ? key_encrypting_key.encrypt(key) : key
+        write_to_file(data)
+      end
+
+      private
+
+      # Read from the file, raising an exception if it is not found
+      def read_from_file
+        ::File.open(file_name, 'rb') { |f| f.read }
+      rescue Errno::ENOENT
+        raise(SymmetricEncryption::ConfigError, "Symmetric Encryption key file: '#{file_name}' not found or readable")
+      end
+
+      # Write to the supplied file_name, backing up the existing file if present
+      def write_to_file(data)
+        key_path = ::File.dirname(file_name)
+        ::FileUtils.mkdir_p(key_path) unless ::File.directory?(key_path)
+        ::File.rename(file_name, "#{file_name}.#{Time.now.to_i}") if ::File.exist?(file_name)
+        ::File.open(file_name, 'wb') { |file| file.write(data) }
+      end
+
+    end
+  end
+end

data/lib/symmetric_encryption/keystore/memory.rb

@@ -0,0 +1,53 @@
+module SymmetricEncryption
+  module Keystore
+    class Memory
+      attr_accessor :key_encrypting_key
+      attr_reader :encrypted_key
+
+      # Returns [Hash] a new cipher, and writes its encrypted key file.
+      #
+      # Increments the supplied version number by 1.
+      #
+      # Notes:
+      # * For development and testing purposes only!!
+      # * Never store the encrypted encryption key in the source code / config file.
+      def self.new_key_config(cipher_name:, app_name:, environment:, version: 0, dek: nil)
+        version >= 255 ? (version = 1) : (version += 1)
+
+        kek = SymmetricEncryption::Key.new(cipher_name: cipher_name)
+        dek ||= SymmetricEncryption::Key.new(cipher_name: cipher_name)
+
+        encrypted_key = new(key_encrypting_key: kek).write(dek.key)
+
+        {
+          cipher_name:        cipher_name,
+          version:            version,
+          encrypted_key:      encrypted_key,
+          iv:                 iv,
+          key_encrypting_key: {
+            key: kek.key,
+            iv:  kek.iv,
+          }
+        }
+      end
+
+      # Stores the Encryption key in a string.
+      # Secures the Encryption key by encrypting it with a key encryption key.
+      def initialize(encrypted_key: nil, key_encrypting_key:)
+        @encrypted_key      = encrypted_key
+        @key_encrypting_key = key_encrypting_key
+      end
+
+      # Returns the Encryption key in the clear.
+      def read
+        key_encrypting_key.decrypt(encrypted_key)
+      end
+
+      # Write the encrypted Encryption key to `encrypted_key` attribute.
+      def write(key)
+        self.encrypted_key = key_encrypting_key.encrypt(key)
+      end
+
+    end
+  end
+end