symmetric-encryption 4.0.1 → 4.1.0

data/lib/symmetric_encryption/utils/aws.rb

@@ -0,0 +1,141 @@
+require 'base64'
+require 'aws-sdk-kms'
+module SymmetricEncryption
+  module Utils
+    # Wrap the AWS KMS client so that it automatically creates the Customer Master Key,
+    # if one does not already exist.
+    #
+    # Map OpenSSL cipher names to AWS KMS key specs.
+    class Aws
+      attr_reader :master_key_alias, :client
+
+      AWS_US_REGIONS = %w[us-east-1 us-east-2 us-west-1 us-west-2].freeze
+
+      # TODO: Map to OpenSSL ciphers
+      AWS_KEY_SPEC_MAP = {
+        'aes-256-cbc' => 'AES_256',
+        'aes-128-cbc' => 'AES_128'
+      }.freeze
+
+      # TODO: Move to Keystore::Aws
+      # Rotate the Customer Master key in each of the supplied regions.
+      # After the master key has been rotated, use `.write_key_files` to generate
+      # a new DEK and re-encrypt with the new CMK in each region.
+      # def self.rotate_master_key(master_key_alias:, cipher_name:, regions: AWS_US_REGIONS)
+      #   Array(regions).collect do |region|
+      #     key_manager = new(region: region, master_key_alias: master_key_alias, cipher_name: cipher_name)
+      #     key_id      = key_manager.create_master_key
+      #     key_manager.create_alias(key_id)
+      #   end
+      # end
+
+      def initialize(region:, master_key_alias:)
+        # Can region be read from environment?
+        # Region is required for filename / env var name
+        @client           = ::Aws::KMS::Client.new(region: region)
+        @master_key_alias = master_key_alias
+      end
+
+      # Returns a new DEK encrypted using the CMK
+      def generate_encrypted_data_key(cipher_name)
+        auto_create_master_key do
+          client.generate_data_key_without_plaintext(key_id: master_key_alias, key_spec: key_spec(cipher_name)).ciphertext_blob
+        end
+      end
+
+      # Returns a new DEK in the clear
+      def generate_data_key(cipher_name)
+        auto_create_master_key do
+          client.generate_data_key(key_id: master_key_alias, key_spec: key_spec(cipher_name)).plaintext
+        end
+      end
+
+      # Decrypt data previously encrypted using the cmk
+      def decrypt(encrypted_data)
+        auto_create_master_key do
+          client.decrypt(ciphertext_blob: encrypted_data).plaintext
+        end
+      end
+
+      # Decrypt data previously encrypted using the cmk
+      def encrypt(data)
+        auto_create_master_key do
+          client.encrypt(key_id: master_key_alias, plaintext: data).ciphertext_blob
+        end
+      end
+
+      # Returns the AWS KMS key spec that matches the supplied OpenSSL cipher name
+      def key_spec(cipher_name)
+        key_spec = AWS_KEY_SPEC_MAP[cipher_name]
+        raise("OpenSSL Cipher: #{cipher_name} has not yet been mapped to an AWS key spec.") unless key_spec
+
+        key_spec
+      end
+
+      # Creates a new master key along with an alias that points to it.
+      # Returns [String] the new master key id that was created.
+      def create_master_key
+        key_id = create_new_master_key
+        create_alias(key_id)
+        key_id
+      end
+
+      # Deletes the current master key and its alias.
+      #
+      # retention_days: Number of days to keep the CMK before completely destroying it.
+      #
+      # NOTE:
+      #   Use with caution, only intended for testing purposes !!!
+      def delete_master_key(retention_days: 30)
+        key_info = client.describe_key(key_id: master_key_alias)
+        ap key_info
+        resp = client.schedule_key_deletion(key_id: key_info.key_metadata.key_id, pending_window_in_days: retention_days)
+        ap client.delete_alias(alias_name: master_key_alias)
+        resp.deletion_date
+      rescue ::Aws::KMS::Errors::NotFoundException
+        nil
+      end
+
+      private
+
+      attr_reader :client
+
+      def whoami
+        @whoami ||= `whoami`.strip
+      rescue StandardError
+        @whoami = 'unknown'
+      end
+
+      # Creates a new Customer Master Key for Symmetric Encryption use.
+      def create_new_master_key
+        # TODO: Add error handling and retry
+
+        resp = client.create_key(
+          description: 'Symmetric Encryption for Ruby Customer Masker Key',
+          tags:        [
+            {tag_key: 'CreatedAt', tag_value: Time.now.to_s},
+            {tag_key: 'CreatedBy', tag_value: whoami}
+          ]
+        )
+        resp.key_metadata.key_id
+      end
+
+      def create_alias(key_id)
+        # TODO: Add error handling and retry
+        # TODO: Move existing alias if any
+        client.create_alias(alias_name: master_key_alias, target_key_id: key_id)
+      end
+
+      def auto_create_master_key
+        attempt = 1
+        yield
+      rescue ::Aws::KMS::Errors::NotFoundException
+        raise if attempt >= 2
+
+        create_master_key
+        attempt += 1
+        retry
+      end
+    end
+  end
+end

data/lib/symmetric_encryption/utils/re_encrypt_files.rb

@@ -52,16 +52,24 @@ module SymmetricEncryption
       def re_encrypt_contents(file_name)
         return 0 if File.size(file_name) > 256 * 1024
 
+        lines              = File.read(file_name)
+        hits, output_lines = re_encrypt_lines(lines)
+
+        File.open(file_name, 'wb') { |file| file.write(output_lines) } if hits.positive?
+        hits
+      end
+
+      # Replaces instances of encrypted data within lines of text with re-encrypted values
+      def re_encrypt_lines(lines)
         hits         = 0
-        lines        = File.read(file_name)
         output_lines = ''
         r            = regexp
         lines.each_line do |line|
           line.force_encoding(SymmetricEncryption::UTF8_ENCODING)
           output_lines <<
             if line.valid_encoding? && (result = line.match(r))
-              encrypted = result[0]
-              new_value = re_encrypt(encrypted)
+              encrypted                        = result[0]
+              new_value                        = re_encrypt(encrypted)
               if new_value != encrypted
                 hits += 1
                 line.gsub(encrypted, new_value)
@@ -72,8 +80,7 @@ module SymmetricEncryption
               line
             end
         end
-        File.open(file_name, 'wb') { |file| file.write(output_lines) } if hits.positive?
-        hits
+        [hits, output_lines]
       end
 
       # Re Encrypt an entire file

data/lib/symmetric_encryption/version.rb

@@ -1,3 +1,3 @@
 module SymmetricEncryption
-  VERSION = '4.0.1'.freeze
+  VERSION = '4.1.0'.freeze
 end

data/lib/symmetric_encryption/writer.rb

@@ -21,7 +21,7 @@ module SymmetricEncryption
     #   compress: [true|false]
     #     Uses Zlib to compress the data before it is encrypted and
     #     written to the file/stream.
-    #     Default: false
+    #     Default: true, unless the file_name extension indicates it is already compressed.
     #
     # Note: Compression occurs before encryption
     #
@@ -47,11 +47,16 @@ module SymmetricEncryption
     #  ensure
     #    csv.close if csv
     #  end
-    def self.open(file_name_or_stream, compress: false, **args)
-      ios = file_name_or_stream.is_a?(String) ? ::File.open(file_name_or_stream, 'wb') : file_name_or_stream
+    def self.open(file_name_or_stream, compress: nil, **args)
+      if file_name_or_stream.is_a?(String)
+        file_name_or_stream = ::File.open(file_name_or_stream, 'wb')
+        compress            = !(/\.(zip|gz|gzip|xls.|)\z/i === file_name_or_stream) if compress.nil?
+      else
+        compress = true if compress.nil?
+      end
 
       begin
-        file = new(ios, compress: compress, **args)
+        file = new(file_name_or_stream, compress: compress, **args)
         file = Zlib::GzipWriter.new(file) if compress
         block_given? ? yield(file) : file
       ensure
@@ -64,7 +69,7 @@ module SymmetricEncryption
     # Notes:
     # * Do not use this method for writing large files.
     def self.write(file_name_or_stream, data, **args)
-      self.open(file_name_or_stream, **args) { |f| f.write(data) }
+      Writer.open(file_name_or_stream, **args) { |f| f.write(data) }
     end
 
     # Encrypt an entire file.
@@ -82,22 +87,10 @@ module SymmetricEncryption
     #     Whether to compress the target file prior to encryption.
     #     Default: false
     #
-    #   block_size: [Integer]
-    #     Number of bytes to read into memory for each read.
-    #     For very large files using a larger block size is faster.
-    #     Default: 65535
-    #
     # Notes:
     # * The file contents are streamed so that the entire file is _not_ loaded into memory.
-    def self.encrypt(source:, target:, block_size: 65_535, **args)
-      source_ios    = source.is_a?(String) ? ::File.open(source, 'rb') : source
-      bytes_written = 0
-      self.open(target, **args) do |output_file|
-        bytes_written += output_file.write(source_ios.read(block_size)) until source_ios.eof?
-      end
-      bytes_written
-    ensure
-      source_ios.close if source_ios&.respond_to?(:closed?) && !source_ios.closed?
+    def self.encrypt(source:, target:, **args)
+      Writer.open(target, **args) { |output_file| IO.copy_stream(source, output_file) }
     end
 
     # Encrypt data before writing to the supplied stream
@@ -149,6 +142,7 @@ module SymmetricEncryption
     # ensure that the encrypted stream is closed before the stream itself is closed.
     def close(close_child_stream = true)
       return if closed?
+
       if size.positive?
         final = @stream_cipher.final
         @ios.write(final) unless final.empty?
@@ -160,14 +154,26 @@ module SymmetricEncryption
     # Write to the IO Stream as encrypted data.
     #
     # Returns [Integer] the number of bytes written.
-    def write(data)
-      return unless data
-
-      bytes   = data.to_s
-      @size   += bytes.size
-      partial = @stream_cipher.update(bytes)
-      @ios.write(partial) unless partial.empty?
-      data.length
+    if defined?(JRuby)
+      def write(data)
+        return unless data
+
+        bytes   = data.to_s
+        @size  += bytes.size
+        partial = @stream_cipher.update(bytes)
+        @ios.write(partial) unless partial.empty?
+        data.length
+      end
+    else
+      def write(data)
+        return unless data
+
+        bytes   = data.to_s
+        @size  += bytes.size
+        partial = @stream_cipher.update(bytes, @cipher_buffer ||= ''.b)
+        @ios.write(partial) unless partial.empty?
+        data.length
+      end
     end
 
     # Write to the IO Stream as encrypted data.

data/lib/symmetric_encryption.rb

@@ -22,19 +22,40 @@ module SymmetricEncryption
   autoload :CLI,                    'symmetric_encryption/cli'
   autoload :Keystore,               'symmetric_encryption/keystore'
   module Utils
-    autoload :Generate,             'symmetric_encryption/utils/generate'
+    autoload :Aws,                  'symmetric_encryption/utils/aws'
     autoload :ReEncryptFiles,       'symmetric_encryption/utils/re_encrypt_files'
   end
 end
 # @formatter:on
 
 # Add support for other libraries only if they have already been loaded
-require 'symmetric_encryption/railtie' if defined?(Rails)
-if defined?(ActiveRecord::Base) && !defined?(AttrEncrypted::Version)
-  require 'symmetric_encryption/extensions/active_record/base'
+
+begin
+  require 'rails/railtie'
+  require 'symmetric_encryption/railtie' if defined?(Rails)
+rescue LoadError
 end
-require 'symmetric_encryption/railties/symmetric_encryption_validator' if defined?(ActiveModel)
-require 'symmetric_encryption/extensions/mongoid/encrypted' if defined?(Mongoid)
+
+begin
+  require 'active_record'
+  if defined?(ActiveRecord::Base) && !defined?(AttrEncrypted::Version)
+    require 'symmetric_encryption/extensions/active_record/base'
+  end
+rescue LoadError
+end
+
+begin
+  require 'active_model'
+  require 'symmetric_encryption/railties/symmetric_encryption_validator' if defined?(ActiveModel)
+rescue LoadError
+end
+
+begin
+  require 'mongoid'
+  require 'symmetric_encryption/extensions/mongoid/encrypted' if defined?(Mongoid)
+rescue LoadError
+end
+
 if defined?(MongoMapper)
   warn 'MongoMapper support is deprecated. Upgrade to Mongoid.'
   require 'symmetric_encryption/extensions/mongo_mapper/plugins/encrypted_key'

data/test/active_record_test.rb

@@ -119,26 +119,26 @@ class ActiveRecordTest < Minitest::Test
     let :user do
       User.new(
         # Encrypted Attribute
-        bank_account_number: bank_account_number,
+        bank_account_number:    bank_account_number,
         # Encrypted Attribute
         social_security_number: social_security_number,
         name:                   person_name,
         # data type specific fields
-        string_value:        STRING_VALUE,
-        long_string_value:   LONG_STRING_VALUE,
-        binary_string_value: BINARY_STRING_VALUE,
-        integer_value:       INTEGER_VALUE,
-        float_value:         FLOAT_VALUE,
-        decimal_value:       DECIMAL_VALUE,
-        datetime_value:      DATETIME_VALUE,
-        time_value:          TIME_VALUE,
-        date_value:          DATE_VALUE,
-        true_value:          true,
-        false_value:         false,
-        data_yaml:           hash_data.dup,
-        data_json:           hash_data.dup,
-        text:                'hello',
-        number:              '21'
+        string_value:           STRING_VALUE,
+        long_string_value:      LONG_STRING_VALUE,
+        binary_string_value:    BINARY_STRING_VALUE,
+        integer_value:          INTEGER_VALUE,
+        float_value:            FLOAT_VALUE,
+        decimal_value:          DECIMAL_VALUE,
+        datetime_value:         DATETIME_VALUE,
+        time_value:             TIME_VALUE,
+        date_value:             DATE_VALUE,
+        true_value:             true,
+        false_value:            false,
+        data_yaml:              hash_data.dup,
+        data_json:              hash_data.dup,
+        text:                   'hello',
+        number:                 '21'
       )
     end
 
@@ -167,17 +167,17 @@ class ActiveRecordTest < Minitest::Test
     describe ':random_iv' do
       it 'false' do
         user.social_security_number = social_security_number
-        assert first_value = user.social_security_number
+        assert first_value          = user.social_security_number
         # Assign the same value
         user.social_security_number = social_security_number
         assert_equal first_value, user.social_security_number
       end
 
       it 'true' do
-        user.string_value = STRING_VALUE
+        user.string_value  = STRING_VALUE
         assert first_value = user.encrypted_string_value
-        user.string_value = 'blah'
-        user.string_value = STRING_VALUE
+        user.string_value  = 'blah'
+        user.string_value  = STRING_VALUE
         refute_equal first_value, user.encrypted_string_value
       end
 
@@ -200,7 +200,7 @@ class ActiveRecordTest < Minitest::Test
 
         it 'does not change when equal' do
           user.save!
-          before             = user.encrypted_string_value
+          before            = user.encrypted_string_value
           user.string_value = STRING_VALUE
           refute user.string_value_changed?
           assert_equal before, user.encrypted_string_value
@@ -405,7 +405,7 @@ class ActiveRecordTest < Minitest::Test
 
       describe '#reload' do
         it 'reverts changes' do
-          new_bank_account_number   = '444444444'
+          new_bank_account_number  = '444444444'
           user.bank_account_number = new_bank_account_number
           assert_equal new_bank_account_number, user.bank_account_number
 
@@ -416,8 +416,8 @@ class ActiveRecordTest < Minitest::Test
         end
 
         it 'reverts changes to encrypted field' do
-          new_bank_account_number             = '111111111'
-          new_encrypted_bank_account_number   = SymmetricEncryption.encrypt(new_bank_account_number)
+          new_bank_account_number            = '111111111'
+          new_encrypted_bank_account_number  = SymmetricEncryption.encrypt(new_bank_account_number)
           user.encrypted_bank_account_number = new_encrypted_bank_account_number
           assert_equal new_encrypted_bank_account_number, user.encrypted_bank_account_number
           assert_equal new_bank_account_number, user.bank_account_number
@@ -591,7 +591,7 @@ class ActiveRecordTest < Minitest::Test
         UniqueUser.destroy_all
         @email      = 'whatever@not-unique.com'
         @username   = 'gibby007'
-        user       = UniqueUser.create!(email: @email)
+        user        = UniqueUser.create!(email: @email)
         @email_user = UniqueUser.create!(username: @username)
       end
 

data/test/cipher_test.rb

@@ -93,7 +93,7 @@ class CipherTest < Minitest::Test
                 always_add_header: always_add_header
               )
 
-              h         = @encrypted_values[cipher_name][encoding] if @encrypted_values[cipher_name]
+              h = @encrypted_values[cipher_name][encoding] if @encrypted_values[cipher_name]
               skip "Add @encrypted_values for cipher_name: #{cipher_name} and encoding: #{encoding}, value: #{@cipher.encrypt(@social_security_number).inspect}" unless h
               @social_security_number_encrypted = h[always_add_header ? :header : :no_header]
 
@@ -150,7 +150,7 @@ class CipherTest < Minitest::Test
 
       describe 'with configuration' do
         before do
-          @cipher                 = SymmetricEncryption::Cipher.new(
+          @cipher = SymmetricEncryption::Cipher.new(
             key:         '1234567890ABCDEF',
             iv:          '1234567890ABCDEF',
             cipher_name: 'aes-128-cbc',
@@ -181,7 +181,7 @@ class CipherTest < Minitest::Test
               key.key,
               key.cipher_name
             )
-            header        = SymmetricEncryption::Header.new
+            header = SymmetricEncryption::Header.new
             header.parse(binary_header)
             assert_equal true, header.compressed?
             assert random_cipher = SymmetricEncryption::Cipher.new(iv: key.iv, key: key.key, cipher_name: key.cipher_name)

data/test/header_test.rb

@@ -179,7 +179,7 @@ class CipherTest < Minitest::Test
         end
 
         it 'encrypted string' do
-          header = SymmetricEncryption::Header.new
+          header           = SymmetricEncryption::Header.new
           assert remainder = header.parse!(binary_encrypted_value)
           assert_equal SymmetricEncryption.cipher.version, header.version
           refute header.compressed?

data/test/key_test.rb

@@ -2,15 +2,6 @@ require_relative 'test_helper'
 
 class KeyTest < Minitest::Test
   describe SymmetricEncryption::Key do
-    before do
-      Dir.mkdir('tmp') unless Dir.exist?('tmp')
-    end
-
-    after do
-      # Cleanup generated encryption key files.
-      `rm tmp/dek_tester* 2> /dev/null`
-    end
-
     let :random_key do
       SymmetricEncryption::Key.new
     end
@@ -27,30 +18,6 @@ class KeyTest < Minitest::Test
       SymmetricEncryption::Key.new(key: stored_key, iv: stored_iv)
     end
 
-    let :stored_key2 do
-      'ABCDEF1234567890ABCDEF1234567890'
-    end
-
-    let :stored_iv2 do
-      '1234567890ABCDEF'
-    end
-
-    let :key2 do
-      SymmetricEncryption::Key.new(key: stored_key2, iv: stored_iv2)
-    end
-
-    let :stored_key3 do
-      'ABCDEF0123456789ABCDEF0123456789'
-    end
-
-    let :stored_iv3 do
-      '0123456789ABCDEF'
-    end
-
-    let :key3 do
-      SymmetricEncryption::Key.new(key: stored_key3, iv: stored_iv3)
-    end
-
     let :ssn do
       '987654321'
     end
@@ -110,129 +77,5 @@ class KeyTest < Minitest::Test
         assert_equal stored_iv, key.iv
       end
     end
-
-    describe '.from_config' do
-      let :config do
-        {key: stored_key, iv: stored_iv}
-      end
-
-      let :config_key do
-        SymmetricEncryption::Key.from_config(config)
-      end
-
-      let :dek_file_name do
-        'tmp/dek_tester_dek.encrypted_key'
-      end
-
-      describe 'key' do
-        it 'key' do
-          assert_equal stored_key, config_key.key
-        end
-
-        it 'iv' do
-          assert_equal stored_iv, config_key.iv
-        end
-
-        it 'cipher_name' do
-          assert_equal 'aes-256-cbc', config_key.cipher_name
-        end
-      end
-
-      describe 'encrypted_key' do
-        let :config do
-          {encrypted_key: key2.encrypt(stored_key), iv: stored_iv, key_encrypting_key: {key: stored_key2, iv: stored_iv2}}
-        end
-
-        it 'key' do
-          assert_equal stored_key, config_key.key
-        end
-
-        it 'iv' do
-          assert_equal stored_iv, config_key.iv
-        end
-
-        it 'cipher_name' do
-          assert_equal 'aes-256-cbc', config_key.cipher_name
-        end
-      end
-
-      describe 'key_filename' do
-        let :config do
-          File.open(dek_file_name, 'wb') { |f| f.write(key2.encrypt(stored_key)) }
-          {key_filename: dek_file_name, iv: stored_iv, key_encrypting_key: {key: stored_key2, iv: stored_iv2}}
-        end
-
-        it 'key' do
-          assert_equal stored_key, config_key.key
-        end
-
-        it 'iv' do
-          assert_equal stored_iv, config_key.iv
-        end
-
-        it 'cipher_name' do
-          assert_equal 'aes-256-cbc', config_key.cipher_name
-        end
-      end
-
-      describe 'key_env_var' do
-        let :env_var do
-          'TEST_KEY'
-        end
-
-        let :config do
-          ENV[env_var] = ::Base64.encode64(key2.encrypt(stored_key))
-          {key_env_var: env_var, iv: stored_iv, key_encrypting_key: {key: stored_key2, iv: stored_iv2}}
-        end
-
-        it 'key' do
-          assert_equal stored_key, config_key.key
-        end
-
-        it 'iv' do
-          assert_equal stored_iv, config_key.iv
-        end
-
-        it 'cipher_name' do
-          assert_equal 'aes-256-cbc', config_key.cipher_name
-        end
-      end
-
-      describe 'file store with kekek' do
-        let :kekek_file_name do
-          'tmp/tester_kekek.key'
-        end
-
-        let :config do
-          File.open(dek_file_name, 'wb') { |f| f.write(key2.encrypt(stored_key)) }
-          encrypted_key = key3.encrypt(stored_key2)
-          File.open(kekek_file_name, 'wb') { |f| f.write(stored_key3) }
-          {
-            key_filename:       dek_file_name,
-            iv:                 stored_iv,
-            key_encrypting_key: {
-              encrypted_key:      encrypted_key,
-              iv:                 stored_iv2,
-              key_encrypting_key: {
-                key_filename: kekek_file_name,
-                iv:           stored_iv3
-              }
-            }
-          }
-        end
-
-        it 'key' do
-          assert_equal stored_key, config_key.key
-        end
-
-        it 'iv' do
-          assert_equal stored_iv, config_key.iv
-        end
-
-        it 'cipher_name' do
-          assert_equal 'aes-256-cbc', config_key.cipher_name
-        end
-      end
-    end
   end
 end