symmetric-encryption 3.9.1 → 4.0.0.beta3

data/lib/symmetric_encryption/utils/re_encrypt_files.rb

@@ -0,0 +1,140 @@
+# Used for re-encrypting encrypted passwords stored in configuration files.
+#
+# Search for any encrypted value and re-encrypt it using the latest encryption key.
+# Note:
+# * Only works with encrypted values that have the standard header.
+#   * The search looks for the header and then replaces the encrypted value.
+#
+# Example:
+#   re_encrypt = SymmetricEncryption::Utils::ReEncryptConfigFiles.new(version: 4)
+#   re_encrypt.process_directory('../../**/*.yml')
+#
+# Notes:
+# * Only supports the output from encrypting data.
+#   * I.e. Manually adding newlines to base 64 output is not supported.
+# * For now only supports one encrypted value per line.
+module SymmetricEncryption
+  module Utils
+    # ReEncrypt files
+    #
+    #   If a file is encrypted, it is re-encrypted with the cipher that has the highest version number.
+    #   A file that is already encrypted with the specified key version is not re-encrypted.
+    #   If an encrypted value cannot be decypted in the current environment it is left unmodified.
+    #
+    #   If a file is not encrypted, the file is searched for any encrypted values, and those values are re-encrypted.
+    #
+    #   symmetric_encryption --reencrypt "**/*.yml"
+    class ReEncryptFiles
+      attr_accessor :cipher, :version
+
+      # Parameters:
+      #   version: [Integer]
+      #     Version of the encryption key to use when re-encrypting the value.
+      #     Default: Default cipher ( first in the list of configured ciphers )
+      def initialize(version: SymmetricEncryption.cipher.version)
+        @version = version || SymmetricEncryption.cipher.version
+        @cipher  = SymmetricEncryption.cipher(@version)
+        raise(ArgumentError, "Undefined encryption key version: #{version}") if @cipher.nil?
+      end
+
+      # Re-encrypt the supplied encrypted value with the new cipher
+      def re_encrypt(encrypted)
+        if unencrypted = SymmetricEncryption.try_decrypt(encrypted)
+          cipher.encrypt(unencrypted)
+        else
+          encrypted
+        end
+      end
+
+      # Process a single file.
+      #
+      # Returns [Integer] number of encrypted values re-encrypted.
+      def re_encrypt_contents(file_name)
+        return 0 if File.size(file_name) > 256 * 1024
+
+        hits         = 0
+        lines        = File.read(file_name)
+        output_lines = ''
+        r            = regexp
+        lines.each_line do |line|
+          line.force_encoding(SymmetricEncryption::UTF8_ENCODING)
+          output_lines <<
+            if line.valid_encoding? && (result = line.match(r))
+              encrypted = result[0]
+              new_value = re_encrypt(encrypted)
+              if new_value != encrypted
+                hits += 1
+                line.gsub(encrypted, new_value)
+              else
+                line
+              end
+            else
+              line
+            end
+        end
+        if hits
+          File.open(file_name, 'wb') { |file| file.write(output_lines) }
+        end
+        hits
+      rescue
+        puts "Failed re-encrypting the file contents of: #{file_name}"
+        raise
+      end
+
+      # Re Encrypt an entire file
+      def re_encrypt_file(file_name)
+        temp_file_name = "__re_encrypting_#{file_name}"
+        SymmetricEncryption::Reader.open(file_name) do |source|
+          SymmetricEncryption::Writer.encrypt(source: source, target: temp_file_name, compress: true, version: version)
+        end
+        File.delete(file_name)
+        File.rename(temp_file_name, file_name)
+      rescue
+        File.delete(temp_file_name) if temp_file_name && File.exist?(temp_file_name)
+        raise
+      end
+
+      # Process a directory of files.
+      #
+      # Parameters:
+      #   path: [String]
+      #     Search path to look for files in.
+      #     Example: '../../**/*.yml'
+      def process_directory(path)
+        Dir[path].each do |file_name|
+          next if File.directory?(file_name)
+
+          if v = encrypted_file_version(file_name)
+            if v == version
+              puts "Skipping already re-encrypted file: #{file_name}"
+            else
+              puts "Re-encrypting entire file: #{file_name}"
+              re_encrypt_file(file_name)
+            end
+          else
+            count = re_encrypt_contents(file_name)
+            puts "Re-encrypted #{count} encrypted value(s) in: #{file_name}" if count > 0
+          end
+        end
+      end
+
+      private
+
+      def regexp
+        @regexp ||= /#{SymmetricEncryption.cipher.encoded_magic_header}([A-Za-z0-9+\/]+=+[\\n]*)/
+      end
+
+      # Returns [Integer] encrypted file key version.
+      # Returns [nil] if the file is not encrypted or does not have a header.
+      def encrypted_file_version(file_name)
+        ::File.open(file_name, 'rb') do |file|
+          reader = SymmetricEncryption::Reader.new(file)
+          reader.version if reader.header_present?
+        end
+      rescue OpenSSL::Cipher::CipherError
+        nil
+      end
+
+    end
+  end
+end

data/lib/symmetric_encryption/version.rb

@@ -1,3 +1,3 @@
 module SymmetricEncryption #:nodoc
-  VERSION = '3.9.1'
+  VERSION = '4.0.0.beta3'
 end

data/lib/symmetric_encryption/writer.rb

@@ -1,89 +1,38 @@
 require 'openssl'
 
 module SymmetricEncryption
-  # Write to encrypted files and other IO streams
+  # Write to encrypted files and other IO streams.
   #
   # Features:
   # * Encryption on the fly whilst writing files.
-  # * Large file support by only buffering small amounts of data in memory
+  # * Large file support by only buffering small amounts of data in memory.
   # * Underlying buffering to ensure that encrypted data fits
-  #   into the Symmetric Encryption Cipher block size
-  #   Only the last block in the file will be padded if it is less than the block size
+  #   into the Symmetric Encryption Cipher block size.
+  #   Only the last block in the file will be padded if it is less than the block size.
   class Writer
-    # Open a file for writing, or use the supplied IO Stream
+    # Open a file for writing, or use the supplied IO Stream.
     #
     # Parameters:
-    #   filename_or_stream:
-    #     The filename to open if a string, otherwise the stream to use
+    #   file_name_or_stream: [String|IO]
+    #     The file_name to open if a string, otherwise the stream to use.
     #     The file or stream will be closed on completion, use .initialize to
-    #     avoid having the stream closed automatically
-    #
-    #   options:
-    #     :compress [true|false]
-    #          Uses Zlib to compress the data before it is encrypted and
-    #          written to the file
-    #          If true, it forces header to true.
-    #          Default: false
-    #
-    #     :random_key [true|false]
-    #          Generates a new random key for every new file or stream
-    #          If true, it forces header to true. Version below then has no effect
-    #          The Random key will be written to the file/stream in encrypted
-    #          form as part of the header
-    #          The key is encrypted using the global key
-    #          Default: true
-    #          Recommended: true.
-    #            Setting to false will eventually expose the
-    #            encryption key since too much data will be encrypted using the
-    #            same encryption key
-    #
-    #     :random_iv [true|false]
-    #          Generates a new random iv for every new file or stream
-    #          If true, it forces header to true.
-    #          The Random iv will be written to the file/stream in encrypted
-    #          form as part of the header
-    #          Default: Value supplied above for :random_key
-    #          Recommended: true. Setting to false will eventually expose the
-    #            encryption key since too much data will be encrypted using the
-    #            same encryption key
-    #
-    #     :header [true|false]
-    #          Whether to include the magic header that indicates the file
-    #          is encrypted and whether its contents are compressed
-    #
-    #          The header contains:
-    #             Version of the encryption key used to encrypt the file
-    #             Indicator if the data was compressed
-    #          Default: true
-    #
-    #     :version
-    #          When random_key is true, the version of the encryption key to use
-    #          when encrypting the header portion of the file
-    #
-    #          When random_key is false, the version of the encryption key to use
-    #          to encrypt the entire file
-    #          Default: SymmetricEncryption.cipher
-    #
-    #     :mode
-    #          See File.open for open modes
-    #          Default: 'w'
-    #
-    #     :cipher_name
-    #          The name of the cipher to use only if both :random_key and
-    #          :random_iv are true.
-    #          Default: SymmetricEncryption.cipher.cipher_name
+    #     avoid having the stream closed automatically.
     #
-    # Note: Compression occurs before encryption
+    #   compress: [true|false]
+    #     Uses Zlib to compress the data before it is encrypted and
+    #     written to the file/stream.
+    #     Default: false
     #
+    # Note: Compression occurs before encryption
     #
     # # Example: Encrypt and write data to a file
-    # SymmetricEncryption::Writer.open('test_file') do |file|
+    # SymmetricEncryption::Writer.open('test_file.enc') do |file|
     #   file.write "Hello World\n"
     #   file.write 'Keep this secret'
     # end
     #
     # # Example: Compress, Encrypt and write data to a file
-    # SymmetricEncryption::Writer.open('encrypted_compressed.zip', compress: true) do |file|
+    # SymmetricEncryption::Writer.open('encrypted_compressed.enc', compress: true) do |file|
     #   file.write "Hello World\n"
     #   file.write "Compress this\n"
     #   file.write "Keep this safe and secure\n"
@@ -93,80 +42,115 @@ module SymmetricEncryption
     #  require 'csv'
     #  begin
     #    # Must supply :row_sep for CSV otherwise it will attempt to read from and then rewind the file
-    #    csv = CSV.new(SymmetricEncryption::Writer.open('csv_encrypted'), row_sep: "\n")
+    #    csv = CSV.new(SymmetricEncryption::Writer.open('csv.enc'), row_sep: "\n")
     #    csv << [1,2,3,4,5]
     #  ensure
     #    csv.close if csv
     #  end
-    def self.open(filename_or_stream, options={}, &block)
-      raise(ArgumentError, 'options must be a hash') unless options.respond_to?(:each_pair)
-      mode     = options.fetch(:mode, 'wb')
-      compress = options.fetch(:compress, false)
-      ios      = filename_or_stream.is_a?(String) ? ::File.open(filename_or_stream, mode) : filename_or_stream
+    def self.open(file_name_or_stream, compress: false, **args)
+      ios = file_name_or_stream.is_a?(String) ? ::File.open(file_name_or_stream, 'wb') : file_name_or_stream
 
       begin
-        file = self.new(ios, options)
+        file = self.new(ios, compress: compress, **args)
         file = Zlib::GzipWriter.new(file) if compress
-        block ? block.call(file) : file
+        block_given? ? yield(file) : file
       ensure
-        file.close if block && file && (file.respond_to?(:closed?) && !file.closed?)
+        file.close if block_given? && file && (file.respond_to?(:closed?) && !file.closed?)
+      end
+    end
+
+    # Write the contents of a string in memory to an encrypted file / stream.
+    #
+    # Notes:
+    # * Do not use this method for writing large files.
+    def self.write(file_name_or_stream, data, **args)
+      open(file_name_or_stream, **args) { |f| f.write(data) }
+    end
+
+    # Encrypt an entire file.
+    #
+    # Returns [Integer] the number of encrypted bytes written to the target file.
+    #
+    # Params:
+    #   source: [String|IO]
+    #     Source file_name or IOStream
+    #
+    #   target: [String|IO]
+    #     Target file_name or IOStream
+    #
+    #   compress: [true|false]
+    #     Whether to compress the target file prior to encryption.
+    #     Default: false
+    #
+    #   block_size: [Integer]
+    #     Number of bytes to read into memory for each read.
+    #     For very large files using a larger block size is faster.
+    #     Default: 65535
+    #
+    # Notes:
+    # * The file contents are streamed so that the entire file is _not_ loaded into memory.
+    def self.encrypt(source:, target:, block_size: 65535, **args)
+      source_ios    = source.is_a?(String) ? ::File.open(source, 'rb') : source
+      bytes_written = 0
+      open(target, **args) do |output_file|
+        while !source_ios.eof?
+          bytes_written += output_file.write(source_ios.read(block_size))
+        end
       end
+      bytes_written
+    ensure
+      source_ios.close if source_ios && source_ios.respond_to?(:closed?) && !source_ios.closed?
     end
 
     # Encrypt data before writing to the supplied stream
-    def initialize(ios, options={})
-      @ios       = ios
-      header     = options.fetch(:header, true)
-      random_key = options.fetch(:random_key, true)
-      random_iv  = options.fetch(:random_iv, random_key)
-      raise(ArgumentError, 'When :random_key is true, :random_iv must also be true') if random_key && !random_iv
+    def initialize(ios, version: nil, cipher_name: nil, header: true, random_key: true, random_iv: true, compress: false)
       # Compress is only used at this point for setting the flag in the header
-      compress    = options.fetch(:compress, false)
-      version     = options[:version]
-      cipher_name = options[:cipher_name]
+      @ios = ios
+      raise(ArgumentError, 'When :random_key is true, :random_iv must also be true') if random_key && !random_iv
       raise(ArgumentError, 'Cannot supply a :cipher_name unless both :random_key and :random_iv are true') if cipher_name && !random_key && !random_iv
 
-      # Force header if compressed or using random iv, key
-      header = true if compress || random_key || random_iv
-
       # Cipher to encrypt the random_key, or the entire file
       cipher = SymmetricEncryption.cipher(version)
       raise(SymmetricEncryption::CipherError, "Cipher with version:#{version} not found in any of the configured SymmetricEncryption ciphers") unless cipher
 
+      # Force header if compressed or using random iv, key
+      if (header == true) || compress || random_key || random_iv
+        header = Header.new(version: cipher.version, compress: compress, cipher_name: cipher_name)
+      end
+
       @stream_cipher = ::OpenSSL::Cipher.new(cipher_name || cipher.cipher_name)
       @stream_cipher.encrypt
 
-      key = random_key ? @stream_cipher.random_key : cipher.send(:key)
-      iv  = random_iv ? @stream_cipher.random_iv : cipher.send(:iv)
-
-      @stream_cipher.key = key
-      @stream_cipher.iv  = iv if iv
+      if random_key
+        header.key = @stream_cipher.key = @stream_cipher.random_key
+      else
+        @stream_cipher.key = cipher.send(:key)
+      end
 
-      # Write the Encryption header including the random iv, key, and cipher
-      if header
-        @ios.write(Cipher.build_header(
-            cipher.version,
-            compress,
-            random_iv ? iv : nil,
-            random_key ? key : nil,
-            cipher_name))
+      if random_iv
+        header.iv = @stream_cipher.iv = @stream_cipher.random_iv
+      else
+        @stream_cipher.iv = cipher.iv if cipher.iv
       end
+
+      @ios.write(header.to_s) if header
+
       @size   = 0
       @closed = false
     end
 
-    # Close the IO Stream
-    # Flushes any unwritten data
+    # Close the IO Stream.
     #
-    # Note: Once an EncryptionWriter has been closed a new instance must be
-    #       created before writing again
-    #
-    # Note: Also closes the passed in io stream or file
-    # Note: This method must be called _before_ the supplied stream is closed
+    # Notes:
+    # * Flushes any unwritten data.
+    # * Once an EncryptionWriter has been closed a new instance must be
+    #   created before writing again.
+    # * Closes the passed in io stream or file.
+    # * `close` must be called _before_ the supplied stream is closed.
     #
     # It is recommended to call Symmetric::EncryptedStream.open
-    # rather than creating an instance of Symmetric::EncryptedStream directly to
-    # ensure that the encrypted stream is closed before the stream itself is closed
+    # rather than creating an instance of Symmetric::Writer directly to
+    # ensure that the encrypted stream is closed before the stream itself is closed.
     def close(close_child_stream = true)
       return if closed?
       if size > 0
@@ -177,8 +161,9 @@ module SymmetricEncryption
       @closed = true
     end
 
-    # Write to the IO Stream as encrypted data
-    # Returns the number of bytes written
+    # Write to the IO Stream as encrypted data.
+    #
+    # Returns [Integer] the number of bytes written.
     def write(data)
       return unless data
 
@@ -189,8 +174,9 @@ module SymmetricEncryption
       data.length
     end
 
-    # Write to the IO Stream as encrypted data
-    # Returns self
+    # Write to the IO Stream as encrypted data.
+    #
+    # Returns [SymmetricEncryption::Writer] self
     #
     # Example:
     #   file << "Hello.\n" << 'This is Jack'
@@ -199,20 +185,21 @@ module SymmetricEncryption
       self
     end
 
-    # Flush the output stream
+    # Flush the output stream.
     # Does not flush internal buffers since encryption requires all data to
-    # be written following the encryption block size
-    #  Needed by XLS gem
+    # be written following the encryption block size.
+    #  Needed by XLS gem.
     def flush
       @ios.flush
     end
 
+    # Returns [true|false] whether this stream is closed.
     def closed?
       @closed || @ios.respond_to?(:closed?) && @ios.closed?
     end
 
     # Returns [Integer] the number of unencrypted and uncompressed bytes
-    # written to the file so far
+    # written to the file so far.
     attr_reader :size
 
   end

data/lib/symmetric_encryption.rb

@@ -13,12 +13,17 @@ module SymmetricEncryption
   autoload :Coerce,                 'symmetric_encryption/coerce'
   autoload :Config,                 'symmetric_encryption/config'
   autoload :Encoder,                'symmetric_encryption/encoder'
-  autoload :KeyEncryptionKey,       'symmetric_encryption/key_encryption_key'
+  autoload :Generator,              'symmetric_encryption/generator'
+  autoload :Header,                 'symmetric_encryption/header'
+  autoload :Key,                    'symmetric_encryption/key'
   autoload :Reader,                 'symmetric_encryption/reader'
+  autoload :RSAKey,                 'symmetric_encryption/rsa_key'
   autoload :Writer,                 'symmetric_encryption/writer'
-  autoload :Generator,              'symmetric_encryption/generator'
+  autoload :CLI,                    'symmetric_encryption/cli'
+  autoload :Keystore,               'symmetric_encryption/keystore'
   module Utils
-    autoload :ReEncryptConfigFiles, 'symmetric_encryption/re_encrypt_config_files'
+    autoload :Generate,             'symmetric_encryption/utils/generate'
+    autoload :ReEncryptFiles,       'symmetric_encryption/utils/re_encrypt_files'
   end
 end
 #@formatter:on
@@ -31,6 +36,6 @@ end
 require 'symmetric_encryption/railties/symmetric_encryption_validator' if defined?(ActiveModel)
 require 'symmetric_encryption/extensions/mongoid/encrypted' if defined?(Mongoid)
 if defined?(MongoMapper)
-  warn 'MongoMapper support is deprecated. Consider upgrading to Mongoid.'
+  warn 'MongoMapper support is deprecated. Upgrade to Mongoid.'
   require 'symmetric_encryption/extensions/mongo_mapper/plugins/encrypted_key'
 end

data/test/active_record_test.rb

@@ -77,15 +77,6 @@ class UniqueUser < ActiveRecord::Base
 end
 #@formatter:on
 
-# Initialize the database connection
-config_file = File.join(File.dirname(__FILE__), 'config', 'database.yml')
-raise 'database config not found. Create a config file at: test/config/database.yml' unless File.exist? config_file
-
-cfg = YAML.load(ERB.new(File.new(config_file).read).result)['test']
-raise("Environment 'test' not defined in test/config/database.yml") unless cfg
-
-User.establish_connection(cfg)
-
 #
 # Unit Test for attr_encrypted extensions in ActiveRecord
 #
@@ -114,26 +105,26 @@ class ActiveRecordTest < Minitest::Test
 
       @user = User.new(
         # Encrypted Attribute
-        bank_account_number:    @bank_account_number,
+        bank_account_number: @bank_account_number,
         # Encrypted Attribute
         social_security_number: @social_security_number,
         name:                   @name,
         # data type specific fields
-        string_value:           STRING_VALUE,
-        long_string_value:      LONG_STRING_VALUE,
-        binary_string_value:    BINARY_STRING_VALUE,
-        integer_value:          INTEGER_VALUE,
-        float_value:            FLOAT_VALUE,
-        decimal_value:          DECIMAL_VALUE,
-        datetime_value:         DATETIME_VALUE,
-        time_value:             TIME_VALUE,
-        date_value:             DATE_VALUE,
-        true_value:             true,
-        false_value:            false,
-        data_yaml:              @h.dup,
-        data_json:              @h.dup,
-        text:                   'hello',
-        number:                 '21'
+        string_value:        STRING_VALUE,
+        long_string_value:   LONG_STRING_VALUE,
+        binary_string_value: BINARY_STRING_VALUE,
+        integer_value:       INTEGER_VALUE,
+        float_value:         FLOAT_VALUE,
+        decimal_value:       DECIMAL_VALUE,
+        datetime_value:      DATETIME_VALUE,
+        time_value:          TIME_VALUE,
+        date_value:          DATE_VALUE,
+        true_value:          true,
+        false_value:         false,
+        data_yaml:           @h.dup,
+        data_json:           @h.dup,
+        text:                'hello',
+        number:              '21'
       )
     end
 
@@ -171,9 +162,9 @@ class ActiveRecordTest < Minitest::Test
       it 'true' do
         @user.string_value = STRING_VALUE
         assert first_value = @user.encrypted_string_value
-        # Assign the same value
-        @user.string_value = STRING_VALUE.dup
-        assert first_value != @user.encrypted_string_value
+        @user.string_value = 'blah'
+        @user.string_value = STRING_VALUE
+        refute_equal first_value, @user.encrypted_string_value
       end
 
       it 'true and compress: true' do
@@ -182,9 +173,48 @@ class ActiveRecordTest < Minitest::Test
 
         refute_equal @user.encrypted_long_string_value, @user.encrypted_string_value
       end
+
+      describe 'changed?' do
+        it 'true for a new instance' do
+          assert @user.string_value_changed?
+        end
+
+        it 'clears after save' do
+          @user.save!
+          refute @user.string_value_changed?
+        end
+
+        it 'does not change when equal' do
+          @user.save!
+          before             = @user.encrypted_string_value
+          @user.string_value = STRING_VALUE
+          refute @user.string_value_changed?
+          assert_equal before, @user.encrypted_string_value
+        end
+      end
     end
 
     describe 'attribute=' do
+      it 'handles nil' do
+        @user.string_value = nil
+        assert_nil @user.string_value
+        assert_nil @user.encrypted_string_value
+        @user.save!
+        @user.reload
+        assert_nil @user.string_value
+        assert_nil @user.encrypted_string_value
+      end
+
+      it 'handles empty string' do
+        @user.string_value = ''
+        assert_equal '', @user.string_value
+        assert_equal '', @user.encrypted_string_value
+        @user.save!
+        @user.reload
+        assert_equal '', @user.string_value
+        assert_equal '', @user.encrypted_string_value
+      end
+
       it 'encrypt' do
         user                     = User.new
         user.bank_account_number = @bank_account_number
@@ -290,7 +320,7 @@ class ActiveRecordTest < Minitest::Test
         assert @user.valid?
         @user.number = ''
         assert_equal false, @user.valid?
-        assert_nil @user.number
+        assert_equal '', @user.number
         assert_equal ["can't be blank"], @user.errors[:number]
         @user.number = nil
         assert_nil @user.number
@@ -413,17 +443,8 @@ class ActiveRecordTest < Minitest::Test
               @user_clone.save!
 
               @user.reload
-              assert_nil @user.send(@attribute)
-              assert_nil @user.send("encrypted_#{@attribute}".to_sym)
-            end
-
-            it 'permit replacing value with a blank string' do
-              @user_clone.send("#{@attribute}=".to_sym, '    ')
-              @user_clone.save!
-
-              @user.reload
-              assert_nil @user.send(@attribute)
-              assert_nil @user.send("encrypted_#{@attribute}".to_sym)
+              assert_equal '', @user.send(@attribute)
+              assert_equal '', @user.send("encrypted_#{@attribute}".to_sym)
             end
 
             it 'permit replacing value' do