activerecord 6.1.6 → 7.0.0.alpha1

data/lib/active_record/encryption/encrypted_attribute_type.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Encryption
+    # An +ActiveModel::Type+ that encrypts/decrypts strings of text.
+    #
+    # This is the central piece that connects the encryption system with +encrypts+ declarations in the
+    # model classes. Whenever you declare an attribute as encrypted, it configures an +EncryptedAttributeType+
+    # for that attribute.
+    class EncryptedAttributeType < ::ActiveRecord::Type::Text
+      include ActiveModel::Type::Helpers::Mutable
+
+      attr_reader :scheme, :cast_type
+
+      delegate :key_provider, :downcase?, :deterministic?, :previous_schemes, :with_context, :fixed?, to: :scheme
+      delegate :accessor, to: :cast_type
+
+      # === Options
+      #
+      # * <tt>:scheme</tt> - An +Scheme+ with the encryption properties for this attribute.
+      # * <tt>:cast_type</tt> - A type that will be used to serialize (before encrypting) and deserialize
+      #   (after decrypting). +ActiveModel::Type::String+ by default.
+      def initialize(scheme:, cast_type: ActiveModel::Type::String.new, previous_type: false)
+        super()
+        @scheme = scheme
+        @cast_type = cast_type
+        @previous_type = previous_type
+      end
+
+      def deserialize(value)
+        cast_type.deserialize decrypt(value)
+      end
+
+      def serialize(value)
+        if serialize_with_oldest?
+          serialize_with_oldest(value)
+        else
+          serialize_with_current(value)
+        end
+      end
+
+      def changed_in_place?(raw_old_value, new_value)
+        old_value = raw_old_value.nil? ? nil : deserialize(raw_old_value)
+        old_value != new_value
+      end
+
+      def previous_types # :nodoc:
+        @previous_types ||= {} # Memoizing on support_unencrypted_data so that we can tweak it during tests
+        @previous_types[support_unencrypted_data?] ||= build_previous_types_for(previous_schemes_including_clean_text)
+      end
+
+      private
+        def previous_schemes_including_clean_text
+          previous_schemes.including((clean_text_scheme if support_unencrypted_data?)).compact
+        end
+
+        def previous_types_without_clean_text
+          @previous_types_without_clean_text ||= build_previous_types_for(previous_schemes)
+        end
+
+        def build_previous_types_for(schemes)
+          schemes.collect do |scheme|
+            EncryptedAttributeType.new(scheme: scheme, previous_type: true)
+          end
+        end
+
+        def previous_type?
+          @previous_type
+        end
+
+        def decrypt(value)
+          with_context do
+            encryptor.decrypt(value, **decryption_options) unless value.nil?
+          end
+        rescue ActiveRecord::Encryption::Errors::Base => error
+          if previous_types_without_clean_text.blank?
+            handle_deserialize_error(error, value)
+          else
+            try_to_deserialize_with_previous_encrypted_types(value)
+          end
+        end
+
+        def try_to_deserialize_with_previous_encrypted_types(value)
+          previous_types.each.with_index do |type, index|
+            break type.deserialize(value)
+          rescue ActiveRecord::Encryption::Errors::Base => error
+            handle_deserialize_error(error, value) if index == previous_types.length - 1
+          end
+        end
+
+        def handle_deserialize_error(error, value)
+          if error.is_a?(Errors::Decryption) && support_unencrypted_data?
+            value
+          else
+            raise error
+          end
+        end
+
+        def serialize_with_oldest?
+          @serialize_with_oldest ||= fixed? && previous_types_without_clean_text.present?
+        end
+
+        def serialize_with_oldest(value)
+          previous_types.first.serialize(value)
+        end
+
+        def serialize_with_current(value)
+          casted_value = cast_type.serialize(value)
+          casted_value = casted_value&.downcase if downcase?
+          encrypt(casted_value.to_s) unless casted_value.nil?
+        end
+
+        def encrypt(value)
+          with_context do
+            encryptor.encrypt(value, **encryption_options)
+          end
+        end
+
+        def encryptor
+          ActiveRecord::Encryption.encryptor
+        end
+
+        def support_unencrypted_data?
+          ActiveRecord::Encryption.config.support_unencrypted_data && !previous_type?
+        end
+
+        def encryption_options
+          @encryption_options ||= { key_provider: key_provider, cipher_options: { deterministic: deterministic? } }.compact
+        end
+
+        def decryption_options
+          @decryption_options ||= { key_provider: key_provider }.compact
+        end
+
+        def clean_text_scheme
+          @clean_text_scheme ||= ActiveRecord::Encryption::Scheme.new(downcase: downcase?, encryptor: ActiveRecord::Encryption::NullEncryptor.new)
+        end
+    end
+  end
+end

data/lib/active_record/encryption/encrypted_fixtures.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Encryption
+    module EncryptedFixtures
+      def initialize(fixture, model_class)
+        @clean_values = {}
+        encrypt_fixture_data(fixture, model_class)
+        process_preserved_original_columns(fixture, model_class)
+        super
+      end
+
+      private
+        def encrypt_fixture_data(fixture, model_class)
+          model_class&.encrypted_attributes&.each do |attribute_name|
+            if clean_value = fixture[attribute_name.to_s]
+              @clean_values[attribute_name.to_s] = clean_value
+
+              type = model_class.type_for_attribute(attribute_name)
+              encrypted_value = type.serialize(clean_value)
+              fixture[attribute_name.to_s] = encrypted_value
+            end
+          end
+        end
+
+        def process_preserved_original_columns(fixture, model_class)
+          model_class&.encrypted_attributes&.each do |attribute_name|
+            if source_attribute_name = model_class.source_attribute_from_preserved_attribute(attribute_name)
+              clean_value = @clean_values[source_attribute_name.to_s]
+              type = model_class.type_for_attribute(attribute_name)
+              encrypted_value = type.serialize(clean_value)
+              fixture[attribute_name.to_s] = encrypted_value
+            end
+          end
+        end
+    end
+  end
+end

data/lib/active_record/encryption/encrypting_only_encryptor.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Encryption
+    # An encryptor that can encrypt data but can't decrypt it.
+    class EncryptingOnlyEncryptor < Encryptor
+      def decrypt(encrypted_text, key_provider: nil, cipher_options: {})
+        encrypted_text
+      end
+    end
+  end
+end

data/lib/active_record/encryption/encryptor.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require "openssl"
+require "zlib"
+require "active_support/core_ext/numeric"
+
+module ActiveRecord
+  module Encryption
+    # An encryptor exposes the encryption API that +ActiveRecord::Encryption::EncryptedAttributeType+
+    # uses for encrypting and decrypting attribute values.
+    #
+    # It interacts with a +KeyProvider+ for getting the keys, and delegate to
+    # +ActiveRecord::Encryption::Cipher+ the actual encryption algorithm.
+    class Encryptor
+      # Encrypts +clean_text+ and returns the encrypted result
+      #
+      # Internally, it will:
+      #
+      # 1. Create a new +ActiveRecord::Encryption::Message+
+      # 2. Compress and encrypt +clean_text+ as the message payload
+      # 3. Serialize it with +ActiveRecord::Encryption.message_serializer+ (+ActiveRecord::Encryption::SafeMarshal+
+      #    by default)
+      # 4. Encode the result with Base 64
+      #
+      # === Options
+      #
+      # [:key_provider]
+      #   Key provider to use for the encryption operation. It will default to
+      #   +ActiveRecord::Encryption.key_provider+ when not provided
+      #
+      # [:cipher_options]
+      #   +Cipher+-specific options that will be passed to the Cipher configured in
+      #   +ActiveRecord::Encryption.cipher+
+      def encrypt(clear_text, key_provider: default_key_provider, cipher_options: {})
+        clear_text = force_encoding_if_needed(clear_text) if cipher_options[:deterministic]
+
+        validate_payload_type(clear_text)
+        serialize_message build_encrypted_message(clear_text, key_provider: key_provider, cipher_options: cipher_options)
+      end
+
+      # Decrypts a +clean_text+ and returns the result as clean text
+      #
+      # === Options
+      #
+      # [:key_provider]
+      #   Key provider to use for the encryption operation. It will default to
+      #   +ActiveRecord::Encryption.key_provider+ when not provided
+      #
+      # [:cipher_options]
+      #   +Cipher+-specific options that will be passed to the Cipher configured in
+      #   +ActiveRecord::Encryption.cipher+
+      def decrypt(encrypted_text, key_provider: default_key_provider, cipher_options: {})
+        message = deserialize_message(encrypted_text)
+        keys = key_provider.decryption_keys(message)
+        raise Errors::Decryption unless keys.present?
+        uncompress_if_needed(cipher.decrypt(message, key: keys.collect(&:secret), **cipher_options), message.headers.compressed)
+      rescue *(ENCODING_ERRORS + DECRYPT_ERRORS)
+        raise Errors::Decryption
+      end
+
+      # Returns whether the text is encrypted or not
+      def encrypted?(text)
+        deserialize_message(text)
+        true
+      rescue Errors::Encoding, *DECRYPT_ERRORS
+        false
+      end
+
+      private
+        DECRYPT_ERRORS = [OpenSSL::Cipher::CipherError, Errors::EncryptedContentIntegrity, Errors::Decryption]
+        ENCODING_ERRORS = [EncodingError, Errors::Encoding]
+        THRESHOLD_TO_JUSTIFY_COMPRESSION = 140.bytes
+
+        def default_key_provider
+          ActiveRecord::Encryption.key_provider
+        end
+
+        def validate_payload_type(clear_text)
+          unless clear_text.is_a?(String)
+            raise ActiveRecord::Encryption::Errors::ForbiddenClass, "The encryptor can only encrypt string values (#{clear_text.class})"
+          end
+        end
+
+        def cipher
+          ActiveRecord::Encryption.cipher
+        end
+
+        def build_encrypted_message(clear_text, key_provider:, cipher_options:)
+          key = key_provider.encryption_key
+
+          clear_text, was_compressed = compress_if_worth_it(clear_text)
+          cipher.encrypt(clear_text, key: key.secret, **cipher_options).tap do |message|
+            message.headers.add(key.public_tags)
+            message.headers.compressed = true if was_compressed
+          end
+        end
+
+        def serialize_message(message)
+          serializer.dump(message)
+        end
+
+        def deserialize_message(message)
+          raise Errors::Encoding unless message.is_a?(String)
+          serializer.load message
+        rescue ArgumentError, TypeError, Errors::ForbiddenClass
+          raise Errors::Encoding
+        end
+
+        def serializer
+          ActiveRecord::Encryption.message_serializer
+        end
+
+        # Under certain threshold, ZIP compression is actually worse that not compressing
+        def compress_if_worth_it(string)
+          if string.bytesize > THRESHOLD_TO_JUSTIFY_COMPRESSION
+            [compress(string), true]
+          else
+            [string, false]
+          end
+        end
+
+        def compress(data)
+          Zlib::Deflate.deflate(data).tap do |compressed_data|
+            compressed_data.force_encoding(data.encoding)
+          end
+        end
+
+        def uncompress_if_needed(data, compressed)
+          if compressed
+            uncompress(data)
+          else
+            data
+          end
+        end
+
+        def uncompress(data)
+          Zlib::Inflate.inflate(data).tap do |uncompressed_data|
+            uncompressed_data.force_encoding(data.encoding)
+          end
+        end
+
+        def force_encoding_if_needed(value)
+          if forced_encoding_for_deterministic_encryption && value && value.encoding != forced_encoding_for_deterministic_encryption
+            value.encode(forced_encoding_for_deterministic_encryption, invalid: :replace, undef: :replace)
+          else
+            value
+          end
+        end
+
+        def forced_encoding_for_deterministic_encryption
+          ActiveRecord::Encryption.config.forced_encoding_for_deterministic_encryption
+        end
+    end
+  end
+end

data/lib/active_record/encryption/envelope_encryption_key_provider.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Encryption
+    # Implements a simple envelope encryption approach where:
+    #
+    # * It generates a random data-encryption key for each encryption operation
+    # * It stores the generated key along with the encrypted payload. It encrypts this key
+    #   with the master key provided in the credential +active_record.encryption.master key+
+    #
+    # This provider can work with multiple master keys. It will use the last one for encrypting.
+    #
+    # When `config.store_key_references` is true, it will also store a reference to
+    # the specific master key that was used to encrypt the data-encryption key. When not set,
+    # it will try all the configured master keys looking for the right one, in order to
+    # return the right decryption key.
+    class EnvelopeEncryptionKeyProvider
+      def encryption_key
+        random_secret = generate_random_secret
+        ActiveRecord::Encryption::Key.new(random_secret).tap do |key|
+          key.public_tags.encrypted_data_key = encrypt_data_key(random_secret)
+          key.public_tags.encrypted_data_key_id = active_primary_key.id if ActiveRecord::Encryption.config.store_key_references
+        end
+      end
+
+      def decryption_keys(encrypted_message)
+        secret = decrypt_data_key(encrypted_message)
+        secret ? [ActiveRecord::Encryption::Key.new(secret)] : []
+      end
+
+      def active_primary_key
+        @active_primary_key ||= primary_key_provider.encryption_key
+      end
+
+      private
+        def encrypt_data_key(random_secret)
+          ActiveRecord::Encryption.cipher.encrypt(random_secret, key: active_primary_key.secret)
+        end
+
+        def decrypt_data_key(encrypted_message)
+          encrypted_data_key = encrypted_message.headers.encrypted_data_key
+          key = primary_key_provider.decryption_keys(encrypted_message)&.collect(&:secret)
+          ActiveRecord::Encryption.cipher.decrypt encrypted_data_key, key: key if key
+        end
+
+        def primary_key_provider
+          @primary_key_provider ||= DerivedSecretKeyProvider.new(ActiveRecord::Encryption.config.primary_key)
+        end
+
+        def generate_random_secret
+          ActiveRecord::Encryption.key_generator.generate_random_key
+        end
+    end
+  end
+end

data/lib/active_record/encryption/errors.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Encryption
+    module Errors
+      class Base < StandardError; end
+      class Encoding < Base; end
+      class Decryption < Base; end
+      class Encryption < Base; end
+      class Configuration < Base; end
+      class ForbiddenClass < Base; end
+      class EncryptedContentIntegrity < Base; end
+    end
+  end
+end

data/lib/active_record/encryption/extended_deterministic_queries.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+# Automatically expand encrypted arguments to support querying both encrypted and unencrypted data
+#
+# Active Record Encryption supports querying the db using deterministic attributes. For example:
+#
+#   Contact.find_by(email_address: "jorge@hey.com")
+#
+# The value "jorge@hey.com" will get encrypted automatically to perform the query. But there is
+# a problem while the data is being encrypted. This won't work. During that time, you need these
+# queries to be:
+#
+#   Contact.find_by(email_address: [ "jorge@hey.com", "<encrypted jorge@hey.com>" ])
+#
+# This patches ActiveRecord to support this automatically. It addresses both:
+#
+# * ActiveRecord::Base: Used in +Contact.find_by_email_address(...)+
+# * ActiveRecord::Relation: Used in +Contact.internal.find_by_email_address(...)+
+#
+# +ActiveRecord::Base+ relies on +ActiveRecord::Relation+ (+ActiveRecord::QueryMethods+) but it does
+# some prepared statements caching. That's why we need to intercept +ActiveRecord::Base+ as soon
+# as it's invoked (so that the proper prepared statement is cached).
+#
+# When modifying this file run performance tests in +test/performance/extended_deterministic_queries_performance_test.rb+ to
+#   make sure performance overhead is acceptable.
+#
+# We will extend this to support previous "encryption context" versions in future iterations
+#
+# @TODO Experimental. Support for every kind of query is pending
+# @TODO It should not patch anything if not needed (no previous schemes or no support for previous encryption schemes)
+module ActiveRecord
+  module Encryption
+    module ExtendedDeterministicQueries
+      def self.install_support
+        ActiveRecord::Relation.prepend(RelationQueries)
+        ActiveRecord::Base.include(CoreQueries)
+        ActiveRecord::Encryption::EncryptedAttributeType.prepend(ExtendedEncryptableType)
+        Arel::Nodes::HomogeneousIn.prepend(InWithAdditionalValues)
+      end
+
+      module EncryptedQueryArgumentProcessor
+        extend ActiveSupport::Concern
+
+        private
+          def process_encrypted_query_arguments(args, check_for_additional_values)
+            if args.is_a?(Array) && (options = args.first).is_a?(Hash)
+              self.deterministic_encrypted_attributes&.each do |attribute_name|
+                type = type_for_attribute(attribute_name)
+                if !type.previous_types.empty? && value = options[attribute_name]
+                  options[attribute_name] = process_encrypted_query_argument(value, check_for_additional_values, type)
+                end
+              end
+            end
+          end
+
+          def process_encrypted_query_argument(value, check_for_additional_values, type)
+            return value if check_for_additional_values && value.is_a?(Array) && value.last.is_a?(AdditionalValue)
+
+            case value
+            when String, Array
+              list = Array(value)
+              list + list.flat_map do |each_value|
+                if check_for_additional_values && each_value.is_a?(AdditionalValue)
+                  each_value
+                else
+                  additional_values_for(each_value, type)
+                end
+              end
+            else
+              value
+            end
+          end
+
+          def additional_values_for(value, type)
+            type.previous_types.collect do |additional_type|
+              AdditionalValue.new(value, additional_type)
+            end
+          end
+      end
+
+      module RelationQueries
+        include EncryptedQueryArgumentProcessor
+
+        def where(*args)
+          process_encrypted_query_arguments_if_needed(args)
+          super
+        end
+
+        def exists?(*args)
+          process_encrypted_query_arguments_if_needed(args)
+          super
+        end
+
+        def find_or_create_by(attributes, &block)
+          find_by(attributes.dup) || create(attributes, &block)
+        end
+
+        def find_or_create_by!(attributes, &block)
+          find_by(attributes.dup) || create!(attributes, &block)
+        end
+
+        private
+          def process_encrypted_query_arguments_if_needed(args)
+            process_encrypted_query_arguments(args, true) unless self.deterministic_encrypted_attributes&.empty?
+          end
+      end
+
+      module CoreQueries
+        extend ActiveSupport::Concern
+
+        class_methods do
+          include EncryptedQueryArgumentProcessor
+
+          def find_by(*args)
+            process_encrypted_query_arguments(args, false) unless self.deterministic_encrypted_attributes&.empty?
+            super
+          end
+        end
+      end
+
+      class AdditionalValue
+        attr_reader :value, :type
+
+        def initialize(value, type)
+          @type = type
+          @value = process(value)
+        end
+
+        private
+          def process(value)
+            type.serialize(value)
+          end
+      end
+
+      module ExtendedEncryptableType
+        def serialize(data)
+          if data.is_a?(AdditionalValue)
+            data.value
+          else
+            super
+          end
+        end
+      end
+
+      module InWithAdditionalValues
+        def proc_for_binds
+          -> value { ActiveModel::Attribute.with_cast_value(attribute.name, value, encryption_aware_type_caster) }
+        end
+
+        def encryption_aware_type_caster
+          if attribute.type_caster.is_a?(ActiveRecord::Encryption::EncryptedAttributeType)
+            attribute.type_caster.cast_type
+          else
+            attribute.type_caster
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/encryption/extended_deterministic_uniqueness_validator.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Encryption
+    module ExtendedDeterministicUniquenessValidator
+      def self.install_support
+        ActiveRecord::Validations::UniquenessValidator.prepend(EncryptedUniquenessValidator)
+      end
+
+      module EncryptedUniquenessValidator
+        def validate_each(record, attribute, value)
+          super(record, attribute, value)
+
+          klass = record.class
+          if klass.deterministic_encrypted_attributes&.each do |attribute_name|
+            encrypted_type = klass.type_for_attribute(attribute_name)
+            [ encrypted_type, *encrypted_type.previous_types ].each do |type|
+              encrypted_value = type.serialize(value)
+              ActiveRecord::Encryption.without_encryption do
+                super(record, attribute, encrypted_value)
+              end
+            end
+          end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/encryption/key.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Encryption
+    # A key is a container for a given +secret+
+    #
+    # Optionally, it can include +public_tags+. These tags are meant to be stored
+    # in clean (public) and can be used, for example, to include information that
+    # references the key for a future retrieval operation.
+    class Key
+      attr_reader :secret, :public_tags
+
+      def initialize(secret)
+        @secret = secret
+        @public_tags = Properties.new
+      end
+
+      def self.derive_from(password)
+        secret = ActiveRecord::Encryption.key_generator.derive_key_from(password)
+        ActiveRecord::Encryption::Key.new(secret)
+      end
+
+      def id
+        Digest::SHA1.hexdigest(secret).first(4)
+      end
+    end
+  end
+end

data/lib/active_record/encryption/key_generator.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module ActiveRecord
+  module Encryption
+    # Utility for generating and deriving random keys.
+    class KeyGenerator
+      # Returns a random key. The key will have a size in bytes of +:length+ (configured +Cipher+'s length by default)
+      def generate_random_key(length: key_length)
+        SecureRandom.random_bytes(length)
+      end
+
+      # Returns a random key in hexadecimal format. The key will have a size in bytes of +:length+ (configured +Cipher+'s
+      # length by default)
+      #
+      # Hexadecimal format is handy for representing keys as printable text. To maximize the space of characters used, it is
+      # good practice including not printable characters. Hexadecimal format ensures that generated keys are representable with
+      # plain text
+      #
+      # To convert back to the original string with the desired length:
+      #
+      #   [ value ].pack("H*")
+      def generate_random_hex_key(length: key_length)
+        generate_random_key(length: length).unpack("H*")[0]
+      end
+
+      # Derives a key from the given password. The key will have a size in bytes of +:length+ (configured +Cipher+'s length
+      # by default)
+      #
+      # The generated key will be salted with the value of +ActiveRecord::Encryption.key_derivation_salt+
+      def derive_key_from(password, length: key_length)
+        ActiveSupport::KeyGenerator.new(password).generate_key(ActiveRecord::Encryption.config.key_derivation_salt, length)
+      end
+
+      private
+        def key_length
+          @key_length ||= ActiveRecord::Encryption.cipher.key_length
+        end
+    end
+  end
+end