activerecord 7.0.6 → 7.1.1

data/lib/active_record/encryption/encryptable_record.rb

@@ -28,8 +28,12 @@ module ActiveRecord
         #   initialization vector based on the encrypted content. This means that the same content will generate
         #   the same ciphertexts. This enables querying encrypted text with Active Record. Deterministic encryption
         #   will use the oldest encryption scheme to encrypt new data by default. You can change this by setting
-        #   +deterministic: { fixed: false }+. That will make it use the newest encryption scheme for encrypting new
+        #   <tt>deterministic: { fixed: false }</tt>. That will make it use the newest encryption scheme for encrypting new
         #   data.
+        # * <tt>:support_unencrypted_data</tt> - If `config.active_record.encryption.support_unencrypted_data` is +true+,
+        #   you can set this to +false+ to opt out of unencrypted data support for this attribute. This is useful for
+        #   scenarios where you encrypt one column, and want to disable support for unencrypted data without having to tweak
+        #   the global setting.
         # * <tt>:downcase</tt> - When true, it converts the encrypted content to downcase automatically. This allows to
         #   effectively ignore case when querying data. Notice that the case is lost. Use +:ignore_case+ if you are interested
         #   in preserving it.
@@ -42,13 +46,11 @@ module ActiveRecord
         # * <tt>:previous</tt> - List of previous encryption schemes. When provided, they will be used in order when trying to read
         #   the attribute. Each entry of the list can contain the properties supported by #encrypts. Also, when deterministic
         #   encryption is used, they will be used to generate additional ciphertexts to check in the queries.
-        def encrypts(*names, key_provider: nil, key: nil, deterministic: false, downcase: false, ignore_case: false, previous: [], **context_properties)
+        def encrypts(*names, key_provider: nil, key: nil, deterministic: false, support_unencrypted_data: nil, downcase: false, ignore_case: false, previous: [], **context_properties)
           self.encrypted_attributes ||= Set.new # not using :default because the instance would be shared across classes
-          scheme = scheme_for key_provider: key_provider, key: key, deterministic: deterministic, downcase: downcase, \
-              ignore_case: ignore_case, previous: previous, **context_properties
 
           names.each do |name|
-            encrypt_attribute name, scheme
+            encrypt_attribute name, key_provider: key_provider, key: key, deterministic: deterministic, support_unencrypted_data: support_unencrypted_data, downcase: downcase, ignore_case: ignore_case, previous: previous, **context_properties
           end
         end
 
@@ -61,32 +63,35 @@ module ActiveRecord
 
         # Given a attribute name, it returns the name of the source attribute when it's a preserved one.
         def source_attribute_from_preserved_attribute(attribute_name)
-          attribute_name.to_s.sub(ORIGINAL_ATTRIBUTE_PREFIX, "") if /^#{ORIGINAL_ATTRIBUTE_PREFIX}/.match?(attribute_name)
+          attribute_name.to_s.sub(ORIGINAL_ATTRIBUTE_PREFIX, "") if attribute_name.start_with?(ORIGINAL_ATTRIBUTE_PREFIX)
         end
 
         private
-          def scheme_for(key_provider: nil, key: nil, deterministic: false, downcase: false, ignore_case: false, previous: [], **context_properties)
+          def scheme_for(key_provider: nil, key: nil, deterministic: false, support_unencrypted_data: nil, downcase: false, ignore_case: false, previous: [], **context_properties)
             ActiveRecord::Encryption::Scheme.new(key_provider: key_provider, key: key, deterministic: deterministic,
-                                                 downcase: downcase, ignore_case: ignore_case, **context_properties).tap do |scheme|
+              support_unencrypted_data: support_unencrypted_data, downcase: downcase, ignore_case: ignore_case, **context_properties).tap do |scheme|
               scheme.previous_schemes = global_previous_schemes_for(scheme) +
-                Array.wrap(previous).collect { |scheme_config| ActiveRecord::Encryption::Scheme.new(**scheme_config) }
+              Array.wrap(previous).collect { |scheme_config| ActiveRecord::Encryption::Scheme.new(**scheme_config) }
             end
           end
 
           def global_previous_schemes_for(scheme)
-            ActiveRecord::Encryption.config.previous_schemes.collect do |previous_scheme|
-              scheme.merge(previous_scheme)
+            ActiveRecord::Encryption.config.previous_schemes.filter_map do |previous_scheme|
+              scheme.merge(previous_scheme) if scheme.compatible_with?(previous_scheme)
             end
           end
 
-          def encrypt_attribute(name, attribute_scheme)
+          def encrypt_attribute(name, key_provider: nil, key: nil, deterministic: false, support_unencrypted_data: nil, downcase: false, ignore_case: false, previous: [], **context_properties)
             encrypted_attributes << name.to_sym
 
             attribute name do |cast_type|
-              ActiveRecord::Encryption::EncryptedAttributeType.new scheme: attribute_scheme, cast_type: cast_type
+              scheme = scheme_for key_provider: key_provider, key: key, deterministic: deterministic, support_unencrypted_data: support_unencrypted_data, \
+                downcase: downcase, ignore_case: ignore_case, previous: previous, **context_properties
+
+              ActiveRecord::Encryption::EncryptedAttributeType.new(scheme: scheme, cast_type: cast_type, default: columns_hash[name.to_s]&.default)
             end
 
-            preserve_original_encrypted(name) if attribute_scheme.ignore_case?
+            preserve_original_encrypted(name) if ignore_case
             ActiveRecord::Encryption.encrypted_attribute_was_declared(self, name)
           end
 
@@ -139,12 +144,16 @@ module ActiveRecord
 
       # Returns whether a given attribute is encrypted or not.
       def encrypted_attribute?(attribute_name)
-        ActiveRecord::Encryption.encryptor.encrypted? ciphertext_for(attribute_name)
+        ActiveRecord::Encryption.encryptor.encrypted? read_attribute_before_type_cast(attribute_name)
       end
 
       # Returns the ciphertext for +attribute_name+.
       def ciphertext_for(attribute_name)
-        read_attribute_before_type_cast(attribute_name)
+        if encrypted_attribute?(attribute_name)
+          read_attribute_before_type_cast(attribute_name)
+        else
+          read_attribute_for_database(attribute_name)
+        end
       end
 
       # Encrypts all the encryptable attributes and saves the changes.
@@ -160,6 +169,15 @@ module ActiveRecord
       private
         ORIGINAL_ATTRIBUTE_PREFIX = "original_"
 
+        def _create_record(attribute_names = self.attribute_names)
+          if has_encrypted_attributes?
+            # Always persist encrypted attributes, because an attribute might be
+            # encrypting a column default value.
+            attribute_names |= self.class.encrypted_attributes.map(&:to_s)
+          end
+          super
+        end
+
         def encrypt_attributes
           validate_encryption_allowed
 
@@ -188,12 +206,12 @@ module ActiveRecord
         end
 
         def build_decrypt_attribute_assignments
-          Array(self.class.encrypted_attributes).collect do |attribute_name|
+          Array(self.class.encrypted_attributes).to_h do |attribute_name|
             type = type_for_attribute(attribute_name)
             encrypted_value = ciphertext_for(attribute_name)
             new_value = type.deserialize(encrypted_value)
             [attribute_name, new_value]
-          end.to_h
+          end
         end
 
         def cant_modify_encrypted_attributes_when_frozen

data/lib/active_record/encryption/encrypted_attribute_type.rb

@@ -20,11 +20,16 @@ module ActiveRecord
       # * <tt>:scheme</tt> - A +Scheme+ with the encryption properties for this attribute.
       # * <tt>:cast_type</tt> - A type that will be used to serialize (before encrypting) and deserialize
       #   (after decrypting). ActiveModel::Type::String by default.
-      def initialize(scheme:, cast_type: ActiveModel::Type::String.new, previous_type: false)
+      def initialize(scheme:, cast_type: ActiveModel::Type::String.new, previous_type: false, default: nil)
         super()
         @scheme = scheme
         @cast_type = cast_type
         @previous_type = previous_type
+        @default = default
+      end
+
+      def cast(value)
+        cast_type.cast(value)
       end
 
       def deserialize(value)
@@ -49,6 +54,10 @@ module ActiveRecord
         @previous_types[support_unencrypted_data?] ||= build_previous_types_for(previous_schemes_including_clean_text)
       end
 
+      def support_unencrypted_data?
+        ActiveRecord::Encryption.config.support_unencrypted_data && scheme.support_unencrypted_data? && !previous_type?
+      end
+
       private
         def previous_schemes_including_clean_text
           previous_schemes.including((clean_text_scheme if support_unencrypted_data?)).compact
@@ -70,7 +79,13 @@ module ActiveRecord
 
         def decrypt(value)
           with_context do
-            encryptor.decrypt(value, **decryption_options) unless value.nil?
+            unless value.nil?
+              if @default && @default == value
+                value
+              else
+                encryptor.decrypt(value, **decryption_options)
+              end
+            end
           end
         rescue ActiveRecord::Encryption::Errors::Base => error
           if previous_types_without_clean_text.blank?
@@ -120,10 +135,6 @@ module ActiveRecord
           ActiveRecord::Encryption.encryptor
         end
 
-        def support_unencrypted_data?
-          ActiveRecord::Encryption.config.support_unencrypted_data && !previous_type?
-        end
-
         def encryption_options
           @encryption_options ||= { key_provider: key_provider, cipher_options: { deterministic: deterministic? } }.compact
         end

data/lib/active_record/encryption/extended_deterministic_queries.rb

@@ -19,101 +19,113 @@ module ActiveRecord
     # * ActiveRecord::Base - Used in <tt>Contact.find_by_email_address(...)</tt>
     # * ActiveRecord::Relation - Used in <tt>Contact.internal.find_by_email_address(...)</tt>
     #
-    # ActiveRecord::Base relies on ActiveRecord::Relation (ActiveRecord::QueryMethods) but it does
-    # some prepared statements caching. That's why we need to intercept +ActiveRecord::Base+ as soon
-    # as it's invoked (so that the proper prepared statement is cached).
-    #
-    # When modifying this file run performance tests in +test/performance/extended_deterministic_queries_performance_test.rb+ to
-    #   make sure performance overhead is acceptable.
-    #
-    # We will extend this to support previous "encryption context" versions in future iterations
-    #
-    # @TODO Experimental. Support for every kind of query is pending
-    # @TODO It should not patch anything if not needed (no previous schemes or no support for previous encryption schemes)
+    # This module is included if `config.active_record.encryption.extend_queries` is `true`.
     module ExtendedDeterministicQueries
       def self.install_support
+        # ActiveRecord::Base relies on ActiveRecord::Relation (ActiveRecord::QueryMethods) but it does
+        # some prepared statements caching. That's why we need to intercept +ActiveRecord::Base+ as soon
+        # as it's invoked (so that the proper prepared statement is cached).
         ActiveRecord::Relation.prepend(RelationQueries)
         ActiveRecord::Base.include(CoreQueries)
         ActiveRecord::Encryption::EncryptedAttributeType.prepend(ExtendedEncryptableType)
         Arel::Nodes::HomogeneousIn.prepend(InWithAdditionalValues)
       end
 
-      module EncryptedQueryArgumentProcessor
-        extend ActiveSupport::Concern
+      # When modifying this file run performance tests in
+      # +activerecord/test/cases/encryption/performance/extended_deterministic_queries_performance_test.rb+
+      # to make sure performance overhead is acceptable.
+      #
+      # @TODO We will extend this to support previous "encryption context" versions in future iterations
+      # @TODO Experimental. Support for every kind of query is pending
+      # @TODO It should not patch anything if not needed (no previous schemes or no support for previous encryption schemes)
+
+      module EncryptedQuery # :nodoc:
+        class << self
+          def process_arguments(owner, args, check_for_additional_values)
+            return args if owner.deterministic_encrypted_attributes&.empty?
 
-        private
-          def process_encrypted_query_arguments(args, check_for_additional_values)
             if args.is_a?(Array) && (options = args.first).is_a?(Hash)
-              self.deterministic_encrypted_attributes&.each do |attribute_name|
-                type = type_for_attribute(attribute_name)
+              options = options.transform_keys do |key|
+                if key.is_a?(Array)
+                  key.map(&:to_s)
+                else
+                  key.to_s
+                end
+              end
+              args[0] = options
+
+              owner.deterministic_encrypted_attributes&.each do |attribute_name|
+                attribute_name = attribute_name.to_s
+                type = owner.type_for_attribute(attribute_name)
                 if !type.previous_types.empty? && value = options[attribute_name]
                   options[attribute_name] = process_encrypted_query_argument(value, check_for_additional_values, type)
                 end
               end
             end
-          end
 
-          def process_encrypted_query_argument(value, check_for_additional_values, type)
-            return value if check_for_additional_values && value.is_a?(Array) && value.last.is_a?(AdditionalValue)
+            args
+          end
 
-            case value
-            when String, Array
-              list = Array(value)
-              list + list.flat_map do |each_value|
-                if check_for_additional_values && each_value.is_a?(AdditionalValue)
-                  each_value
-                else
-                  additional_values_for(each_value, type)
+          private
+            def process_encrypted_query_argument(value, check_for_additional_values, type)
+              return value if check_for_additional_values && value.is_a?(Array) && value.last.is_a?(AdditionalValue)
+
+              case value
+              when String, Array
+                list = Array(value)
+                list + list.flat_map do |each_value|
+                  if check_for_additional_values && each_value.is_a?(AdditionalValue)
+                    each_value
+                  else
+                    additional_values_for(each_value, type)
+                  end
                 end
+              else
+                value
               end
-            else
-              value
             end
-          end
 
-          def additional_values_for(value, type)
-            type.previous_types.collect do |additional_type|
-              AdditionalValue.new(value, additional_type)
+            def additional_values_for(value, type)
+              type.previous_types.collect do |additional_type|
+                AdditionalValue.new(value, additional_type)
+              end
             end
-          end
+        end
       end
 
       module RelationQueries
-        include EncryptedQueryArgumentProcessor
-
         def where(*args)
-          process_encrypted_query_arguments_if_needed(args)
-          super
+          super(*EncryptedQuery.process_arguments(self, args, true))
         end
 
         def exists?(*args)
-          process_encrypted_query_arguments_if_needed(args)
-          super
+          super(*EncryptedQuery.process_arguments(self, args, true))
         end
 
-        def find_or_create_by(attributes, &block)
-          find_by(attributes.dup) || create(attributes, &block)
-        end
+        def scope_for_create
+          return super unless klass.deterministic_encrypted_attributes&.any?
 
-        def find_or_create_by!(attributes, &block)
-          find_by(attributes.dup) || create!(attributes, &block)
-        end
+          scope_attributes = super
+          wheres = where_values_hash
 
-        private
-          def process_encrypted_query_arguments_if_needed(args)
-            process_encrypted_query_arguments(args, true) unless self.deterministic_encrypted_attributes&.empty?
+          klass.deterministic_encrypted_attributes.each do |attribute_name|
+            attribute_name = attribute_name.to_s
+            values = wheres[attribute_name]
+            if values.is_a?(Array) && values[1..].all?(AdditionalValue)
+              scope_attributes[attribute_name] = values.first
+            end
           end
+
+          scope_attributes
+        end
       end
 
       module CoreQueries
         extend ActiveSupport::Concern
 
         class_methods do
-          include EncryptedQueryArgumentProcessor
-
           def find_by(*args)
-            process_encrypted_query_arguments(args, false) unless self.deterministic_encrypted_attributes&.empty?
-            super
+            super(*EncryptedQuery.process_arguments(self, args, false))
           end
         end
       end

data/lib/active_record/encryption/extended_deterministic_uniqueness_validator.rb

@@ -12,9 +12,9 @@ module ActiveRecord
           super(record, attribute, value)
 
           klass = record.class
-          klass.deterministic_encrypted_attributes&.each do |attribute_name|
-            encrypted_type = klass.type_for_attribute(attribute_name)
-            [ encrypted_type, *encrypted_type.previous_types ].each do |type|
+          if klass.deterministic_encrypted_attributes&.include?(attribute)
+            encrypted_type = klass.type_for_attribute(attribute)
+            encrypted_type.previous_types.each do |type|
               encrypted_value = type.serialize(value)
               ActiveRecord::Encryption.without_encryption do
                 super(record, attribute, encrypted_value)

data/lib/active_record/encryption/key_generator.rb

@@ -6,6 +6,12 @@ module ActiveRecord
   module Encryption
     # Utility for generating and deriving random keys.
     class KeyGenerator
+      attr_reader :hash_digest_class
+
+      def initialize(hash_digest_class: ActiveRecord::Encryption.config.hash_digest_class)
+        @hash_digest_class = hash_digest_class
+      end
+
       # Returns a random key. The key will have a size in bytes of +:length+ (configured +Cipher+'s length by default)
       def generate_random_key(length: key_length)
         SecureRandom.random_bytes(length)
@@ -30,10 +36,15 @@ module ActiveRecord
       #
       # The generated key will be salted with the value of +ActiveRecord::Encryption.key_derivation_salt+
       def derive_key_from(password, length: key_length)
-        ActiveSupport::KeyGenerator.new(password).generate_key(ActiveRecord::Encryption.config.key_derivation_salt, length)
+        ActiveSupport::KeyGenerator.new(password, hash_digest_class: hash_digest_class)
+          .generate_key(key_derivation_salt, length)
       end
 
       private
+        def key_derivation_salt
+          @key_derivation_salt ||= ActiveRecord::Encryption.config.key_derivation_salt
+        end
+
         def key_length
           @key_length ||= ActiveRecord::Encryption.cipher.key_length
         end

data/lib/active_record/encryption/message_serializer.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "base64"
+
 module ActiveRecord
   module Encryption
     # A message serializer that serializes +Messages+ with JSON.

data/lib/active_record/encryption/properties.rb

@@ -14,10 +14,10 @@ module ActiveRecord
     #
     # See +Properties::DEFAULT_PROPERTIES+, Key, Message
     class Properties
-      ALLOWED_VALUE_CLASSES = [String, ActiveRecord::Encryption::Message, Numeric, TrueClass, FalseClass, Symbol, NilClass]
+      ALLOWED_VALUE_CLASSES = [String, ActiveRecord::Encryption::Message, Numeric, Integer, Float, BigDecimal, TrueClass, FalseClass, Symbol, NilClass]
 
       delegate_missing_to :data
-      delegate :==, to: :data
+      delegate :==, :[], :each, :key?, to: :data
 
       # For each entry it generates an accessor exposing the full name
       DEFAULT_PROPERTIES = {
@@ -54,7 +54,7 @@ module ActiveRecord
       end
 
       def validate_value_type(value)
-        unless ALLOWED_VALUE_CLASSES.find { |klass| value.is_a?(klass) }
+        unless ALLOWED_VALUE_CLASSES.include?(value.class) || ALLOWED_VALUE_CLASSES.any? { |klass| value.is_a?(klass) }
           raise ActiveRecord::Encryption::Errors::ForbiddenClass, "Can't store a #{value.class}, only properties of type #{ALLOWED_VALUE_CLASSES.inspect} are allowed"
         end
       end

data/lib/active_record/encryption/scheme.rb

@@ -10,7 +10,7 @@ module ActiveRecord
     class Scheme
       attr_accessor :previous_schemes
 
-      def initialize(key_provider: nil, key: nil, deterministic: nil, downcase: nil, ignore_case: nil,
+      def initialize(key_provider: nil, key: nil, deterministic: nil, support_unencrypted_data: nil, downcase: nil, ignore_case: nil,
                      previous_schemes: nil, **context_properties)
         # Initializing all attributes to +nil+ as we want to allow a "not set" semantics so that we
         # can merge schemes without overriding values with defaults. See +#merge+
@@ -18,6 +18,7 @@ module ActiveRecord
         @key_provider_param = key_provider
         @key = key
         @deterministic = deterministic
+        @support_unencrypted_data = support_unencrypted_data
         @downcase = downcase || ignore_case
         @ignore_case = ignore_case
         @previous_schemes_param = previous_schemes
@@ -36,7 +37,11 @@ module ActiveRecord
       end
 
       def deterministic?
-        @deterministic
+        !!@deterministic
+      end
+
+      def support_unencrypted_data?
+        @support_unencrypted_data.nil? ? ActiveRecord::Encryption.config.support_unencrypted_data : @support_unencrypted_data
       end
 
       def fixed?
@@ -45,10 +50,7 @@ module ActiveRecord
       end
 
       def key_provider
-        @key_provider ||= begin
-          validate_keys!
-          @key_provider_param || build_key_provider
-        end
+        @key_provider ||= @key_provider_param || build_key_provider || default_key_provider
       end
 
       def merge(other_scheme)
@@ -56,7 +58,7 @@ module ActiveRecord
       end
 
       def to_h
-        { key_provider: @key_provider_param, key: @key, deterministic: @deterministic, downcase: @downcase, ignore_case: @ignore_case,
+        { key_provider: @key_provider_param, deterministic: @deterministic, downcase: @downcase, ignore_case: @ignore_case,
           previous_schemes: @previous_schemes_param, **@context_properties }.compact
       end
 
@@ -68,32 +70,27 @@ module ActiveRecord
         end
       end
 
+      def compatible_with?(other_scheme)
+        deterministic? == other_scheme.deterministic?
+      end
+
       private
         def validate_config!
           raise Errors::Configuration, "ignore_case: can only be used with deterministic encryption" if @ignore_case && !@deterministic
           raise Errors::Configuration, "key_provider: and key: can't be used simultaneously" if @key_provider_param && @key
         end
 
-        def validate_keys!
-          validate_credential :key_derivation_salt
-          validate_credential :primary_key, "needs to be configured to use non-deterministic encryption" unless @deterministic
-          validate_credential :deterministic_key, "needs to be configured to use deterministic encryption" if @deterministic
-        end
-
-        def validate_credential(key, error_message = "is not configured")
-          unless ActiveRecord::Encryption.config.public_send(key).present?
-            raise Errors::Configuration, "#{key} #{error_message}. Please configure it via credential "\
-              "active_record_encryption.#{key} or by setting config.active_record.encryption.#{key}"
-          end
-        end
-
         def build_key_provider
           return DerivedSecretKeyProvider.new(@key) if @key.present?
 
-          if @deterministic && (deterministic_key = ActiveRecord::Encryption.config.deterministic_key)
-            DeterministicKeyProvider.new(deterministic_key)
+          if @deterministic
+            DeterministicKeyProvider.new(ActiveRecord::Encryption.config.deterministic_key)
           end
         end
+
+        def default_key_provider
+          ActiveRecord::Encryption.key_provider
+        end
     end
   end
 end

data/lib/active_record/encryption.rb

@@ -8,6 +8,7 @@ module ActiveRecord
     extend ActiveSupport::Autoload
 
     eager_autoload do
+      autoload :AutoFilteredParameters
       autoload :Cipher
       autoload :Config
       autoload :Configurable