mongoid 9.1.0 → 9.1.2

data/lib/mongoid/criteria/includable.rb

@@ -44,11 +44,13 @@ module Mongoid
         clone
       end
 
-      # Returns whether to use $lookup aggregation for eager loading.
+      # Returns whether to use $lookup aggregation for eager loading. Only when
+      # eager_load was requested and there is something to load: an empty
+      # inclusion list (e.g. eager_load([])) falls back to the normal path.
       #
       # @return [ true | false ] Whether to use $lookup.
       def use_lookup?
-        !!@use_lookup
+        !!@use_lookup && inclusions.any?
       end
 
       # Get a list of criteria that are to be executed for eager loading.
@@ -96,20 +98,42 @@ module Mongoid
       #   The names of the association(s) to eager load.
       def extract_includes_list(_parent_class, parent, is_eager_load = false, *relations_list)
         relations_list.flatten.each do |relation_object|
-          if relation_object.is_a?(Hash)
-            relation_object.each do |relation, _includes|
-              association = _parent_class.reflect_on_association(relation)
-              raise_eager_error(is_eager_load, _klass, relation) unless association
+          # Normalize a bare association name to a hash with no nested
+          # inclusions, so both forms share one resolution path below.
+          relations = relation_object.is_a?(Hash) ? relation_object : { relation_object => nil }
+
+          relations.each do |relation, nested|
+            associations = resolve_inclusion_associations(_parent_class, relation, is_eager_load)
+            raise_eager_error(is_eager_load, _parent_class, relation) if associations.empty?
+
+            associations.each do |association|
               add_inclusion(association, parent)
-              extract_includes_list(association.klass, association.name, is_eager_load, _includes)
+              extract_includes_list(association.klass, association.name, is_eager_load, nested) if nested
             end
-          else
-            association = _parent_class.reflect_on_association(relation_object)
-            raise_eager_error(is_eager_load, _parent_class, relation_object) unless association
-            add_inclusion(association, parent)
           end
         end
       end
+
+      # Resolve the association(s) matching the given relation name. For the
+      # regular #includes path, only the parent class is consulted. For the
+      # #eager_load ($lookup) path, its subclasses are consulted as well, so
+      # associations defined only on a subclass can also be eager-loaded when
+      # querying through the superclass.
+      #
+      # @param [ Class ] parent_class The class to start the lookup from.
+      # @param [ Symbol | String ] relation The association name.
+      # @param [ Boolean ] is_eager_load Whether to consider subclasses.
+      #
+      # @return [ Array<Mongoid::Association::Relatable> ] Matching associations.
+      def resolve_inclusion_associations(parent_class, relation, is_eager_load)
+        if association = parent_class.reflect_on_association(relation)
+          return [ association ]
+        end
+
+        return [] unless is_eager_load
+
+        parent_class.descendants.filter_map { |sub| sub.reflect_on_association(relation) }
+      end
     end
 
     def raise_eager_error(is_eager_load, klass, relation)

data/lib/mongoid/criteria/queryable/mergeable.rb

@@ -320,6 +320,10 @@ module Mongoid
               end
             end
           end
+          # Every query method that takes a user-supplied expression normalizes
+          # it here, so this is where the operator guard is enforced. See
+          # Selectable#_mongoid_validate_operators! for what it does not cover.
+          _mongoid_validate_operators!(result)
           result
         end
 

data/lib/mongoid/criteria/queryable/selectable.rb

@@ -491,16 +491,16 @@ module Mongoid
           !!negating
         end
 
-        # Negate the arguments, or the next selection if no arguments are given.
-        #
-        # @example Negate the next selection.
-        #   selectable.not.in(field: [ 1, 2 ])
+        # Negate the arguments.
         #
         # @example Add the $not criterion.
-        #   selectable.not(name: /Bob/)
+        #   Person.not(name: /Bob/)
         #
         # @example Execute a $not in a where query.
-        #   selectable.where(:field.not => /Bob/)
+        #   Person.where(:name.not => /Bob/)
+        #
+        # @example Negate a Criteria instance
+        #   Person.not(Person.where(...))
         #
         # @param [ [ Hash | Criteria ]... ] *criteria The key/value pair
         #   matches or Criteria objects to negate.
@@ -508,6 +508,14 @@ module Mongoid
         # @return [ Selectable ] The new selectable.
         def not(*criteria)
           if criteria.empty?
+            # @deprecated
+            Mongoid.deprecation_warning(
+              :not_sans_arguments,
+              'Calling `#not` without arguments is deprecated, and will be ' \
+              'removed in the next major version. Instead, pass the Hash or ' \
+              'Criteria instance to negate.',
+              caller_locations
+            )
             dup.tap { |query| query.negating = !query.negating }
           else
             criteria.compact.inject(clone) do |c, new_s|
@@ -768,6 +776,12 @@ module Mongoid
             # only ever specify one criterion to #where.
             @criterion = criterion
             if criterion.is_a?(String)
+              unless Mongoid.allow_unsafe_query_operators?
+                raise Errors::InvalidQuery,
+                      "String criteria are not allowed because they compile to the '$where' operator, " \
+                      'which is not allowed in a query expression. Set Mongoid.allow_unsafe_query_operators = true ' \
+                      'to permit all operators.'
+              end
               query.js_query(criterion)
             else
               query.expr_query(criterion)
@@ -775,6 +789,19 @@ module Mongoid
           end
         end
 
+        # Operators permitted at the top level of a query expression without
+        # opt-in. Excludes $where (JS execution) and other operators not needed
+        # for ordinary application queries.
+        ALLOWED_QUERY_OPERATORS = %w[
+          $and $or $nor $not $text $comment $expr $jsonSchema $alwaysFalse $alwaysTrue
+        ].freeze
+
+        # Operators that execute server-side JavaScript. These are rejected at
+        # any depth, not just at the top level: the allowlist above permits
+        # $expr and the logical operators, and their values are arbitrary
+        # nested expressions that can carry $function or $where.
+        JAVASCRIPT_QUERY_OPERATORS = %w[$where $function $accumulator].freeze
+
         # Adds the specified expression to the query.
         #
         # Criterion must be a hash in one of the following forms:
@@ -798,12 +825,13 @@ module Mongoid
             raise Errors::InvalidQuery, "Expression must be a Hash: #{Errors::InvalidQuery.truncate_expr(criterion)}"
           end
 
+          # The operator guard is applied by _mongoid_expand_keys, which every
+          # query method that accepts a user-supplied expression passes through.
           normalized = _mongoid_expand_keys(criterion)
           clone.tap do |query|
             normalized.each do |field, value|
               field_s = field.to_s
               if field_s.start_with?('$')
-                # Query expression-level operator, like $and or $where
                 query.add_operator_expression(field_s, value)
               else
                 query.add_field_expression(field, value)
@@ -837,6 +865,94 @@ module Mongoid
 
         private
 
+        # Enforces the operator rules governed by the
+        # +allow_unsafe_query_operators+ configuration option against a
+        # normalized query expression.
+        #
+        # Two rules apply, and both are skipped when the option is true:
+        #
+        # - An operator at the top level of the expression must appear in
+        #   ALLOWED_QUERY_OPERATORS.
+        # - An operator in JAVASCRIPT_QUERY_OPERATORS is rejected at any depth.
+        #
+        # This is called from #_mongoid_expand_keys rather than from the
+        # individual query methods, because that is the one point every query
+        # method taking a user-supplied expression passes through on its way to
+        # the selector.
+        #
+        # It deliberately does not cover the APIs that ask for JavaScript
+        # outright, such as #js_query and Criteria#for_js: there the developer
+        # has chosen server-side JavaScript, so there is nothing to guard
+        # against. The same goes for the low-level Storable methods
+        # (#add_field_expression, #add_operator_expression), which write to the
+        # selector directly.
+        #
+        # @param [ Hash ] expr A normalized query expression.
+        #
+        # @raise [ Errors::InvalidQuery ] If a disallowed operator is present.
+        #
+        # @api private
+        def _mongoid_validate_operators!(expr)
+          return if Mongoid.allow_unsafe_query_operators?
+
+          expr.each_key do |field|
+            field_s = field.to_s
+            next unless field_s.start_with?('$')
+            next if ALLOWED_QUERY_OPERATORS.include?(field_s)
+
+            raise Errors::InvalidQuery,
+                  "Operator '#{field_s}' is not allowed in a query expression. " \
+                  'Set Mongoid.allow_unsafe_query_operators = true to permit all operators.'
+          end
+
+          _mongoid_validate_no_javascript!(expr)
+        end
+
+        # Walks a query expression looking for operators that execute
+        # server-side JavaScript, descending through both hashes and arrays so
+        # that nested forms such as {'$expr' => {'$function' => ...}} and
+        # {'$or' => [ {'$where' => ...} ]} are caught.
+        #
+        # The walk does not distinguish operator position from value position,
+        # so it also rejects queries where a JavaScript operator name appears as
+        # data rather than as an operator. Server 5.0+ permits $-prefixed field
+        # names in stored documents, which makes this reachable:
+        #
+        #   Doc.where(payload: { '$eq' => { '$function' => 'abc' } })
+        #
+        # Here the $eq marks its argument as a literal value to compare, so the
+        # server never evaluates it, but the guard raises anyway. The only
+        # workaround today is the global allow_unsafe_query_operators flag.
+        #
+        # TODO: discuss whether to track operator position (skipping the subtree
+        # under $eq, $ne, $in, $nin, and $elemMatch values) in a future
+        # iteration. It removes the false positive but adds exactly the kind of
+        # state that a real bypass could hide in, so it was left out for now.
+        #
+        # @param [ Object ] object A fragment of a query expression.
+        #
+        # @raise [ Errors::InvalidQuery ] If a JavaScript operator is present.
+        #
+        # @api private
+        def _mongoid_validate_no_javascript!(object)
+          case object
+          when Hash
+            object.each do |key, value|
+              key_s = key.to_s
+              if JAVASCRIPT_QUERY_OPERATORS.include?(key_s)
+                raise Errors::InvalidQuery,
+                      "Operator '#{key_s}' executes server-side JavaScript and is not allowed " \
+                      'anywhere in a query expression. Set Mongoid.allow_unsafe_query_operators = true ' \
+                      'to permit all operators.'
+              end
+
+              _mongoid_validate_no_javascript!(value)
+            end
+          when Array
+            object.each { |value| _mongoid_validate_no_javascript!(value) }
+          end
+        end
+
         # Force the values of the criterion to be evolved.
         #
         # @api private

data/lib/mongoid/deprecable.rb

@@ -7,6 +7,29 @@ module Mongoid
   #
   # @api private
   module Deprecable
+    # A Mongoid::Deprecation instance to use for reporting deprecations
+    def deprecator
+      @deprecator ||= Mongoid::Deprecation.new
+    end
+
+    # Resets all deprecation warnings. For use in tests.
+    def reset_deprecation_warnings!
+      DEPRECATION_WARNING_MUTEX.synchronize { @deprecation_warnings = {} }
+    end
+
+    # Emits a warning using the current deprecator. If the given warning (as
+    # identified by `id`) has already been issued previously, this is a no-op.
+    #
+    # @param [ Symbol ] id The unique identifier for this warning.
+    # @param [ String ] warning The warning message to emit.
+    # @param [ Array<Thread::Backtrace::Location> | nil ] callstack The backtrace at the call site.
+    def deprecation_warning(id, warning, callstack = nil)
+      site = callstack&.first
+      deprecation_warning_guard(id, site ? "#{site.path}:#{site.lineno}" : nil) do
+        deprecator.warn(warning, callstack)
+      end
+    end
+
     # Declares method(s) as deprecated.
     #
     # @example Deprecate a method.
@@ -25,8 +48,28 @@ module Mongoid
     # @param [ [ Symbol | Hash<Symbol, [ Symbol | String ]> ]... ] *method_descriptors
     #   The methods to deprecate, with optional replacement instructions.
     def deprecate(target_module, *method_descriptors)
-      @_deprecator ||= Mongoid::Deprecation.new
-      @_deprecator.deprecate_methods(target_module, *method_descriptors)
+      deprecator.deprecate_methods(target_module, *method_descriptors)
+    end
+
+    private
+
+    # The Mutex instance used to guard the deprecation warning flags.
+    DEPRECATION_WARNING_MUTEX = Mutex.new
+
+    # Wraps access to the warnings Hash in a synchronize block. If the given
+    # id+callsite has not been warned already, the method will yield to a block and then
+    # flag the id. Otherwise, it returns immediately.
+    def deprecation_warning_guard(id, callsite)
+      DEPRECATION_WARNING_MUTEX.synchronize do
+        @deprecation_warnings ||= {}
+
+        key = "#{id}:#{callsite}"
+        return if @deprecation_warnings.key?(key)
+
+        yield
+
+        @deprecation_warnings[key] = true
+      end
     end
   end
 end

data/lib/mongoid/document.rb

@@ -410,6 +410,8 @@ module Mongoid
       def instantiate_document(attrs = nil, selected_fields = nil, options = {}, &block)
         execute_callbacks = options.fetch(:execute_callbacks, Threaded.execute_callbacks?)
         attributes = attrs.to_h
+        preloaded_fields = attributes.keys.select { |key| key.start_with?(Association::PRELOADED_PREFIX) }
+        preloaded = attributes.extract!(*preloaded_fields)
 
         doc = allocate
         doc.__selected_fields = selected_fields
@@ -417,6 +419,7 @@ module Mongoid
         doc.instance_variable_set(:@attributes_before_type_cast, attributes.dup)
 
         doc._handle_callbacks_after_instantiation(execute_callbacks, &block)
+        doc.load_preloaded_associations(preloaded)
 
         doc.remember_storage_options!
         doc

data/lib/mongoid/encryptable.rb

@@ -36,6 +36,79 @@ module Mongoid
         !encrypt_metadata.empty? || fields.any? { |_, field| field.is_a?(Mongoid::Fields::Encrypted) }
       end
 
+      # Whether an encryption schema has to be generated for this model.
+      #
+      # True when the model declares encryption itself, and also when any model
+      # reachable through its embeds_one relations does. A model in the second
+      # group has no encrypted field of its own, but its collection still needs
+      # a schema, otherwise the embedded fields are written in plaintext.
+      #
+      # This runs on the persistence path, so the answer is memoized. A true
+      # answer is always memoized. A false answer is memoized only when every
+      # embeds_one target in the walk resolved to a class; if one could not be
+      # resolved yet, the answer is recomputed next time, so a class loaded
+      # later is still detected. Declaring encryption on a model after it has
+      # already been persisted is not supported.
+      #
+      # @return [ true | false ] Whether the model needs an encryption schema.
+      #
+      # @api private
+      def requires_encryption_schema?
+        return @requires_encryption_schema if defined?(@requires_encryption_schema)
+        return @requires_encryption_schema = true if encrypted?
+
+        found, complete = embedded_encryption_scan([ self ])
+        @requires_encryption_schema = found if found || complete
+        found
+      end
+
+      # Walks the models reachable through this model's embeds_one relations,
+      # looking for one that declares encryption. Subclasses of an
+      # association target are included: any of them can be assigned to the
+      # association.
+      #
+      # embeds_many is not considered: libmongocrypt cannot express per-field
+      # encryption under array items, so those fields are never mapped.
+      #
+      # @param [ Array<Class> ] path The models the walk is already inside of.
+      #   A model embedding itself terminates here.
+      #
+      # @return [ Array<true | false> ] A pair: whether an embedded model is
+      #   encrypted, and whether every embeds_one target the walk needed
+      #   resolved to a class. The second value is only meaningful when the
+      #   first is false.
+      #
+      # @api private
+      def embedded_encryption_scan(path)
+        complete = true
+
+        relations.each_value do |relation|
+          next unless relation.is_a?(Association::Embedded::EmbedsOne)
+
+          klass = relation.try_relation_class
+          # The class an association names does not have to exist yet.
+          if klass.nil?
+            complete = false
+            next
+          end
+
+          # An association target does not have to be a Mongoid document.
+          next unless klass.respond_to?(:encrypted?)
+
+          ([ klass ] + klass.descendants).uniq.each do |candidate|
+            next if path.include?(candidate)
+            return [ true, true ] if candidate.encrypted?
+
+            found, nested_complete = candidate.embedded_encryption_scan(path + [ candidate ])
+            return [ true, true ] if found
+
+            complete &&= nested_complete
+          end
+        end
+
+        [ false, complete ]
+      end
+
       # Override the key_id for the model.
       #
       # This method is solely for testing purposes and should not be used in

data/lib/mongoid/errors/config_redactor.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Mongoid
+  module Errors
+    # Redacts credentials from a client configuration hash before it is
+    # interpolated into an exception message.
+    module ConfigRedactor
+      extend self
+
+      REDACTED = '[REDACTED]'
+
+      # Top-level keys whose values should be replaced wholesale.
+      SENSITIVE_KEYS = %w[password auto_encryption_options].freeze
+
+      # Match the userinfo portion of a MongoDB connection string.
+      URI_USERINFO = %r{\A(mongodb(?:\+srv)?://)[^@/]+@}.freeze
+
+      # Return a copy of the given config hash with sensitive values redacted.
+      # Recurses into nested hashes so that, e.g., `:options =>
+      # { :auto_encryption_options => ... }` is also covered. Non-hash inputs
+      # are returned unchanged.
+      def redact(config)
+        return config unless config.is_a?(Hash)
+
+        config.each_with_object({}) do |(key, value), result|
+          result[key] = redact_value(key, value)
+        end
+      end
+
+      def redact_value(key, value)
+        if SENSITIVE_KEYS.include?(key.to_s)
+          REDACTED
+        elsif key.to_s == 'uri' && value.is_a?(String)
+          value.sub(URI_USERINFO, "\\1#{REDACTED}@")
+        elsif value.is_a?(Hash)
+          redact(value)
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/lib/mongoid/errors/in_memory_regexp_timeout.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Mongoid
+  module Errors
+    # This error is raised when evaluating a query in memory exceeds
+    # Mongoid::Config.in_memory_regexp_time_limit.
+    #
+    # What the limit bounds depends on the Ruby in use. Where per-Regexp
+    # timeouts are available it is the time spent executing regular
+    # expressions; elsewhere it is the elapsed time of the whole in-memory
+    # evaluation. The message is worded to hold either way.
+    class InMemoryRegexpTimeout < MongoidError
+      # Create the new error.
+      #
+      # @example Create the new in-memory regexp timeout error.
+      #   InMemoryRegexpTimeout.new(5.0)
+      #
+      # @param [ Float ] limit The limit that was exceeded, in seconds. Not
+      #   always the configured one: a global Regexp.timeout stricter than the
+      #   configuration takes its place.
+      def initialize(limit)
+        super(compose_message('in_memory_regexp_timeout', limit: limit))
+      end
+    end
+  end
+end

data/lib/mongoid/errors/invalid_encryption_schema.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Mongoid
+  module Errors
+    # This error is raised when two models writing to the same namespace
+    # describe a field, or the collection metadata, differently. The schema map
+    # has one entry per namespace, so the two descriptions cannot both be
+    # honored.
+    class InvalidEncryptionSchema < MongoidError
+      # Create the new error.
+      #
+      # @example Create the error.
+      #   InvalidEncryptionSchema.new('music.bands', 'properties.code')
+      #
+      # @param [ String | nil ] namespace The namespace being described.
+      # @param [ String ] path The conflicting path in the schema.
+      def initialize(namespace, path)
+        super(
+          compose_message(
+            'invalid_encryption_schema',
+            { namespace: namespace || 'the collection', path: path }
+          )
+        )
+      end
+    end
+  end
+end

data/lib/mongoid/errors/mixed_client_configuration.rb

@@ -16,7 +16,7 @@ module Mongoid
         super(
           compose_message(
             'mixed_client_configuration',
-            { name: name, config: config }
+            { name: name, config: ConfigRedactor.redact(config) }
           )
         )
       end

data/lib/mongoid/errors/no_client_database.rb

@@ -15,7 +15,7 @@ module Mongoid
         super(
           compose_message(
             'no_client_database',
-            { name: name, config: config }
+            { name: name, config: ConfigRedactor.redact(config) }
           )
         )
       end

data/lib/mongoid/errors/no_client_hosts.rb

@@ -15,7 +15,7 @@ module Mongoid
         super(
           compose_message(
             'no_client_hosts',
-            { name: name, config: config }
+            { name: name, config: ConfigRedactor.redact(config) }
           )
         )
       end

data/lib/mongoid/errors/no_encryption_schema.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Mongoid
+  module Errors
+    # This error is raised when a model declares encrypted fields, but the
+    # namespace it is about to be persisted to is not covered by the automatic
+    # encryption schema of the client in use. Without a schema the driver has
+    # nothing to encrypt with, and the fields would be stored in plaintext.
+    class NoEncryptionSchema < MongoidError
+      # Create the new error.
+      #
+      # @example Create the error.
+      #   NoEncryptionSchema.new(Band, 'music.bands', :default)
+      #
+      # @param [ Class ] klass The model class.
+      # @param [ String ] namespace The namespace the model resolved to.
+      # @param [ String | Symbol ] client The name of the client in use.
+      # @param [ String | nil ] field The encrypted field missing from the
+      #   schema entry, if the namespace itself is covered.
+      def initialize(klass, namespace, client, field = nil)
+        super(
+          compose_message(
+            field ? 'no_encryption_schema_field' : 'no_encryption_schema',
+            { klass: klass, namespace: namespace, client: client, field: field }
+          )
+        )
+      end
+    end
+  end
+end

data/lib/mongoid/errors/readonly_association.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Mongoid
+  module Errors
+    # Raised when attempting to write to a :through association, which is
+    # read-only.
+    class ReadonlyAssociation < MongoidError
+      # Instantiate the exception.
+      #
+      # @example Create the error.
+      #   ReadonlyAssociation.new(Physician, association)
+      #
+      # @param [ Class ] klass The owner class.
+      # @param [ Mongoid::Association::Relatable ] association The through association.
+      def initialize(klass, association)
+        super(
+          compose_message(
+            'readonly_association',
+            {
+              klass: klass,
+              name: association.name,
+              through: association.options[:through]
+            }
+          )
+        )
+      end
+    end
+  end
+end

data/lib/mongoid/errors.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'mongoid/errors/mongoid_error'
+require 'mongoid/errors/config_redactor'
 require 'mongoid/errors/ambiguous_relationship'
 require 'mongoid/errors/attribute_not_loaded'
 require 'mongoid/errors/callback'
@@ -11,6 +12,7 @@ require 'mongoid/errors/document_not_found'
 require 'mongoid/errors/empty_config_file'
 require 'mongoid/errors/immutable_attribute'
 require 'mongoid/errors/in_memory_collation_not_supported'
+require 'mongoid/errors/in_memory_regexp_timeout'
 require 'mongoid/errors/invalid_auto_encryption_configuration'
 require 'mongoid/errors/invalid_around_callback'
 require 'mongoid/errors/invalid_async_query_executor'
@@ -18,6 +20,7 @@ require 'mongoid/errors/invalid_collection'
 require 'mongoid/errors/invalid_config_file'
 require 'mongoid/errors/invalid_config_option'
 require 'mongoid/errors/invalid_dependent_strategy'
+require 'mongoid/errors/invalid_encryption_schema'
 require 'mongoid/errors/invalid_field'
 require 'mongoid/errors/invalid_field_option'
 require 'mongoid/errors/invalid_field_type'
@@ -51,6 +54,7 @@ require 'mongoid/errors/mixed_client_configuration'
 require 'mongoid/errors/nested_attributes_metadata_not_found'
 require 'mongoid/errors/no_default_client'
 require 'mongoid/errors/no_environment'
+require 'mongoid/errors/no_encryption_schema'
 require 'mongoid/errors/no_map_reduce_output'
 require 'mongoid/errors/no_metadata'
 require 'mongoid/errors/no_parent'
@@ -60,6 +64,7 @@ require 'mongoid/errors/no_client_database'
 require 'mongoid/errors/no_client_hosts'
 require 'mongoid/errors/readonly_attribute'
 require 'mongoid/errors/readonly_document'
+require 'mongoid/errors/readonly_association'
 require 'mongoid/errors/rollback'
 require 'mongoid/errors/sessions_not_supported'
 require 'mongoid/errors/scope_overwrite'