mongoid 9.1.0 → 9.1.2

data/lib/mongoid/persistable/creatable.rb

@@ -39,13 +39,79 @@ module Mongoid
         { atomic_insert_modifier => { atomic_position => as_attributes } }
       end
 
+      # Determine whether merging the touch updates into the insert
+      # operations would produce an update whose operators target both a
+      # path and one of its ancestors or descendants. MongoDB rejects such
+      # an update with error 40. This can happen for embeds_many inserts,
+      # where the operations target the array (via $push) but the touches
+      # include a path inside that array (e.g. items.0.updated_at) when a
+      # sibling has a pending touch.
+      #
+      # @api private
+      #
+      # @param [ Hash ] operations The atomic insert operations.
+      # @param [ Hash ] touches The touch updates to merge.
+      #
+      # @return [ true | false ] Whether any touch path conflicts with a
+      #   path the insert operations target.
+      def conflicting_touch_paths?(operations, touches)
+        targeted = operations.each_value.flat_map do |doc|
+          doc.respond_to?(:keys) ? doc.keys.map(&:to_s) : []
+        end
+
+        touches.each_key.any? do |touch_path|
+          path = touch_path.to_s
+          targeted.any? do |target|
+            path == target || path.start_with?("#{target}.") ||
+              target.start_with?("#{path}.")
+          end
+        end
+      end
+
+      # Rewrite the indices in the touch updates to the positional operator,
+      # as the final update will do to the merged operations. The touches of
+      # the parent chain target the inserted element, so rewriting them is
+      # safe, and merging the rewritten keys into the operations lets the
+      # final update reuse them as-is. A pending touch on a different element
+      # of the same array, however, would be rewritten to the same positional
+      # path and silently touch the wrong document; in that case this method
+      # returns nil and the caller persists the touch updates in their own
+      # round-trip after the insert instead of merging them.
+      #
+      # @api private
+      #
+      # @param [ Hash ] selector The update selector.
+      # @param [ Document ] parent The parent document being touched.
+      # @param [ Symbol, String, nil ] field The parent's custom touch field.
+      # @param [ Hash ] touches The touch updates to rewrite.
+      #
+      # @return [ Hash | nil ] The rewritten touch updates, or nil if any
+      #   touch would be misdirected by the rewriting.
+      def rewritten_touch_updates(selector, parent, field, touches)
+        rewritten = positionally(selector, { '$set' => touches })['$set']
+
+        chain_paths = parent._touchable_chain_paths(field)
+        touches.each_key do |key|
+          path = key.to_s
+          next if rewritten.key?(path)
+
+          return nil unless chain_paths.include?(path)
+        end
+        rewritten
+      end
+
       # Insert the embedded document.
       #
       # When the parent association is touchable (which is the default for
-      # +embedded_in+), the touch timestamp updates are merged into the
-      # same +update_one+ call that performs the insert. This avoids a
-      # second round-trip that the +after_save+ touch callback would
-      # otherwise issue.
+      # +embedded_in+), the touch updates are merged into the same
+      # +update_one+ call that performs the insert. This avoids a second
+      # round-trip that the +after_save+ touch callback would otherwise
+      # issue. The merge is skipped when a touch path would conflict with a
+      # path the insert targets (see +conflicting_touch_paths?+) or when
+      # rewriting the touch indices to the positional operator would
+      # misdirect a touch (see +rewritten_touch_updates+); such touches are
+      # persisted in their own round-trip right after the insert, so their
+      # persistence does not depend on the callback chain running.
       #
       # @api private
       #
@@ -62,19 +128,59 @@ module Mongoid
           selector = _parent.atomic_selector
           operations = atomic_inserts
 
-          if _touchable_parent?
-            touches = _parent._gather_touch_updates(Time.current)
-            if touches.present?
-              operations['$set'] = touches
-              Threaded.begin_touch_merged(self)
-            end
+          deferred_touches = merge_touch_updates(selector, operations)
+
+          begin
+            _root.collection.find(selector).update_one(
+              positionally(selector, operations),
+              session: _session
+            )
+            _root.persist_atomic_operations('$set' => deferred_touches) if deferred_touches
+          rescue StandardError
+            # If the insert failed, the after_save callback will not run to
+            # consume and clear the merged-touch flag, so clear it here.
+            Threaded.exit_touch_merged(self)
+            raise
           end
+        end
+      end
+
+      # Merge the parent chain's pending touch updates into the insert
+      # operations when doing so would not produce a conflicting update
+      # (see +conflicting_touch_paths?+ and +rewritten_touch_updates+).
+      # Either way, marks the touch as merged so the after_save callback
+      # does not persist the updates a second time. Returns the touch
+      # updates that could not be merged; they are persisted in their own
+      # round-trip right after the insert, rather than relying on the
+      # after_save callback, so that an aborted callback chain cannot
+      # silently drop them.
+      #
+      # @api private
+      #
+      # @param [ Hash ] selector The update selector.
+      # @param [ Hash ] operations The atomic insert operations, modified
+      #   in place when the touches are merged.
+      #
+      # @return [ Hash | nil ] The deferred touch updates, if any.
+      def merge_touch_updates(selector, operations)
+        return nil unless _touchable_parent?
 
-          _root.collection.find(selector).update_one(
-            positionally(selector, operations),
-            session: _session
-          )
+        field = _association&.inverse_association&.touch_field
+        touches = _parent._gather_touch_updates(Time.current, field)
+        return nil if touches.blank?
+
+        if conflicting_touch_paths?(operations, touches)
+          deferred_touches = touches
+        else
+          rewritten = rewritten_touch_updates(selector, _parent, field, touches)
+          if rewritten
+            operations['$set'] = (operations['$set'] || {}).merge(rewritten)
+          else
+            deferred_touches = touches
+          end
         end
+        Threaded.begin_touch_merged(self)
+        deferred_touches
       end
 
       # Insert the root document.

data/lib/mongoid/persistable/updatable.rb

@@ -103,7 +103,7 @@ module Mongoid
 
         process_flagged_destroys
         update_children = cascadable_children(:update)
-        process_touch_option(options, update_children) do
+        process_touch_option(options) do
           run_all_callbacks_for_update(update_children) do
             result = yield(self)
             self.previously_new_record = false
@@ -160,24 +160,22 @@ module Mongoid
         end
       end
 
-      # If there is a touch option and it is false, this method will call the
-      # timeless method so that the updated_at attribute is not updated. It
-      # will call the timeless method on all of the cascadable children as
-      # well. Note that timeless is cleared in the before_update callback.
+      # If there is a touch option and it is false, this method suppresses
+      # timestamping for the duration of the update using a block-based
+      # timeless scope, which covers this document and every cascaded child
+      # (at any nesting depth), and also suppresses touch callbacks.
       #
       # @param [ Hash ] options The options.
-      # @param [ Array<Document> ] children The children that the :update
-      #   callbacks will be executed on.
       #
       # @option options [ true | false ] :touch Whether or not the updated_at
       #   attribute will be updated with the current time.
-      def process_touch_option(options, children, &block)
+      def process_touch_option(options, &block)
         if options.fetch(:touch, true)
           yield
         else
-          timeless
-          children.each(&:timeless)
-          suppress_touch_callbacks(&block)
+          Mongoid::Timestamps::Timeless.with_timeless do
+            suppress_touch_callbacks(&block)
+          end
         end
       end
 

data/lib/mongoid/persistable.rb

@@ -142,6 +142,21 @@ module Mongoid
       raise Errors::Callback.new(self.class, method)
     end
 
+    # Persist the atomic operations.
+    #
+    # @api private
+    #
+    # @example Persist the atomic operations.
+    #   persist_atomic_operations(ops)
+    #
+    # @param [ Hash ] operations The atomic operations.
+    def persist_atomic_operations(operations)
+      return unless persisted? && operations && !operations.empty?
+
+      selector = atomic_selector
+      _root.collection.find(selector).update_one(positionally(selector, operations), session: _session)
+    end
+
     private
 
     # Are we executing an atomically block on the current document?
@@ -300,20 +315,5 @@ module Mongoid
         persist_atomic_operations(operation)
       end
     end
-
-    # Persist the atomic operations.
-    #
-    # @api private
-    #
-    # @example Persist the atomic operations.
-    #   persist_atomic_operations(ops)
-    #
-    # @param [ Hash ] operations The atomic operations.
-    def persist_atomic_operations(operations)
-      return unless persisted? && operations && !operations.empty?
-
-      selector = atomic_selector
-      _root.collection.find(selector).update_one(positionally(selector, operations), session: _session)
-    end
   end
 end

data/lib/mongoid/persistence_context.rb

@@ -125,6 +125,8 @@ module Mongoid
 
         client = client.with(options) unless options.empty?
 
+        verify_encryption_schema!(client)
+
         client
       end
     end
@@ -199,6 +201,81 @@ module Mongoid
       name.respond_to?(:call) ? name.call.to_sym : name.to_sym
     end
 
+    # Refuse to use a client that would store a model's encrypted fields in
+    # plaintext.
+    #
+    # The automatic encryption schema is keyed by namespace and is built once,
+    # when the client is created. The driver looks the target namespace up in
+    # that schema, and when it is absent it asks the server for a schema
+    # instead; a collection without a validator then yields no encryption at
+    # all, and no error. Checking the namespace here is what turns that silent
+    # downgrade into a failure. When the namespace is present, its entry is
+    # checked against the fields the model declares as encrypted too: a
+    # hand-written schema can cover the namespace and still leave out a field.
+    #
+    # @param [ Mongo::Client ] client The client this context resolved to.
+    #
+    # @raise [ Errors::NoEncryptionSchema ] if the model needs an encryption
+    #   schema and the client's schema does not cover the target namespace, or
+    #   the namespace entry does not cover every encrypted field of the model.
+    def verify_encryption_schema!(client)
+      klass = @object.is_a?(Class) ? @object : @object.class
+      # A model whose encrypted fields all live on embedded models declares no
+      # encryption of its own, and still needs a schema for its collection.
+      return unless klass.respond_to?(:requires_encryption_schema?) && klass.requires_encryption_schema?
+      # An embedded document is persisted as part of its parent, so the
+      # parent's context is what governs the namespace.
+      return if klass.embedded?
+      # Collection and index management sends no document data, so it does not
+      # need an encryption-capable client.
+      return if Threaded.managing_collection?
+
+      schema_map = client.options.dig(:auto_encryption_options, :schema_map)
+      namespace = "#{client.database.name}.#{collection_name}"
+      entry = schema_map && schema_map[namespace]
+      raise Errors::NoEncryptionSchema.new(klass, namespace, client_name) unless entry
+
+      # The schema map may have been written by hand, with symbol keys rather
+      # than strings; both are tolerated at every level.
+      Mongoid.config.encrypted_paths_for(klass).each do |path|
+        next if encryption_schema_has_path?(entry, path)
+
+        raise Errors::NoEncryptionSchema.new(
+          klass, namespace, client_name, path.join('.')
+        )
+      end
+    end
+
+    # Whether a schema entry encrypts the property at the given path.
+    #
+    # @param [ Hash ] entry The namespace schema entry.
+    # @param [ Array<String> ] path The property path, outermost first.
+    #
+    # @return [ true | false ] Whether the leaf is marked as encrypted.
+    def encryption_schema_has_path?(entry, path)
+      node = entry
+      path.each do |name|
+        properties = schema_value(node, 'properties')
+        return false unless properties.is_a?(Hash)
+
+        node = schema_value(properties, name)
+        return false unless node.is_a?(Hash)
+      end
+      !schema_value(node, 'encrypt').nil?
+    end
+
+    # Read a schema key that may be written as a string or a symbol.
+    #
+    # @param [ Hash ] hash The schema object.
+    # @param [ String ] key The key to read.
+    #
+    # @return [ Object ] The value, or nil when the key is absent.
+    def schema_value(hash, key)
+      return hash[key] if hash.key?(key)
+
+      hash[key.to_sym]
+    end
+
     def client_options
       @client_options ||= begin
         opts = options.select do |k, _v|

data/lib/mongoid/search_indexable.rb

@@ -69,6 +69,9 @@ module Mongoid
     # @param [ Integer ] limit The maximum number of results (default: 10).
     # @param [ Integer | nil ] num_candidates The number of candidates to
     #   consider during the ANN search; defaults to limit * 10.
+    # @param [ true | false ] exact Use exact nearest-neighbor (ENN) search
+    #   instead of ANN (default: false). When true, numCandidates is omitted.
+    #   Required when using a flat vector search index.
     # @param [ Hash | nil ] filter An optional MongoDB filter to pre-filter
     #   candidates before scoring.
     # @param [ Array ] pipeline Additional aggregation stages to append after
@@ -76,7 +79,7 @@ module Mongoid
     #
     # @return [ Array<Mongoid::Document> ] matching documents, each with
     #   a populated +vector_search_score+ attribute.
-    def vector_search(index: nil, path: nil, limit: 10, num_candidates: nil, filter: nil, pipeline: [])
+    def vector_search(index: nil, path: nil, limit: 10, num_candidates: nil, exact: false, filter: nil, pipeline: []) # rubocop:disable Metrics/ParameterLists
       _index, resolved_path = self.class.send(:resolve_vector_index, index, path)
       query_vector = public_send(resolved_path)
 
@@ -85,17 +88,19 @@ module Mongoid
               "#{resolved_path} is nil on this document; cannot perform vector search"
       end
 
-      self_filter = { '_id' => { '$ne' => _id } }
-      combined_filter = filter ? { '$and' => [ self_filter, filter ] } : self_filter
+      self_exclusion = { '$match' => { '_id' => { '$ne' => _id } } }
+      post_pipeline = [ self_exclusion, { '$limit' => limit }, *Array(pipeline) ]
+      effective_candidates = num_candidates || (limit * 10)
 
       self.class.vector_search(
         query_vector,
         index: index,
         path: path,
-        limit: limit,
-        num_candidates: num_candidates,
-        filter: combined_filter,
-        pipeline: pipeline
+        limit: limit + 1,
+        num_candidates: effective_candidates,
+        exact: exact,
+        filter: filter,
+        pipeline: post_pipeline
       )
     end
 
@@ -129,19 +134,20 @@ module Mongoid
               "#{resolved_path} is nil on this document; cannot perform auto-embed search"
       end
 
-      self_filter = { '_id' => { '$ne' => _id } }
-      combined_filter = filter ? { '$and' => [ self_filter, filter ] } : self_filter
+      self_exclusion = { '$match' => { '_id' => { '$ne' => _id } } }
+      post_pipeline = [ self_exclusion, { '$limit' => limit }, *Array(pipeline) ]
+      effective_candidates = num_candidates || (limit * 10)
 
       self.class.auto_embed_search(
         text,
         index: index,
         path: path,
-        limit: limit,
-        num_candidates: num_candidates,
-        filter: combined_filter,
+        limit: limit + 1,
+        num_candidates: effective_candidates,
+        filter: filter,
         exact: exact,
         model: model,
-        pipeline: pipeline
+        pipeline: post_pipeline
       )
     end
 
@@ -155,7 +161,9 @@ module Mongoid
       def create_search_indexes
         return if search_index_specs.empty?
 
-        collection.search_indexes.create_many(search_index_specs)
+        Threaded.with_collection_management do
+          collection.search_indexes.create_many(search_index_specs)
+        end
       end
 
       # Waits for the named search indexes to be created.
@@ -186,7 +194,9 @@ module Mongoid
       # @option options [ Hash ] :aggregate The options hash to pass to the
       #    aggregate command (optional)
       def search_indexes(options = {})
-        collection.search_indexes(options)
+        Threaded.with_collection_management do
+          collection.search_indexes(options)
+        end
       end
 
       # Removes the search index specified by the given name or id. Either
@@ -195,12 +205,14 @@ module Mongoid
       # @param [ String | nil ] name the name of the index to remove
       # @param [ String | nil ] id the id of the index to remove
       def remove_search_index(name: nil, id: nil)
-        logger.info(
-          "MONGOID: Removing search index '#{name || id}' " \
-          "on collection '#{collection.name}'."
-        )
+        Threaded.with_collection_management do
+          logger.info(
+            "MONGOID: Removing search index '#{name || id}' " \
+            "on collection '#{collection.name}'."
+          )
 
-        collection.search_indexes.drop_one(name: name, id: id)
+          collection.search_indexes.drop_one(name: name, id: id)
+        end
       end
 
       # Request the removal of all registered search indexes. Note
@@ -253,6 +265,15 @@ module Mongoid
       #     vector_search_index :my_vector_index, { fields: [...] }
       #   end
       #
+      # @example Create a flat vector search index.
+      #   class Person
+      #     include Mongoid::Document
+      #     vector_search_index fields: [
+      #       { type: 'vector', path: 'embedding', numDimensions: 1536,
+      #         similarity: 'cosine', indexingMethod: 'flat' }
+      #     ]
+      #   end
+      #
       # @param [ Symbol | String | Hash ] name_or_defn Either the name of the index to
       #    define, or the index definition.
       # @param [ Hash ] defn The vector search index definition.
@@ -260,6 +281,8 @@ module Mongoid
         name = name_or_defn
         name, defn = nil, name if name.is_a?(Hash)
 
+        validate_vector_index_definition!(defn)
+
         spec = { type: 'vectorSearch', definition: defn }.tap { |s| s[:name] = name.to_s if name }
         search_index_specs.push(spec)
 
@@ -290,22 +313,25 @@ module Mongoid
       #   consider during the ANN search; defaults to limit * 10.
       # @param [ Hash | nil ] filter An optional MongoDB filter to pre-filter
       #   candidates before scoring.
+      # @param [ true | false ] exact Use exact nearest-neighbor (ENN) search
+      #   instead of ANN (default: false). When true, numCandidates is omitted.
+      #   Required when using a flat vector search index.
       # @param [ Array ] pipeline Additional aggregation stages to append after
       #   the vector search and score projection.
       #
       # @return [ Array<Mongoid::Document> ] matching documents, each with
       #   a populated +vector_search_score+ attribute.
-      def vector_search(vector, index: nil, path: nil, limit: 10, num_candidates: nil, filter: nil, pipeline: []) # rubocop:disable Metrics/ParameterLists
+      def vector_search(vector, index: nil, path: nil, limit: 10, num_candidates: nil, exact: false, filter: nil, pipeline: []) # rubocop:disable Metrics/ParameterLists
         resolved_index, resolved_path = resolve_vector_index(index, path)
-        num_candidates ||= limit * 10
 
         vs_options = {
           'index' => resolved_index,
           'path' => resolved_path,
           'queryVector' => vector,
-          'numCandidates' => num_candidates,
           'limit' => limit
         }
+        vs_options['numCandidates'] = num_candidates || (limit * 10) unless exact
+        vs_options['exact'] = true if exact
         vs_options['filter'] = filter if filter
 
         agg_pipeline = [
@@ -374,13 +400,32 @@ module Mongoid
 
       private
 
+      # Validates the vector index definition, raising ArgumentError for
+      # combinations that MongoDB does not support.
+      #
+      # @param [ Hash ] defn The vector search index definition.
+      def validate_vector_index_definition!(defn)
+        fields = defn[:fields] || defn['fields'] || []
+        fields.each do |field|
+          method = field[:indexingMethod] || field['indexingMethod']
+          next unless method.to_s == 'flat'
+          next unless field[:hnswOptions] || field['hnswOptions']
+
+          raise ArgumentError, 'hnswOptions is only supported with indexingMethod: hnsw'
+        end
+      end
+
       # Retrieves the index records for the indexes with the given names.
       #
       # @param [ Array<String> ] names the index names to query
       #
       # @return [ Array<Hash> ] the raw index documents
       def get_indexes(names)
-        collection.search_indexes.select { |i| names.include?(i['name']) }
+        # Reading index metadata sends no document data, so it is exempt from
+        # the encryption schema check, like the other index operations here.
+        Threaded.with_collection_management do
+          collection.search_indexes.select { |i| names.include?(i['name']) }
+        end
       end
 
       # Resolves the index name and vector path from the declared specs,

data/lib/mongoid/serializable.rb

@@ -127,10 +127,18 @@ module Mongoid
       inclusions = options[:include]
       relation_names(inclusions).each do |name|
         association = relations[name.to_s]
-        if association && relation = send(association.name)
-          attributes[association.name.to_s] =
-            relation.serializable_hash(relation_options(inclusions, options, name))
-        end
+        next unless association
+
+        relation =
+          if association.fallback?
+            without_autobuild { send(association.name) }
+          else
+            send(association.name)
+          end
+        next unless relation
+
+        attributes[association.name.to_s] =
+          relation.serializable_hash(relation_options(inclusions, options, name))
       end
     end
 

data/lib/mongoid/tasks/database.rb

@@ -64,15 +64,18 @@ module Mongoid
       # @param [ true | false ] wait whether to wait for the indexes to be
       #   built.
       def create_search_indexes(models = ::Mongoid.models, wait: true)
-        searchable = models.select { |m| m.search_index_specs.any? }
+        Threaded.with_collection_management do
+          searchable = models.select { |m| m.search_index_specs.any? }
 
-        # queue up the search index creation requests
-        index_names_by_model = searchable.each_with_object({}) do |model, obj|
-          logger.info("MONGOID: Creating search indexes on #{model}...")
-          obj[model] = model.create_search_indexes
-        end
+          # queue up the search index creation requests
+          index_names_by_model = searchable.each_with_object({}) do |model, obj|
+            logger.info("MONGOID: Creating search indexes on #{model}...")
+            obj[model] = model.create_search_indexes
+          end
 
-        wait_for_search_indexes(index_names_by_model) if wait
+          wait_for_search_indexes(index_names_by_model) if wait
+          index_names_by_model
+        end
       end
 
       # Return the list of indexes by model that exist in the database but aren't
@@ -83,6 +86,12 @@ module Mongoid
       #
       # @return [ Array<Hash> ] The list of undefined indexes by model.
       def undefined_indexes(models = ::Mongoid.models)
+        Threaded.with_collection_management do
+          undefined_indexes_for(models)
+        end
+      end
+
+      def undefined_indexes_for(models)
         undefined_by_model = {}
 
         models.each do |model|
@@ -116,15 +125,17 @@ module Mongoid
       #
       # @return [ Hash{Class => Array(Hash)}] The list of indexes that were removed by model.
       def remove_undefined_indexes(models = ::Mongoid.models)
-        undefined_indexes(models).each do |model, indexes|
-          indexes.each do |index|
-            key = index['key'].symbolize_keys
-            collection = model.collection
-            collection.indexes(session: model.send(:_session)).drop_one(key)
-            logger.info(
-              "MONGOID: Removed index '#{index['name']}' on collection " +
-              "'#{collection.name}' in database '#{collection.database.name}'."
-            )
+        Threaded.with_collection_management do
+          undefined_indexes(models).each do |model, indexes|
+            indexes.each do |index|
+              key = index['key'].symbolize_keys
+              collection = model.collection
+              collection.indexes(session: model.send(:_session)).drop_one(key)
+              logger.info(
+                "MONGOID: Removed index '#{index['name']}' on collection " +
+                "'#{collection.name}' in database '#{collection.database.name}'."
+              )
+            end
           end
         end
       end
@@ -172,6 +183,12 @@ module Mongoid
       #
       # @return [ Array<Class> ] The sharded models
       def shard_collections(models = ::Mongoid.models)
+        Threaded.with_collection_management do
+          shard_collections_for(models)
+        end
+      end
+
+      def shard_collections_for(models)
         models.map do |model|
           next if model.shard_config.nil?