mongoose 9.7.0 → 9.7.1

package/lib/document.js

@@ -562,27 +562,26 @@ function $applyDefaultsToNested(val, path, doc) {
 Document.prototype.$__buildDoc = function(obj, fields, skipId, exclude, hasIncludedChildren) {
   const doc = {};
 
-  const paths = Object.keys(this.$__schema.paths).
-    // Don't build up any paths that are underneath a map, we don't know
-    // what the keys will be
-    filter(p => !p.includes('$*'));
+  const paths = Object.keys(this.$__schema.paths);
   const plen = paths.length;
   let ii = 0;
 
   for (; ii < plen; ++ii) {
     const p = paths[ii];
 
-    if (p === '_id') {
-      if (skipId) {
-        continue;
-      }
-      if (obj && '_id' in obj) {
-        continue;
-      }
+    // Don't build up any paths that are underneath a map, we don't know
+    // what the keys will be
+    if (p.includes('$*')) {
+      continue;
     }
 
     const path = this.$__schema.paths[p].splitPath();
     const len = path.length;
+    // This loop only creates intermediate objects for nested paths: it never
+    // sets any leaf values, so single-segment paths are no-ops.
+    if (len === 1) {
+      continue;
+    }
     const last = len - 1;
     let curPath = '';
     let doc_ = doc;
@@ -1133,7 +1132,11 @@ Document.prototype.$set = function $set(path, val, type, options) {
       return this;
     }
 
-    options = Object.assign({}, options, { _skipMinimizeTopLevel: false });
+    // Only need to clone if we're overwriting `_skipMinimizeTopLevel`, recursive
+    // calls treat a missing `_skipMinimizeTopLevel` as `false`
+    if (_skipMinimizeTopLevel) {
+      options = Object.assign({}, options, { _skipMinimizeTopLevel: false });
+    }
 
     for (let i = 0; i < len; ++i) {
       key = keys[i];
@@ -1220,15 +1223,9 @@ Document.prototype.$set = function $set(path, val, type, options) {
   val = handleSpreadDoc(val, true);
 
   // if this doc is being constructed we should not trigger getters
-  const priorVal = (() => {
-    if (this.$__.priorDoc != null) {
-      return this.$__.priorDoc.$__getValue(path);
-    }
-    if (constructing) {
-      return void 0;
-    }
-    return this.$__getValue(path);
-  })();
+  const priorVal = this.$__.priorDoc != null ?
+    this.$__.priorDoc.$__getValue(path) :
+    (constructing ? void 0 : this.$__getValue(path));
 
   if (pathType === 'nested' && val) {
     if (typeof val === 'object' && val != null) {
@@ -1410,13 +1407,10 @@ Document.prototype.$set = function $set(path, val, type, options) {
   try {
     // If the user is trying to set a ref path to a document with
     // the correct model name, treat it as populated
-    const refMatches = (() => {
+    const refMatches = !(val instanceof Document) ? false : (() => {
       if (schema.options == null) {
         return false;
       }
-      if (!(val instanceof Document)) {
-        return false;
-      }
       const model = val.constructor;
 
       // Check ref
@@ -2870,7 +2864,8 @@ Document.prototype.validate = async function validate(pathsToValidate, options)
 
     await this._execDocumentPostHooks('validate', options, error);
   } finally {
-    delete this.$__.validateModifiedOnly;
+    // Assign rather than `delete` to avoid putting `$__` in dictionary mode
+    this.$__.validateModifiedOnly = undefined;
     this.$op = null;
     this.$__.validating = null;
   }
@@ -2913,7 +2908,7 @@ function _completeValidate(doc) {
     }
   }
 
-  doc.$__.cachedRequired = {};
+  doc.$__.cachedRequired = null;
   doc.$emit('validate', doc);
   doc.constructor.emit('validate', doc);
 
@@ -3204,7 +3199,7 @@ function _pushNestedArrayPaths(val, paths, path) {
  * ignore
  */
 
-Document.prototype._execDocumentPreHooks = async function _execDocumentPreHooks(opName, options, argsForHooks) {
+Document.prototype._execDocumentPreHooks = function _execDocumentPreHooks(opName, options, argsForHooks) {
   const filter = buildMiddlewareFilter(options, 'pre');
   return this.$__middleware.execPre(opName, this, argsForHooks || [], { filter });
 };
@@ -3213,7 +3208,7 @@ Document.prototype._execDocumentPreHooks = async function _execDocumentPreHooks(
  * ignore
  */
 
-Document.prototype._execDocumentPostHooks = async function _execDocumentPostHooks(opName, options, error) {
+Document.prototype._execDocumentPostHooks = function _execDocumentPostHooks(opName, options, error) {
   const filter = buildMiddlewareFilter(options, 'post');
   return this.$__middleware.execPost(opName, this, [this], { error, filter });
 };
@@ -3630,7 +3625,11 @@ Document.prototype.$__reset = function reset() {
   // Clear atomics on dirty paths. Walk the modified and default paths
   // directly instead of calling $__dirty(), which builds intermediate
   // arrays, a Map, and does parent-path deduplication we don't need here.
-  this.$__resetAtomics();
+  // Skip entirely if the doc only has primitive values, because only arrays
+  // and maps have atomics.
+  if (!onlyPrimitiveValues) {
+    this.$__resetAtomics();
+  }
 
   this.$__.backup = {};
   this.$__.backup.activePaths = {

package/lib/helpers/document/applyDefaults.js

@@ -17,6 +17,11 @@ module.exports = function applyDefaults(doc, fields, exclude, hasIncludedChildre
     }
 
     const type = doc.$__schema.paths[p];
+    // `getDefault()` returns undefined when `defaultValue` is undefined, in which
+    // case this loop never modifies the doc, so skip traversing the path entirely.
+    if (type.defaultValue === undefined) {
+      continue;
+    }
     const path = type.splitPath();
     const len = path.length;
     if (path[len - 1] === '$*') {

package/lib/helpers/populate/createPopulateQueryFilter.js

@@ -25,7 +25,15 @@ module.exports = function createPopulateQueryFilter(ids, _match, _foreignField,
     for (let i = 0; i < _parentPaths.length - 1; ++i) {
       const cur = _parentPaths[i];
       if (match[cur] != null && match[cur].$elemMatch != null) {
-        match[cur].$elemMatch[foreignField.slice(cur.length + 1)] = trusted({ $in: ids });
+        // Copy rather than mutate so the user's `match` stays unchanged and split populate
+        // queries (gh-5890) each get their own `$in` rather than sharing one object
+        match[cur] = {
+          ...match[cur],
+          $elemMatch: {
+            ...match[cur].$elemMatch,
+            [foreignField.slice(cur.length + 1)]: trusted({ $in: ids })
+          }
+        };
         delete match[foreignField];
         break;
       }

package/lib/helpers/populate/getModelsMapForPopulate.js

@@ -649,11 +649,7 @@ function addModelNamesToMap(model, map, available, modelNames, options, data, re
       ids = matchIdsToRefPaths(ret, modelNamesForRefPath, modelName);
     }
 
-    const perDocumentLimit = options.perDocumentLimit == null ?
-      get(options, 'options.perDocumentLimit', null) :
-      options.perDocumentLimit;
-
-    if (!available[modelName] || perDocumentLimit != null) {
+    if (!available[modelName]) {
       const currentOptions = {
         model: Model
       };
@@ -682,6 +678,7 @@ function addModelNamesToMap(model, map, available, modelNames, options, data, re
         isVirtual: data.isVirtual,
         virtual: data.virtual,
         count: data.count,
+        isRefPath: !!data.isRefPath,
         [populateModelSymbol]: Model
       };
       map.push(available[modelName]);
@@ -692,6 +689,7 @@ function addModelNamesToMap(model, map, available, modelNames, options, data, re
       available[modelName].ids.push(ids);
       available[modelName].allIds.push(ret);
       available[modelName].unpopulatedValues.push(unpopulatedValue);
+      available[modelName].isRefPath = available[modelName].isRefPath || !!data.isRefPath;
       if (data.hasMatchFunction) {
         available[modelName].match.push(data.match);
       }

package/lib/helpers/populate/splitPopulateQuery.js

@@ -0,0 +1,81 @@
+'use strict';
+
+const createPopulateQueryFilter = require('./createPopulateQueryFilter');
+const get = require('../get');
+const utils = require('../../utils');
+
+module.exports = splitPopulateQuery;
+
+/*!
+ * If a single populate query would have more than this many elements in its `$in` filter,
+ * Mongoose splits the populate into a separate query per document to avoid going over
+ * MongoDB's 16 MB BSON size limit on queries. Overwritable for testing purposes. See gh-5890.
+ */
+
+splitPopulateQuery.maxInFilterLength = 50000;
+
+/*!
+ * Split a populate models-map entry into a separate query per document if either:
+ *
+ * 1. The `perDocumentLimit` option is set, so each document needs its own query with its
+ *    own `limit` (gh-7318), or
+ * 2. A single populate query for `mod` would have too many elements in its `$in` filter
+ *    (gh-5890). With multiple foreign fields, `createPopulateQueryFilter()` repeats the ids
+ *    under `$or` once per foreign field, so the threshold counts one copy of `ids` per
+ *    foreign field.
+ *
+ * Returns a list of `[mod, match, select, assignmentOpts]` params, one per document, for
+ * `_execPopulateQuery()`. Returns `null` if the populate query doesn't need to be split.
+ * A `null` `match` means the document has no ids to query: `_execPopulateQuery()` skips
+ * executing a query, and `_assign()` just sets the document's populated path to the
+ * default value.
+ *
+ * Splitting on document boundaries means each document's populated value is the result of
+ * exactly one query, so split entries can typically be assigned from only their own query's
+ * results (`_assignFromOwnResults`) rather than scanning every populate query's results.
+ * refPath is the exception: a single document's array can contain ids for multiple models,
+ * so assigning refPath populate results relies on every query's results being available for
+ * every document. refPath entries are therefore only split when `perDocumentLimit` requires
+ * it, not to keep the `$in` filter small. A single document whose ids alone overflow the
+ * BSON size limit cannot be split.
+ */
+
+function splitPopulateQuery(mod, ids, select, assignmentOpts) {
+  if (mod.docs.length <= 1) {
+    return null;
+  }
+  const perDocumentLimit = mod.options.perDocumentLimit == null ?
+    get(mod.options, 'options.perDocumentLimit', null) :
+    mod.options.perDocumentLimit;
+  const numInFilterElements = ids.length * mod.foreignField.size;
+  if (perDocumentLimit == null && (numInFilterElements <= splitPopulateQuery.maxInFilterLength || mod.isRefPath)) {
+    return null;
+  }
+
+  return mod.docs.map((doc, i) => {
+    const subMod = {
+      ...mod,
+      docs: [doc],
+      ids: [mod.ids[i]],
+      allIds: [mod.allIds[i]],
+      unpopulatedValues: [mod.unpopulatedValues[i]],
+      match: Array.isArray(mod.match) ? [mod.match[i]] : mod.match,
+      _assignFromOwnResults: !mod.isRefPath
+    };
+    let subIds = utils.array.flatten(subMod.ids, flatten);
+    subIds = utils.array.unique(subIds);
+    const match = subIds.length === 0 || subIds.every(utils.isNullOrUndefined) ?
+      null :
+      createPopulateQueryFilter(subIds, subMod.match, subMod.foreignField, subMod.model, subMod.options.skipInvalidIds);
+    return [subMod, match, select, assignmentOpts];
+  });
+}
+
+/*!
+ * ignore
+ */
+
+function flatten(item) {
+  // no need to include undefined values in our query
+  return undefined !== item;
+}

package/lib/helpers/schema/getKeysInSchemaOrder.js

@@ -3,25 +3,22 @@
 const get = require('../get');
 
 module.exports = function getKeysInSchemaOrder(schema, val, path) {
+  const valKeys = Object.keys(val);
+  if (valKeys.length <= 1) {
+    return valKeys;
+  }
+
   const schemaKeys = path != null ? Object.keys(get(schema.tree, path, {})) : Object.keys(schema.tree);
-  const valKeys = new Set(Object.keys(val));
+  const remaining = new Set(valKeys);
 
-  let keys;
-  if (valKeys.size > 1) {
-    keys = new Set();
-    for (const key of schemaKeys) {
-      if (valKeys.has(key)) {
-        keys.add(key);
-      }
+  const keys = [];
+  for (const key of schemaKeys) {
+    if (remaining.delete(key)) {
+      keys.push(key);
     }
-    for (const key of valKeys) {
-      if (!keys.has(key)) {
-        keys.add(key);
-      }
-    }
-    keys = Array.from(keys);
-  } else {
-    keys = Array.from(valKeys);
+  }
+  for (const key of remaining) {
+    keys.push(key);
   }
 
   return keys;

package/lib/model.js

@@ -71,6 +71,7 @@ const prepareDiscriminatorPipeline = require('./helpers/aggregate/prepareDiscrim
 const pushNestedArrayPaths = require('./helpers/model/pushNestedArrayPaths');
 const removeDeselectedForeignField = require('./helpers/populate/removeDeselectedForeignField');
 const setDottedPath = require('./helpers/path/setDottedPath');
+const splitPopulateQuery = require('./helpers/populate/splitPopulateQuery');
 const { buildMiddlewareFilter } = require('./helpers/buildMiddlewareFilter');
 const util = require('util');
 const utils = require('./utils');
@@ -4543,7 +4544,6 @@ async function _populatePath(model, docs, populateOptions) {
       mod.foreignField.clear();
       mod.foreignField.add(populateOptions.foreignField);
     }
-    const match = createPopulateQueryFilter(ids, mod.match, mod.foreignField, mod.model, mod.options.skipInvalidIds);
     if (assignmentOpts.excludeId) {
       // override the exclusion from the query so we can use the _id
       // for document matching during assignment. we'll delete the
@@ -4564,6 +4564,17 @@ async function _populatePath(model, docs, populateOptions) {
     } else if (mod.options.limit != null) {
       assignmentOpts.originalLimit = mod.options.limit;
     }
+
+    // Execute a separate query per document if `perDocumentLimit` is set (gh-7318), or if
+    // there are so many ids that the `$in` filter may overflow MongoDB's 16 MB BSON size
+    // limit on queries (gh-5890).
+    const splitParams = splitPopulateQuery(mod, ids, select, assignmentOpts);
+    if (splitParams != null) {
+      params.push(...splitParams);
+      continue;
+    }
+
+    const match = createPopulateQueryFilter(ids, mod.match, mod.foreignField, mod.model, mod.options.skipInvalidIds);
     params.push([mod, match, select, assignmentOpts]);
   }
   if (!hasOne) {
@@ -4584,6 +4595,9 @@ async function _populatePath(model, docs, populateOptions) {
 
   // Track deferred populates per-param (per model) to avoid mixing them
   const deferredPopulatesPerParam = new Map();
+  // Query results per param. Batches that were split off of a large populate query (gh-5890)
+  // are assigned from only their own query's results rather than the combined `vals`.
+  const valsByParam = [];
 
   if (populateOptions.ordered) {
     // Populate in series, primarily for transactions because MongoDB doesn't support multiple operations on
@@ -4591,7 +4605,10 @@ async function _populatePath(model, docs, populateOptions) {
     for (let i = 0; i < params.length; i++) {
       const arr = params[i];
       const { docs, deferredPopulates } = await _execPopulateQuery.apply(null, arr);
-      vals = vals.concat(docs);
+      valsByParam.push(docs);
+      if (!arr[0]._assignFromOwnResults) {
+        vals = vals.concat(docs);
+      }
       if (deferredPopulates.length > 0) {
         deferredPopulatesPerParam.set(i, deferredPopulates);
       }
@@ -4606,7 +4623,10 @@ async function _populatePath(model, docs, populateOptions) {
     const results = await Promise.all(promises);
     for (let i = 0; i < results.length; i++) {
       const { docs, deferredPopulates } = results[i];
-      vals = vals.concat(docs);
+      valsByParam.push(docs);
+      if (!params[i][0]._assignFromOwnResults) {
+        vals = vals.concat(docs);
+      }
       if (deferredPopulates.length > 0) {
         deferredPopulatesPerParam.set(i, deferredPopulates);
       }
@@ -4614,13 +4634,15 @@ async function _populatePath(model, docs, populateOptions) {
   }
 
 
-  for (const arr of params) {
+  for (let i = 0; i < params.length; i++) {
+    const arr = params[i];
     const mod = arr[0];
     const assignmentOpts = arr[3];
-    for (const val of vals) {
+    const valsForMod = mod._assignFromOwnResults ? valsByParam[i] : vals;
+    for (const val of valsForMod) {
       mod.options._childDocs.push(val);
     }
-    _assign(model, vals, mod, assignmentOpts);
+    _assign(model, valsForMod, mod, assignmentOpts);
   }
 
   // Handle deferred populate for cases with per-document match functions.
@@ -4660,13 +4682,15 @@ async function _populatePath(model, docs, populateOptions) {
     }
   }
 
-  for (const arr of params) {
-    removeDeselectedForeignField(arr[0].foreignField, arr[0].options, vals);
+  for (let i = 0; i < params.length; i++) {
+    const mod = params[i][0];
+    removeDeselectedForeignField(mod.foreignField, mod.options, mod._assignFromOwnResults ? valsByParam[i] : vals);
   }
-  for (const arr of params) {
-    const mod = arr[0];
+  for (let i = 0; i < params.length; i++) {
+    const mod = params[i][0];
     if (mod.options?.options?._leanTransform) {
-      for (const doc of vals) {
+      const valsForMod = mod._assignFromOwnResults ? valsByParam[i] : vals;
+      for (const doc of valsForMod) {
         mod.options.options._leanTransform(doc);
       }
     }
@@ -4678,6 +4702,12 @@ async function _populatePath(model, docs, populateOptions) {
  */
 
 function _execPopulateQuery(mod, match, select) {
+  // `null` match means `mod`'s documents have no ids to query, possible if a large populate
+  // was split into a query per document (gh-5890). Skip executing a query: `_assign()` still
+  // sets the documents' populated paths to the appropriate default value.
+  if (match == null) {
+    return Promise.resolve({ docs: [], deferredPopulates: [] });
+  }
   let subPopulate = clone(mod.options.populate);
   const queryOptions = {};
   if (mod.options.skip !== undefined) {

package/lib/plugins/saveSubdocs.js

@@ -16,7 +16,10 @@ module.exports = function saveSubdocs(schema) {
 };
 
 
-async function saveSubdocsPreSave() {
+// These hooks are deliberately not `async` so that they don't allocate a
+// promise and force an extra microtask hop on every `save()` when there are
+// no subdocuments. kareem only awaits hooks that return a promise.
+function saveSubdocsPreSave() {
   if (this.$isSubdocument) {
     return;
   }
@@ -28,15 +31,15 @@ async function saveSubdocsPreSave() {
   }
 
   const options = this.$__.saveOptions;
-  await Promise.all(subdocs.map(subdoc => subdoc._execDocumentPreHooks('save', options, [options])));
-
-  // Invalidate subdocs cache because subdoc pre hooks can add new subdocuments
-  if (this.$__.saveOptions) {
-    this.$__.saveOptions.__subdocs = null;
-  }
+  return Promise.all(subdocs.map(subdoc => subdoc._execDocumentPreHooks('save', options, [options]))).then(() => {
+    // Invalidate subdocs cache because subdoc pre hooks can add new subdocuments
+    if (this.$__.saveOptions) {
+      this.$__.saveOptions.__subdocs = null;
+    }
+  });
 }
 
-async function saveSubdocsPostSave() {
+function saveSubdocsPostSave() {
   if (this.$isSubdocument) {
     return;
   }
@@ -53,10 +56,10 @@ async function saveSubdocsPostSave() {
     promises.push(subdoc._execDocumentPostHooks('save', options));
   }
 
-  await Promise.all(promises);
+  return Promise.all(promises);
 }
 
-async function saveSubdocsPreDeleteOne() {
+function saveSubdocsPreDeleteOne() {
   const removedSubdocs = this.$__.removedSubdocs;
   if (!removedSubdocs?.length) {
     return;
@@ -68,10 +71,10 @@ async function saveSubdocsPreDeleteOne() {
     promises.push(subdoc._execDocumentPreHooks('deleteOne', options));
   }
 
-  await Promise.all(promises);
+  return Promise.all(promises);
 }
 
-async function saveSubdocsPostDeleteOne() {
+function saveSubdocsPostDeleteOne() {
   const removedSubdocs = this.$__.removedSubdocs;
   if (!removedSubdocs?.length) {
     return;
@@ -84,7 +87,7 @@ async function saveSubdocsPostDeleteOne() {
   }
 
   this.$__.removedSubdocs = null;
-  await Promise.all(promises);
+  return Promise.all(promises);
 }
 
 

package/lib/schema/objectId.js

@@ -317,7 +317,13 @@ function resetId(v) {
  */
 
 SchemaObjectId.prototype.toJSONSchema = function toJSONSchema(options) {
-  return this._createJSONSchemaTypeDefinition('string', 'objectId', options);
+  const jsonSchema = this._createJSONSchemaTypeDefinition('string', 'objectId', options);
+  // `bsonType: 'objectId'` already validates ObjectIds, so the regex is only useful
+  // for the portable `type: 'string'` representation (e.g. for ajv, zod, etc.)
+  if (!options?.useBsonType) {
+    jsonSchema.pattern = '^[A-Fa-f0-9]{24}$';
+  }
+  return jsonSchema;
 };
 
 SchemaObjectId.prototype.autoEncryptionType = function autoEncryptionType() {

package/lib/stateMachine.js

@@ -69,8 +69,10 @@ StateMachine.prototype._changeState = function _changeState(path, nextState) {
   if (prevState === nextState) {
     return;
   }
-  const prevBucket = this.states[prevState];
-  if (prevBucket) delete prevBucket[path];
+  if (prevState !== undefined) {
+    const prevBucket = this.states[prevState];
+    if (prevBucket) delete prevBucket[path];
+  }
 
   this.paths[path] = nextState;
   this.states[nextState] = this.states[nextState] || {};
@@ -86,13 +88,16 @@ StateMachine.prototype.clear = function clear(state) {
     return;
   }
   const keys = Object.keys(this.states[state]);
+  if (keys.length === 0) {
+    return;
+  }
+  // Replace the bucket rather than deleting each key: repeated `delete` puts
+  // the object in dictionary mode, which is significantly slower.
+  this.states[state] = {};
   let i = keys.length;
-  let path;
 
   while (i--) {
-    path = keys[i];
-    delete this.states[state][path];
-    delete this.paths[path];
+    delete this.paths[keys[i]];
   }
 };
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mongoose",
   "description": "Mongoose MongoDB ODM",
-  "version": "9.7.0",
+  "version": "9.7.1",
   "author": "Guillermo Rauch <guillermo@learnboost.com>",
   "keywords": [
     "mongodb",
@@ -63,7 +63,8 @@
     "tstyche": "^7.0.0",
     "typescript": "5.9.3",
     "typescript-eslint": "^8.31.1",
-    "uuid": "14.0.0"
+    "uuid": "14.0.0",
+    "xss": "1.0.15"
   },
   "directories": {
     "lib": "./lib/mongoose"