mongoose 9.6.3 → 9.7.1

package/lib/model.js

@@ -7,6 +7,10 @@
 const Aggregate = require('./aggregate');
 const ChangeStream = require('./cursor/changeStream');
 const Document = require('./document');
+const { createTracedChannel } = require('./tracing');
+const { trace: traceSave } = createTracedChannel('mongoose:model:save');
+const { trace: traceInsertMany } = createTracedChannel('mongoose:model:insertMany');
+const { trace: traceBulkWrite } = createTracedChannel('mongoose:model:bulkWrite');
 const DocumentNotFoundError = require('./error/notFound');
 const EventEmitter = require('events').EventEmitter;
 const Kareem = require('kareem');
@@ -37,6 +41,7 @@ const applyVirtualsHelper = require('./helpers/document/applyVirtuals');
 const assignVals = require('./helpers/populate/assignVals');
 const castBulkWrite = require('./helpers/model/castBulkWrite');
 const clone = require('./helpers/clone');
+const convertErrorToStandardSchemaIssues = require('./standardSchema/convertErrorToIssues');
 const createPopulateQueryFilter = require('./helpers/populate/createPopulateQueryFilter');
 const decorateUpdateWithVersionKey = require('./helpers/update/decorateUpdateWithVersionKey');
 const getDefaultBulkwriteResult = require('./helpers/getDefaultBulkwriteResult');
@@ -66,6 +71,7 @@ const prepareDiscriminatorPipeline = require('./helpers/aggregate/prepareDiscrim
 const pushNestedArrayPaths = require('./helpers/model/pushNestedArrayPaths');
 const removeDeselectedForeignField = require('./helpers/populate/removeDeselectedForeignField');
 const setDottedPath = require('./helpers/path/setDottedPath');
+const splitPopulateQuery = require('./helpers/populate/splitPopulateQuery');
 const { buildMiddlewareFilter } = require('./helpers/buildMiddlewareFilter');
 const util = require('util');
 const utils = require('./utils');
@@ -663,19 +669,29 @@ Model.prototype.save = async function save(options) {
 
   this.$__.saveOptions = options;
 
-  try {
-    await this.$__save(options);
-  } catch (error) {
-    this.$__handleReject(error);
-    throw error;
-  } finally {
-    this.$__.saving = null;
-    this.$__.saveOptions = null;
-    this.$__.$versionError = null;
-    this.$op = null;
-  }
-
-  return this;
+  const _this = this;
+  return traceSave(async function maybeTracedSave() {
+    try {
+      await _this.$__save(options);
+    } catch (error) {
+      _this.$__handleReject(error);
+      throw error;
+    } finally {
+      _this.$__.saving = null;
+      _this.$__.saveOptions = null;
+      _this.$__.$versionError = null;
+      _this.$op = null;
+    }
+
+    return _this;
+  }, () => ({
+    operation: 'save',
+    collection: _this.constructor.collection.name,
+    database: _this.constructor.db?.name,
+    serverAddress: _this.constructor.db?.host,
+    serverPort: _this.constructor.db?.port,
+    args: { options }
+  }));
 };
 
 Model.prototype.$save = Model.prototype.save;
@@ -3015,6 +3031,18 @@ Model.insertMany = async function insertMany(arr, options) {
     throw new MongooseError('Model.insertMany() no longer accepts a callback');
   }
 
+  const ThisModel = this;
+  return traceInsertMany(function maybeTracedInsertMany() { return _insertMany.call(ThisModel, arr, options); }, () => ({
+    operation: 'insertMany',
+    collection: ThisModel.collection.name,
+    database: ThisModel.db?.name,
+    serverAddress: ThisModel.db?.host,
+    serverPort: ThisModel.db?.port,
+    args: { docs: arr, options }
+  }));
+};
+
+async function _insertMany(arr, options) {
   options = options || {};
   const preFilter = buildMiddlewareFilter(options, 'pre');
   const postFilter = buildMiddlewareFilter(options, 'post');
@@ -3250,7 +3278,7 @@ Model.insertMany = async function insertMany(arr, options) {
 
   const [result] = await this._middleware.execPost('insertMany', this, [docAttributes], { filter: postFilter });
   return result;
-};
+}
 
 /*!
  * ignore
@@ -3378,6 +3406,19 @@ Model.bulkWrite = async function bulkWrite(ops, options) {
       typeof arguments[2] === 'function') {
     throw new MongooseError('Model.bulkWrite() no longer accepts a callback');
   }
+
+  const ThisModel = this;
+  return traceBulkWrite(function maybeTracedBulkWrite() { return _bulkWrite.call(ThisModel, ops, options); }, () => ({
+    operation: 'bulkWrite',
+    collection: ThisModel.collection.name,
+    database: ThisModel.db?.name,
+    serverAddress: ThisModel.db?.host,
+    serverPort: ThisModel.db?.port,
+    args: { ops, options }
+  }));
+};
+
+async function _bulkWrite(ops, options) {
   options = options || {};
   const preFilter = buildMiddlewareFilter(options, 'pre');
   const postFilter = buildMiddlewareFilter(options, 'post');
@@ -3516,7 +3557,7 @@ Model.bulkWrite = async function bulkWrite(ops, options) {
   await this.hooks.execPost('bulkWrite', this, [res], { filter: postFilter });
 
   return res;
-};
+}
 
 /**
  * Takes an array of documents, gets the changes and inserts/updates documents in the database
@@ -3861,7 +3902,7 @@ Model.buildBulkWriteOperations = function buildBulkWriteOperations(documents, op
         throw new MongooseError(`documents.${i} was not a mongoose document, documents must be an array of mongoose documents (instanceof mongoose.Document).`);
       }
       if (options.validateBeforeSave == null || options.validateBeforeSave) {
-        const err = document.validateSync();
+        const err = document.$__validateSync();
         if (err != null) {
           throw err;
         }
@@ -4188,13 +4229,24 @@ Model.aggregate = function aggregate(pipeline, options) {
  *       age: { type: Number, required: true }
  *     });
  *
+ *     // Succeeds
+ *     await Model.validate({ name: 'John Smith', age: 31 });
+ *
  *     try {
- *       await Model.validate({ name: null }, ['name'])
+ *       await Model.validate({ name: null });
  *     } catch (err) {
  *       err instanceof mongoose.Error.ValidationError; // true
  *       Object.keys(err.errors); // ['name']
  *     }
  *
+ * Note: the `pathsToSkip` and `pathsToValidate` options **only** apply to validation, not
+ * casting. This function will still throw an error for values that cannot be casted to the
+ * schema-specified type. Remove any paths you do not want to cast.
+ *
+ *     // The following will still throw an error because the value of `age` cannot be
+ *     // casted to a number. Remove the `age` property before calling `validate()`.
+ *     await Model.validate({ name: 'Test', age: 'not a number' }, ['name']);
+ *
  * @param {object} obj
  * @param {object|Array|string} pathsOrOptions
  * @param {object} [context]
@@ -4299,6 +4351,33 @@ Model.validate = async function validate(obj, pathsOrOptions, context) {
   return obj;
 };
 
+/**
+ * Standard Schema adapter for this model.
+ * Calls [`Model.validate()`](https://mongoosejs.com/docs/api/model.html#Model.validate()) internally with the provided `libraryOptions`
+ * to cast and validate the given value.
+ *
+ * @api public
+ * @property ~standard
+ * @memberOf Model
+ * @static
+ */
+
+Object.defineProperty(Model, '~standard', {
+  configurable: true,
+  get() {
+    return {
+      version: 1,
+      vendor: 'mongoose',
+      validate: (value, options) => {
+        return this.validate(value, options?.libraryOptions).then(
+          value => ({ value }),
+          error => ({ issues: convertErrorToStandardSchemaIssues(error) })
+        );
+      }
+    };
+  }
+});
+
 /**
  * Populates document references.
  *
@@ -4465,7 +4544,6 @@ async function _populatePath(model, docs, populateOptions) {
       mod.foreignField.clear();
       mod.foreignField.add(populateOptions.foreignField);
     }
-    const match = createPopulateQueryFilter(ids, mod.match, mod.foreignField, mod.model, mod.options.skipInvalidIds);
     if (assignmentOpts.excludeId) {
       // override the exclusion from the query so we can use the _id
       // for document matching during assignment. we'll delete the
@@ -4486,6 +4564,17 @@ async function _populatePath(model, docs, populateOptions) {
     } else if (mod.options.limit != null) {
       assignmentOpts.originalLimit = mod.options.limit;
     }
+
+    // Execute a separate query per document if `perDocumentLimit` is set (gh-7318), or if
+    // there are so many ids that the `$in` filter may overflow MongoDB's 16 MB BSON size
+    // limit on queries (gh-5890).
+    const splitParams = splitPopulateQuery(mod, ids, select, assignmentOpts);
+    if (splitParams != null) {
+      params.push(...splitParams);
+      continue;
+    }
+
+    const match = createPopulateQueryFilter(ids, mod.match, mod.foreignField, mod.model, mod.options.skipInvalidIds);
     params.push([mod, match, select, assignmentOpts]);
   }
   if (!hasOne) {
@@ -4506,6 +4595,9 @@ async function _populatePath(model, docs, populateOptions) {
 
   // Track deferred populates per-param (per model) to avoid mixing them
   const deferredPopulatesPerParam = new Map();
+  // Query results per param. Batches that were split off of a large populate query (gh-5890)
+  // are assigned from only their own query's results rather than the combined `vals`.
+  const valsByParam = [];
 
   if (populateOptions.ordered) {
     // Populate in series, primarily for transactions because MongoDB doesn't support multiple operations on
@@ -4513,7 +4605,10 @@ async function _populatePath(model, docs, populateOptions) {
     for (let i = 0; i < params.length; i++) {
       const arr = params[i];
       const { docs, deferredPopulates } = await _execPopulateQuery.apply(null, arr);
-      vals = vals.concat(docs);
+      valsByParam.push(docs);
+      if (!arr[0]._assignFromOwnResults) {
+        vals = vals.concat(docs);
+      }
       if (deferredPopulates.length > 0) {
         deferredPopulatesPerParam.set(i, deferredPopulates);
       }
@@ -4528,7 +4623,10 @@ async function _populatePath(model, docs, populateOptions) {
     const results = await Promise.all(promises);
     for (let i = 0; i < results.length; i++) {
       const { docs, deferredPopulates } = results[i];
-      vals = vals.concat(docs);
+      valsByParam.push(docs);
+      if (!params[i][0]._assignFromOwnResults) {
+        vals = vals.concat(docs);
+      }
       if (deferredPopulates.length > 0) {
         deferredPopulatesPerParam.set(i, deferredPopulates);
       }
@@ -4536,13 +4634,15 @@ async function _populatePath(model, docs, populateOptions) {
   }
 
 
-  for (const arr of params) {
+  for (let i = 0; i < params.length; i++) {
+    const arr = params[i];
     const mod = arr[0];
     const assignmentOpts = arr[3];
-    for (const val of vals) {
+    const valsForMod = mod._assignFromOwnResults ? valsByParam[i] : vals;
+    for (const val of valsForMod) {
       mod.options._childDocs.push(val);
     }
-    _assign(model, vals, mod, assignmentOpts);
+    _assign(model, valsForMod, mod, assignmentOpts);
   }
 
   // Handle deferred populate for cases with per-document match functions.
@@ -4582,13 +4682,15 @@ async function _populatePath(model, docs, populateOptions) {
     }
   }
 
-  for (const arr of params) {
-    removeDeselectedForeignField(arr[0].foreignField, arr[0].options, vals);
+  for (let i = 0; i < params.length; i++) {
+    const mod = params[i][0];
+    removeDeselectedForeignField(mod.foreignField, mod.options, mod._assignFromOwnResults ? valsByParam[i] : vals);
   }
-  for (const arr of params) {
-    const mod = arr[0];
+  for (let i = 0; i < params.length; i++) {
+    const mod = params[i][0];
     if (mod.options?.options?._leanTransform) {
-      for (const doc of vals) {
+      const valsForMod = mod._assignFromOwnResults ? valsByParam[i] : vals;
+      for (const doc of valsForMod) {
         mod.options.options._leanTransform(doc);
       }
     }
@@ -4600,6 +4702,12 @@ async function _populatePath(model, docs, populateOptions) {
  */
 
 function _execPopulateQuery(mod, match, select) {
+  // `null` match means `mod`'s documents have no ids to query, possible if a large populate
+  // was split into a query per document (gh-5890). Skip executing a query: `_assign()` still
+  // sets the documents' populated paths to the appropriate default value.
+  if (match == null) {
+    return Promise.resolve({ docs: [], deferredPopulates: [] });
+  }
   let subPopulate = clone(mod.options.populate);
   const queryOptions = {};
   if (mod.options.skip !== undefined) {

package/lib/plugins/saveSubdocs.js

@@ -16,7 +16,10 @@ module.exports = function saveSubdocs(schema) {
 };
 
 
-async function saveSubdocsPreSave() {
+// These hooks are deliberately not `async` so that they don't allocate a
+// promise and force an extra microtask hop on every `save()` when there are
+// no subdocuments. kareem only awaits hooks that return a promise.
+function saveSubdocsPreSave() {
   if (this.$isSubdocument) {
     return;
   }
@@ -28,15 +31,15 @@ async function saveSubdocsPreSave() {
   }
 
   const options = this.$__.saveOptions;
-  await Promise.all(subdocs.map(subdoc => subdoc._execDocumentPreHooks('save', options, [options])));
-
-  // Invalidate subdocs cache because subdoc pre hooks can add new subdocuments
-  if (this.$__.saveOptions) {
-    this.$__.saveOptions.__subdocs = null;
-  }
+  return Promise.all(subdocs.map(subdoc => subdoc._execDocumentPreHooks('save', options, [options]))).then(() => {
+    // Invalidate subdocs cache because subdoc pre hooks can add new subdocuments
+    if (this.$__.saveOptions) {
+      this.$__.saveOptions.__subdocs = null;
+    }
+  });
 }
 
-async function saveSubdocsPostSave() {
+function saveSubdocsPostSave() {
   if (this.$isSubdocument) {
     return;
   }
@@ -53,10 +56,10 @@ async function saveSubdocsPostSave() {
     promises.push(subdoc._execDocumentPostHooks('save', options));
   }
 
-  await Promise.all(promises);
+  return Promise.all(promises);
 }
 
-async function saveSubdocsPreDeleteOne() {
+function saveSubdocsPreDeleteOne() {
   const removedSubdocs = this.$__.removedSubdocs;
   if (!removedSubdocs?.length) {
     return;
@@ -68,10 +71,10 @@ async function saveSubdocsPreDeleteOne() {
     promises.push(subdoc._execDocumentPreHooks('deleteOne', options));
   }
 
-  await Promise.all(promises);
+  return Promise.all(promises);
 }
 
-async function saveSubdocsPostDeleteOne() {
+function saveSubdocsPostDeleteOne() {
   const removedSubdocs = this.$__.removedSubdocs;
   if (!removedSubdocs?.length) {
     return;
@@ -84,7 +87,7 @@ async function saveSubdocsPostDeleteOne() {
   }
 
   this.$__.removedSubdocs = null;
-  await Promise.all(promises);
+  return Promise.all(promises);
 }
 
 

package/lib/query.js

@@ -42,6 +42,8 @@ const updateValidators = require('./helpers/updateValidators');
 const util = require('util');
 const utils = require('./utils');
 const queryMiddlewareFunctions = require('./constants').queryMiddlewareFunctions;
+const { createTracedChannel } = require('./tracing');
+const { trace: traceQuery } = createTracedChannel('mongoose:query');
 
 const queryOptionMethods = new Set([
   'allowDiskUse',
@@ -4778,42 +4780,56 @@ Query.prototype.exec = async function exec(op) {
   }
   this._execCount++;
 
-  let skipWrappedFunction = null;
-  try {
-    await this._hooks.execPre('exec', this, []);
-  } catch (err) {
-    if (err instanceof Kareem.skipWrappedFunction) {
-      skipWrappedFunction = err;
-    } else {
-      throw err;
+  const _this = this;
+  return traceQuery(async function maybeTracedQueryExec() {
+    let skipWrappedFunction = null;
+    try {
+      await _this._hooks.execPre('exec', _this, []);
+    } catch (err) {
+      if (err instanceof Kareem.skipWrappedFunction) {
+        skipWrappedFunction = err;
+      } else {
+        throw err;
+      }
     }
-  }
 
-  let res;
+    let res;
 
-  let error = null;
-  try {
-    await _executePreHooks(this);
-    res = skipWrappedFunction ? skipWrappedFunction.args[0] : await this[thunk]();
+    let error = null;
+    try {
+      await _executePreHooks(_this);
+      res = skipWrappedFunction ? skipWrappedFunction.args[0] : await _this[thunk]();
 
-    for (const fn of this._transforms) {
-      res = fn(res);
-    }
-  } catch (err) {
-    if (err instanceof Kareem.skipWrappedFunction) {
-      res = err.args[0];
-    } else {
-      error = err;
-    }
+      for (const fn of _this._transforms) {
+        res = fn(res);
+      }
+    } catch (err) {
+      if (err instanceof Kareem.skipWrappedFunction) {
+        res = err.args[0];
+      } else {
+        error = err;
+      }
 
-    error = this.model.schema._transformDuplicateKeyError(error);
-  }
+      error = _this.model.schema._transformDuplicateKeyError(error);
+    }
 
-  res = await _executePostHooks(this, res, error);
+    res = await _executePostHooks(_this, res, error);
 
-  await this._hooks.execPost('exec', this, []);
+    await _this._hooks.execPost('exec', _this, []);
 
-  return res;
+    return res;
+  }, () => ({
+    operation: _this.op,
+    collection: _this.mongooseCollection.name,
+    database: _this.model.db?.name,
+    serverAddress: _this.model.db?.host,
+    serverPort: _this.model.db?.port,
+    args: {
+      filter: _this.getFilter(),
+      fields: _this._fields,
+      options: _this._mongooseOptions
+    }
+  }));
 };
 
 /*!

package/lib/schema/date.js

@@ -188,16 +188,18 @@ SchemaDate.prototype.expires = function(when) {
 SchemaDate._checkRequired = v => v instanceof Date;
 
 /**
- * Override the function the required validator uses to check whether a string
+ * Override the function the required validator uses to check whether a date
  * passes the `required` check.
  *
  * #### Example:
  *
- *     // Allow empty strings to pass `required` check
- *     mongoose.Schema.Types.String.checkRequired(v => v != null);
+ *     // Disallow "invalid date"
+ *     mongoose.Schema.Types.Date.checkRequired(v => v instanceof Date && !isNaN(v.valueOf()));
  *
- *     const M = mongoose.model({ str: { type: String, required: true } });
- *     new M({ str: '' }).validateSync(); // `null`, validation passes!
+ *     const schema = new mongoose.Schema({ myDate: { type: Date, required: true } });
+ *     const M = mongoose.model('Test', schema);
+ *     // returns a ValidationError due to date being invalid
+ *     new M({ myDate: new Date('invalid') }).validateSync();
  *
  * @param {Function} fn
  * @return {Function}

package/lib/schema/documentArray.js

@@ -314,7 +314,7 @@ SchemaDocumentArray.prototype.doValidateSync = function(array, scope, options) {
       continue;
     }
 
-    const subdocValidateError = doc.validateSync(options);
+    const subdocValidateError = doc.$__validateSync(options);
 
     if (subdocValidateError && resultError == null) {
       resultError = subdocValidateError;

package/lib/schema/objectId.js

@@ -317,7 +317,13 @@ function resetId(v) {
  */
 
 SchemaObjectId.prototype.toJSONSchema = function toJSONSchema(options) {
-  return this._createJSONSchemaTypeDefinition('string', 'objectId', options);
+  const jsonSchema = this._createJSONSchemaTypeDefinition('string', 'objectId', options);
+  // `bsonType: 'objectId'` already validates ObjectIds, so the regex is only useful
+  // for the portable `type: 'string'` representation (e.g. for ajv, zod, etc.)
+  if (!options?.useBsonType) {
+    jsonSchema.pattern = '^[A-Fa-f0-9]{24}$';
+  }
+  return jsonSchema;
 };
 
 SchemaObjectId.prototype.autoEncryptionType = function autoEncryptionType() {

package/lib/schema/subdocument.js

@@ -310,7 +310,7 @@ SchemaSubdocument.prototype.doValidateSync = function(value, scope, options) {
   if (!value) {
     return;
   }
-  return value.validateSync();
+  return value.$__validateSync();
 };
 
 /**

package/lib/schema/union.js

@@ -112,8 +112,8 @@ class Union extends SchemaType {
         return schemaTypeError;
       }
     }
-    if (value != null && typeof value.validateSync === 'function') {
-      return value.validateSync();
+    if (value != null && typeof value.$__validateSync === 'function') {
+      return value.$__validateSync();
     }
   }
 

package/lib/schemaType.js

@@ -1198,7 +1198,8 @@ SchemaType.prototype.required = function(required, message) {
  *       name: { type: String, allowNull: false }
  *     });
  *
- *     new Model({ name: undefined }).validateSync(); // OK
+ *     new Model({}).validateSync(); // OK
+ *     new Model({ name: undefined }).validateSync(); // OK, Mongoose strips out `name` when its value is `undefined`
  *     new Model({ name: null }).validateSync(); // ValidationError
  *
  * @param {boolean} allowNull

package/lib/standardSchema/convertErrorToIssues.js

@@ -0,0 +1,20 @@
+'use strict';
+
+const ValidationError = require('../error/validation');
+
+module.exports = function convertErrorToIssues(error) {
+  if (error instanceof ValidationError) {
+    return Object.keys(error.errors).map(path => {
+      const err = error.errors[path];
+      return {
+        message: err.message,
+        path: path.split('.').map(part => {
+          const num = +part;
+          return Number.isInteger(num) && String(num) === part ? num : part;
+        })
+      };
+    });
+  }
+
+  return [{ message: error.message }];
+};

package/lib/stateMachine.js

@@ -69,8 +69,10 @@ StateMachine.prototype._changeState = function _changeState(path, nextState) {
   if (prevState === nextState) {
     return;
   }
-  const prevBucket = this.states[prevState];
-  if (prevBucket) delete prevBucket[path];
+  if (prevState !== undefined) {
+    const prevBucket = this.states[prevState];
+    if (prevBucket) delete prevBucket[path];
+  }
 
   this.paths[path] = nextState;
   this.states[nextState] = this.states[nextState] || {};
@@ -86,13 +88,16 @@ StateMachine.prototype.clear = function clear(state) {
     return;
   }
   const keys = Object.keys(this.states[state]);
+  if (keys.length === 0) {
+    return;
+  }
+  // Replace the bucket rather than deleting each key: repeated `delete` puts
+  // the object in dictionary mode, which is significantly slower.
+  this.states[state] = {};
   let i = keys.length;
-  let path;
 
   while (i--) {
-    path = keys[i];
-    delete this.states[state][path];
-    delete this.paths[path];
+    delete this.paths[keys[i]];
   }
 };
 

package/lib/tracing.js

@@ -0,0 +1,38 @@
+'use strict';
+
+let dc;
+try {
+  dc = (typeof process !== 'undefined' && 'getBuiltinModule' in process)
+    ? process.getBuiltinModule('node:diagnostics_channel')
+    : require('node:diagnostics_channel');
+} catch {
+  // diagnostics_channel not available
+}
+
+const hasTracingChannel = !!dc && typeof dc.tracingChannel === 'function';
+
+function shouldTrace(channel) {
+  // Node 18 and Bun don't expose hasSubscribers on TracingChannel
+  // so undefined means it may or may not have subscribers, so we'll trace anyway
+  return !!channel && channel.hasSubscribers !== false;
+}
+
+function createTracedChannel(name) {
+  const ch = hasTracingChannel ? dc.tracingChannel(name) : undefined;
+
+  function trace(fn, contextFactory) {
+    if (!shouldTrace(ch)) {
+      return fn();
+    }
+
+    const traced = ch.tracePromise(fn, contextFactory());
+    return traced;
+  }
+
+  return { channel: ch, trace };
+}
+
+module.exports = {
+  createTracedChannel,
+  cursorNextChannel: createTracedChannel('mongoose:cursor:next')
+};