@depup/mongoose 9.3.3-depup.0 → 9.8.0-depup.0

package/lib/drivers/node-mongodb-native/collection.js

@@ -89,12 +89,11 @@ NativeCollection.prototype._getCollection = function _getCollection() {
 function iter(i) {
   NativeCollection.prototype[i] = function() {
     const collection = this._getCollection();
-    const args = Array.from(arguments);
+    const args = arguments;
     const _this = this;
     const globalDebug = _this?.conn?.base?.options?.debug;
     const connectionDebug = _this?.conn?.options?.debug;
     const debug = connectionDebug == null ? globalDebug : connectionDebug;
-    const opId = new ObjectId();
 
     // If user force closed, queueing will hang forever. See #5664
     if (this.conn.$wasForceClosed) {
@@ -108,9 +107,21 @@ function iter(i) {
       }
     }
 
+    // Lazily generate opId and argsArray only if necessary because they have
+    // a non-trivial performance cost.
+    let opId = null;
+    let argsArray = null;
+    const hasOperationListeners = this.conn.listenerCount('operation-start') > 0 || this.conn.listenerCount('operation-end') > 0;
+    if (hasOperationListeners || debug) {
+      opId = new ObjectId();
+    }
+
     let timeout = null;
     let waitForBufferPromise = null;
     if (this._shouldBufferCommands() && this.buffer) {
+      if (opId == null) {
+        opId = new ObjectId();
+      }
       this.conn.emit('buffer', {
         _id: opId,
         modelName: _this.modelName,
@@ -145,11 +156,14 @@ function iter(i) {
 
     if (debug) {
       if (typeof debug === 'function') {
+        if (argsArray == null) {
+          argsArray = Array.from(args);
+        }
         let argsToAdd = null;
-        if (typeof args[args.length - 1] == 'function') {
-          argsToAdd = args.slice(0, args.length - 1);
+        if (typeof argsArray[argsArray.length - 1] == 'function') {
+          argsToAdd = argsArray.slice(0, argsArray.length - 1);
         } else {
-          argsToAdd = args;
+          argsToAdd = argsArray;
         }
         debug.apply(_this,
           [_this.name, i].concat(argsToAdd));
@@ -158,11 +172,17 @@ function iter(i) {
       } else {
         const color = debug.color == null ? true : debug.color;
         const shell = debug.shell == null ? false : debug.shell;
-        this.$print(_this.name, i, args, color, shell);
+        const timestamp = debug.timestamp == null ? false : debug.timestamp;
+        this.$print(_this.name, i, args, color, shell, timestamp);
       }
     }
 
-    this.conn.emit('operation-start', { _id: opId, modelName: _this.modelName, collectionName: this.name, method: i, params: args });
+    if (hasOperationListeners) {
+      if (argsArray == null) {
+        argsArray = Array.from(args);
+      }
+      this.conn.emit('operation-start', { _id: opId, modelName: _this.modelName, collectionName: this.name, method: i, params: argsArray });
+    }
 
     try {
       if (collection == null) {
@@ -179,20 +199,26 @@ function iter(i) {
             if (timeout != null) {
               clearTimeout(timeout);
             }
-            this.conn.emit('operation-end', { _id: opId, modelName: _this.modelName, collectionName: this.name, method: i, result });
+            if (hasOperationListeners) {
+              this.conn.emit('operation-end', { _id: opId, modelName: _this.modelName, collectionName: this.name, method: i, result });
+            }
             return result;
           },
           error => {
             if (timeout != null) {
               clearTimeout(timeout);
             }
-            this.conn.emit('operation-end', { _id: opId, modelName: _this.modelName, collectionName: this.name, method: i, error });
+            if (hasOperationListeners) {
+              this.conn.emit('operation-end', { _id: opId, modelName: _this.modelName, collectionName: this.name, method: i, error });
+            }
             throw error;
           }
         );
       }
 
-      this.conn.emit('operation-end', { _id: opId, modelName: _this.modelName, collectionName: this.name, method: i, result: ret });
+      if (hasOperationListeners) {
+        this.conn.emit('operation-end', { _id: opId, modelName: _this.modelName, collectionName: this.name, method: i, result: ret });
+      }
       if (timeout != null) {
         clearTimeout(timeout);
       }
@@ -201,7 +227,9 @@ function iter(i) {
       if (timeout != null) {
         clearTimeout(timeout);
       }
-      this.conn.emit('operation-end', { _id: opId, modelName: _this.modelName, collectionName: this.name, method: i, error: error });
+      if (hasOperationListeners) {
+        this.conn.emit('operation-end', { _id: opId, modelName: _this.modelName, collectionName: this.name, method: i, error: error });
+      }
       throw error;
     }
   };
@@ -229,7 +257,12 @@ for (const key of Object.getOwnPropertyNames(Collection.prototype)) {
  * @method $print
  */
 
-NativeCollection.prototype.$print = function(name, i, args, color, shell) {
+NativeCollection.prototype.$print = function(name, i, args, color, shell, timestamp) {
+  let prefix = '';
+  if (timestamp) {
+    const ts = new Date().toISOString();
+    prefix = color ? `\x1B[0;90m[${ts}]\x1B[0m ` : `[${ts}] `;
+  }
   const moduleName = color ? '\x1B[0;36mMongoose:\x1B[0m ' : 'Mongoose: ';
   const functionCall = [name, i].join('.');
   const _args = [];
@@ -240,14 +273,14 @@ NativeCollection.prototype.$print = function(name, i, args, color, shell) {
   }
   const params = '(' + _args.join(', ') + ')';
 
-  console.info(moduleName + functionCall + params);
+  console.info(prefix + moduleName + functionCall + params);
 };
 
 /**
  * Debug print helper
  *
  * @api public
- * @method $print
+ * @method $printToStream
  */
 
 NativeCollection.prototype.$printToStream = function(name, i, args, stream) {

package/lib/drivers/node-mongodb-native/connection.js

@@ -14,6 +14,10 @@ const setTimeout = require('../../helpers/timers').setTimeout;
 const utils = require('../../utils');
 const Schema = require('../../schema');
 
+// Snapshot the native Date constructor to ensure Date.now()
+// bypasses timer mocks such as those set up by useFakeTimers().
+const Date = globalThis.Date;
+
 /**
  * A [node-mongodb-native](https://github.com/mongodb/node-mongodb-native) connection implementation.
  *
@@ -482,6 +486,15 @@ function _setClient(conn, client, options, dbName) {
     for (const otherDb of conn.otherDbs) {
       otherDb._lastHeartbeatAt = conn._lastHeartbeatAt;
     }
+    // Flush buffered operations if the connection is no longer stale (gh-16183)
+    if (conn._queue.length > 0 && conn.readyState === STATES.connected) {
+      conn._flushQueue();
+    }
+    for (const otherDb of conn.otherDbs) {
+      if (otherDb._queue.length > 0 && otherDb.readyState === STATES.connected) {
+        otherDb._flushQueue();
+      }
+    }
   });
 
   if (options.monitorCommands) {

package/lib/error/cast.js

@@ -42,6 +42,7 @@ class CastError extends MongooseError {
       message: this.message
     };
   }
+
   /*!
    * ignore
    */
@@ -76,7 +77,7 @@ class CastError extends MongooseError {
    */
   setModel(model) {
     this.message = formatMessage(model, this.kind, this.value, this.path,
-      this.messageFormat, this.valueType);
+      this.messageFormat, this.valueType, this.reason);
   }
 }
 
@@ -122,10 +123,11 @@ function getMessageFormat(schemaType) {
 function formatMessage(model, kind, value, path, messageFormat, valueType, reason) {
   if (typeof messageFormat === 'string') {
     const stringValue = getStringValue(value);
-    let ret = messageFormat.
-      replace('{KIND}', kind).
-      replace('{VALUE}', stringValue).
-      replace('{PATH}', path);
+    let ret = messageFormat
+      .replace('{KIND}', kind)
+      .replace('{VALUE}', stringValue)
+      .replace('{PATH}', path);
+
     if (model != null) {
       ret = ret.replace('{MODEL}', model.modelName);
     }
@@ -136,17 +138,24 @@ function formatMessage(model, kind, value, path, messageFormat, valueType, reaso
   } else {
     const stringValue = getStringValue(value);
     const valueTypeMsg = valueType ? ' (type ' + valueType + ')' : '';
+
     let ret = 'Cast to ' + kind + ' failed for value ' +
       stringValue + valueTypeMsg + ' at path "' + path + '"';
+
     if (model != null) {
       ret += ' for model "' + model.modelName + '"';
     }
-    if (reason != null &&
-        typeof reason.constructor === 'function' &&
-        reason.constructor.name !== 'AssertionError' &&
-        reason.constructor.name !== 'Error') {
+
+    if (
+      reason != null &&
+      typeof reason.constructor === 'function' &&
+      reason.constructor.name !== 'AssertionError' &&
+      reason.constructor.name !== 'Error' &&
+      reason.constructor.name !== 'BSONError'
+    ) {
       ret += ' because of "' + reason.constructor.name + '"';
     }
+
     return ret;
   }
 }

package/lib/error/divergentArray.js

@@ -17,7 +17,7 @@ class DivergentArrayError extends MongooseError {
 
   constructor(paths) {
     const msg = 'For your own good, using `document.save()` to update an array '
-            + 'which was selected using an $elemMatch projection OR '
+            + 'which was selected using an $elemMatch or $slice projection OR '
             + 'populated using skip, limit, query conditions, or exclusion of '
             + 'the _id field when the operation results in a $pop or $set of '
             + 'the entire array is not supported. The following '

package/lib/error/index.js

@@ -32,7 +32,7 @@ const MongooseError = require('./mongooseError');
  * - `ValidationError`: error returned from [`validate()`](https://mongoosejs.com/docs/api/document.html#Document.prototype.validate()) or [`validateSync()`](https://mongoosejs.com/docs/api/document.html#Document.prototype.validateSync()). Contains zero or more `ValidatorError` instances in `.errors` property.
  * - `MissingSchemaError`: You called `mongoose.Document()` without a schema
  * - `ObjectExpectedError`: Thrown when you set a nested path to a non-object value with [strict mode set](https://mongoosejs.com/docs/guide.html#strict).
- * - `ObjectParameterError`: Thrown when you pass a non-object value to a function which expects an object as a paramter
+ * - `ObjectParameterError`: Thrown when you pass a non-object value to a function which expects an object as a parameter
  * - `OverwriteModelError`: Thrown when you call [`mongoose.model()`](https://mongoosejs.com/docs/api/mongoose.html#Mongoose.model()) to re-define a model that was already defined.
  * - `ParallelSaveError`: Thrown when you call [`save()`](https://mongoosejs.com/docs/api/model.html#Model.prototype.save()) on a document when the same document instance is already saving.
  * - `StrictModeError`: Thrown when you set a path that isn't the schema and [strict mode](https://mongoosejs.com/docs/guide.html#strict) is set to `throw`.

package/lib/error/messages.js

@@ -30,6 +30,7 @@ msg.DocumentNotFoundError = null;
 msg.general = {};
 msg.general.default = 'Validator failed for path `{PATH}` with value `{VALUE}`';
 msg.general.required = 'Path `{PATH}` is required.';
+msg.general.allowNull = 'Path `{PATH}` does not allow null values.';
 
 msg.Number = {};
 msg.Number.min = 'Path `{PATH}` ({VALUE}) is less than minimum allowed value ({MIN}).';

package/lib/helpers/clone.js

@@ -202,10 +202,15 @@ function cloneArray(arr, options) {
 
   let ret = null;
   if (options?.retainDocuments) {
+    // Use the cloned parent doc (options.parentDoc) when available so the new
+    // array is wired up to the clone, not the source. Falling back to the
+    // source's parent preserves the previous behavior for callers that don't
+    // pass parentDoc.
+    const newParent = options.parentDoc ?? (arr.isMongooseArray ? arr.$parent() : undefined);
     if (arr.isMongooseDocumentArray) {
-      ret = new (arr.$schemaType().schema.base.Types.DocumentArray)([], arr.$path(), arr.$parent(), arr.$schemaType());
+      ret = new (arr.$schemaType().schema.base.Types.DocumentArray)([], arr.$path(), newParent, arr.$schemaType());
     } else if (arr.isMongooseArray) {
-      ret = new (arr.$parent().schema.base.Types.Array)([], arr.$path(), arr.$parent(), arr.$schemaType());
+      ret = new (arr.$parent().schema.base.Types.Array)([], arr.$path(), newParent, arr.$schemaType());
     } else {
       ret = new Array(len);
     }
@@ -218,9 +223,21 @@ function cloneArray(arr, options) {
     // Create new options object to avoid mutating the shared options.
     // Subdocs need parentArray to point to their own cloned array.
     options = { ...options, parentArray: ret };
-  }
-  for (i = 0; i < len; ++i) {
-    ret[i] = clone(arr[i], options, true);
+    // Skip change-tracking while populating the freshly cloned array: every
+    // index assignment would otherwise call markModified on the cloned parent
+    // doc and (before this guard) on the source parent doc.
+    for (i = 0; i < len; ++i) {
+      const doc = clone(arr[i], options, true);
+      // Document arrays can contain null entries, so only restamp actual subdocs.
+      if (typeof doc?.$setIndex === 'function') {
+        doc.$setIndex(i);
+      }
+      ret.set(i, doc, true);
+    }
+  } else {
+    for (i = 0; i < len; ++i) {
+      ret[i] = clone(arr[i], options, true);
+    }
   }
 
   return ret;

package/lib/helpers/document/applyDefaults.js

@@ -17,6 +17,11 @@ module.exports = function applyDefaults(doc, fields, exclude, hasIncludedChildre
     }
 
     const type = doc.$__schema.paths[p];
+    // `getDefault()` returns undefined when `defaultValue` is undefined, in which
+    // case this loop never modifies the doc, so skip traversing the path entirely.
+    if (type.defaultValue === undefined) {
+      continue;
+    }
     const path = type.splitPath();
     const len = path.length;
     if (path[len - 1] === '$*') {

package/lib/helpers/populate/createPopulateQueryFilter.js

@@ -25,7 +25,15 @@ module.exports = function createPopulateQueryFilter(ids, _match, _foreignField,
     for (let i = 0; i < _parentPaths.length - 1; ++i) {
       const cur = _parentPaths[i];
       if (match[cur] != null && match[cur].$elemMatch != null) {
-        match[cur].$elemMatch[foreignField.slice(cur.length + 1)] = trusted({ $in: ids });
+        // Copy rather than mutate so the user's `match` stays unchanged and split populate
+        // queries (gh-5890) each get their own `$in` rather than sharing one object
+        match[cur] = {
+          ...match[cur],
+          $elemMatch: {
+            ...match[cur].$elemMatch,
+            [foreignField.slice(cur.length + 1)]: trusted({ $in: ids })
+          }
+        };
         delete match[foreignField];
         break;
       }

package/lib/helpers/populate/getModelsMapForPopulate.js

@@ -649,11 +649,7 @@ function addModelNamesToMap(model, map, available, modelNames, options, data, re
       ids = matchIdsToRefPaths(ret, modelNamesForRefPath, modelName);
     }
 
-    const perDocumentLimit = options.perDocumentLimit == null ?
-      get(options, 'options.perDocumentLimit', null) :
-      options.perDocumentLimit;
-
-    if (!available[modelName] || perDocumentLimit != null) {
+    if (!available[modelName]) {
       const currentOptions = {
         model: Model
       };
@@ -682,6 +678,7 @@ function addModelNamesToMap(model, map, available, modelNames, options, data, re
         isVirtual: data.isVirtual,
         virtual: data.virtual,
         count: data.count,
+        isRefPath: !!data.isRefPath,
         [populateModelSymbol]: Model
       };
       map.push(available[modelName]);
@@ -692,6 +689,7 @@ function addModelNamesToMap(model, map, available, modelNames, options, data, re
       available[modelName].ids.push(ids);
       available[modelName].allIds.push(ret);
       available[modelName].unpopulatedValues.push(unpopulatedValue);
+      available[modelName].isRefPath = available[modelName].isRefPath || !!data.isRefPath;
       if (data.hasMatchFunction) {
         available[modelName].match.push(data.match);
       }
@@ -873,7 +871,11 @@ function _findRefPathForDiscriminators(doc, modelSchema, data, options, normaliz
 }
 
 /**
- * Throw an error if there are any $where keys
+ * Throw an error if there are any $where keys to defend against [CVE-2024-53900](https://nvd.nist.gov/vuln/detail/CVE-2024-53900)
+ *
+ * Note that this is ONLY for $where because sift executes $where in Node.js memory.
+ * Other forms of MongoDB server-side execution, like $expr, are NOT filtered out.
+ * This function is not meant to protect against server-side execution in MongoDB.
  */
 
 function throwOn$where(match) {

package/lib/helpers/populate/splitPopulateQuery.js

@@ -0,0 +1,81 @@
+'use strict';
+
+const createPopulateQueryFilter = require('./createPopulateQueryFilter');
+const get = require('../get');
+const utils = require('../../utils');
+
+module.exports = splitPopulateQuery;
+
+/*!
+ * If a single populate query would have more than this many elements in its `$in` filter,
+ * Mongoose splits the populate into a separate query per document to avoid going over
+ * MongoDB's 16 MB BSON size limit on queries. Overwritable for testing purposes. See gh-5890.
+ */
+
+splitPopulateQuery.maxInFilterLength = 50000;
+
+/*!
+ * Split a populate models-map entry into a separate query per document if either:
+ *
+ * 1. The `perDocumentLimit` option is set, so each document needs its own query with its
+ *    own `limit` (gh-7318), or
+ * 2. A single populate query for `mod` would have too many elements in its `$in` filter
+ *    (gh-5890). With multiple foreign fields, `createPopulateQueryFilter()` repeats the ids
+ *    under `$or` once per foreign field, so the threshold counts one copy of `ids` per
+ *    foreign field.
+ *
+ * Returns a list of `[mod, match, select, assignmentOpts]` params, one per document, for
+ * `_execPopulateQuery()`. Returns `null` if the populate query doesn't need to be split.
+ * A `null` `match` means the document has no ids to query: `_execPopulateQuery()` skips
+ * executing a query, and `_assign()` just sets the document's populated path to the
+ * default value.
+ *
+ * Splitting on document boundaries means each document's populated value is the result of
+ * exactly one query, so split entries can typically be assigned from only their own query's
+ * results (`_assignFromOwnResults`) rather than scanning every populate query's results.
+ * refPath is the exception: a single document's array can contain ids for multiple models,
+ * so assigning refPath populate results relies on every query's results being available for
+ * every document. refPath entries are therefore only split when `perDocumentLimit` requires
+ * it, not to keep the `$in` filter small. A single document whose ids alone overflow the
+ * BSON size limit cannot be split.
+ */
+
+function splitPopulateQuery(mod, ids, select, assignmentOpts) {
+  if (mod.docs.length <= 1) {
+    return null;
+  }
+  const perDocumentLimit = mod.options.perDocumentLimit == null ?
+    get(mod.options, 'options.perDocumentLimit', null) :
+    mod.options.perDocumentLimit;
+  const numInFilterElements = ids.length * mod.foreignField.size;
+  if (perDocumentLimit == null && (numInFilterElements <= splitPopulateQuery.maxInFilterLength || mod.isRefPath)) {
+    return null;
+  }
+
+  return mod.docs.map((doc, i) => {
+    const subMod = {
+      ...mod,
+      docs: [doc],
+      ids: [mod.ids[i]],
+      allIds: [mod.allIds[i]],
+      unpopulatedValues: [mod.unpopulatedValues[i]],
+      match: Array.isArray(mod.match) ? [mod.match[i]] : mod.match,
+      _assignFromOwnResults: !mod.isRefPath
+    };
+    let subIds = utils.array.flatten(subMod.ids, flatten);
+    subIds = utils.array.unique(subIds);
+    const match = subIds.length === 0 || subIds.every(utils.isNullOrUndefined) ?
+      null :
+      createPopulateQueryFilter(subIds, subMod.match, subMod.foreignField, subMod.model, subMod.options.skipInvalidIds);
+    return [subMod, match, select, assignmentOpts];
+  });
+}
+
+/*!
+ * ignore
+ */
+
+function flatten(item) {
+  // no need to include undefined values in our query
+  return undefined !== item;
+}

package/lib/helpers/query/castUpdate.js

@@ -98,6 +98,10 @@ module.exports = function castUpdate(schema, obj, options, context, filter) {
     moveImmutableProperties(schema, obj, context);
   }
 
+  if (obj?.$__ && typeof obj.toObject === 'function') {
+    obj = obj.toObject(internalToObjectOptions);
+  }
+
   const ops = Object.keys(obj);
   let i = ops.length;
   const ret = {};

package/lib/helpers/schema/getKeysInSchemaOrder.js

@@ -3,25 +3,22 @@
 const get = require('../get');
 
 module.exports = function getKeysInSchemaOrder(schema, val, path) {
+  const valKeys = Object.keys(val);
+  if (valKeys.length <= 1) {
+    return valKeys;
+  }
+
   const schemaKeys = path != null ? Object.keys(get(schema.tree, path, {})) : Object.keys(schema.tree);
-  const valKeys = new Set(Object.keys(val));
+  const remaining = new Set(valKeys);
 
-  let keys;
-  if (valKeys.size > 1) {
-    keys = new Set();
-    for (const key of schemaKeys) {
-      if (valKeys.has(key)) {
-        keys.add(key);
-      }
+  const keys = [];
+  for (const key of schemaKeys) {
+    if (remaining.delete(key)) {
+      keys.push(key);
     }
-    for (const key of valKeys) {
-      if (!keys.has(key)) {
-        keys.add(key);
-      }
-    }
-    keys = Array.from(keys);
-  } else {
-    keys = Array.from(valKeys);
+  }
+  for (const key of remaining) {
+    keys.push(key);
   }
 
   return keys;

package/lib/helpers/update/applyTimestampsToUpdate.js

@@ -82,7 +82,9 @@ function applyTimestampsToUpdate(now, createdAt, updatedAt, currentUpdate, optio
 
   if (!skipCreatedAt && createdAt) {
     const overwriteImmutable = get(options, 'overwriteImmutable', false);
-    const hasUserCreatedAt = currentUpdate[createdAt] != null || currentUpdate.$set?.[createdAt] != null;
+    const hasUserCreatedAt = currentUpdate[createdAt] != null
+      || currentUpdate.$set?.[createdAt] != null
+      || currentUpdate.$setOnInsert?.[createdAt] != null;
 
     // If overwriteImmutable is true and user provided createdAt, keep their value
     if (overwriteImmutable && hasUserCreatedAt) {
@@ -91,7 +93,7 @@ function applyTimestampsToUpdate(now, createdAt, updatedAt, currentUpdate, optio
         updates.$set[createdAt] = currentUpdate[createdAt];
         delete currentUpdate[createdAt];
       }
-      // User's value is already in $set, nothing more to do
+      // User's value is already in $set or $setOnInsert, nothing more to do
     } else {
       if (currentUpdate[createdAt]) {
         delete currentUpdate[createdAt];