mongoose 5.0.13 → 5.0.17

package/History.md

@@ -1,3 +1,46 @@
+5.0.17 / 2018-04-30
+===================
+ * docs(migration): certain chars in passwords may cause connection failures #6401 [markstos](https://github.com/markstos)
+ * fix(document): don't throw when `push()` on a nested doc array #6398
+ * fix(model): apply hooks to custom methods if specified #6385
+ * fix(schema): support opting out of one timestamp field but not the other for `insertMany()` #6381
+ * fix(documentarray): handle `required: true` within documentarray definition #6364
+ * fix(document): ensure `isNew` is set before default functions run on init #3793
+
+5.0.16 / 2018-04-23
+===================
+ * docs(api): sort api methods based on their string property #6374 [lineus](https://github.com/lineus)
+ * docs(connection): fix typo in `createCollection()` #6370 [mattc41190](https://github.com/mattc41190)
+ * docs(document): remove vestigial reference to `numAffected` #6367 [ekulabuhov](https://github.com/ekulabuhov)
+ * docs(schema): fix typo #6366 [dhritzkiv](https://github.com/dhritzkiv)
+ * docs(schematypes): add missing `minlength` and `maxlength` docs #6365 [treble-snake](https://github.com/treble-snake)
+ * docs(queries): fix formatting #6360 [treble-snake](https://github.com/treble-snake)
+ * docs(api): add cursors to API docs #6353 #6344 [lineus](https://github.com/lineus)
+ * docs(aggregate): remove reference to non-existent `.select()` method #6346
+ * fix(update): handle `required` array with update validators and $pull #6341
+ * fix(update): avoid setting __v in findOneAndUpdate if it is `$set` #5973
+
+5.0.15 / 2018-04-16
+===================
+ * fix: add ability for casting from number to decimal128 #6336 #6331 [lineus](https://github.com/lineus)
+ * docs(middleware): enumerate the ways to error out in a hook #6315
+ * fix(document): respect schema-level depopulate option for toObject() #6313
+ * fix: bump mongodb driver -> 3.0.6 #6310
+ * fix(number): check for `valueOf()` function to support Decimal.js #6306 #6299 [lineus](https://github.com/lineus)
+ * fix(query): run array setters on query if input value is an array #6277
+ * fix(versioning): don't require matching version when using array.pull() #6190
+ * fix(document): add `toHexString()` function so you don't need to check whether a path is populated to get an id #6115
+
+5.0.14 / 2018-04-09
+===================
+ * fix(schema): clone aliases and alternative option syntax correctly
+ * fix(query): call utils.toObject in query.count like in query.find #6325 [lineus](https://github.com/lineus)
+ * docs(populate): add middleware examples #6320 [BorntraegerMarc](https://github.com/BorntraegerMarc)
+ * docs(compatibility): fix dead link #6319 [lacivert](https://github.com/lacivert)
+ * docs(api): fix markdown parsing for parameters #6318 #6314 [lineus](https://github.com/lineus)
+ * fix(populate): handle space-delimited paths in array populate #6296 #6284 [lineus](https://github.com/lineus)
+ * fix(populate): support basic virtual populate underneath embedded discriminators #6273
+
 5.0.13 / 2018-04-05
 ===================
  * docs(faq): add middleware to faq arrow function warning #6309 [lineus](https://github.com/lineus)

package/lib/connection.js

@@ -243,7 +243,7 @@ Connection.prototype.config;
  * Options are passed down without modification to the [MongoDB driver's `createCollection()` function](http://mongodb.github.io/node-mongodb-native/2.2/api/Db.html#createCollection)
  *
  * @method createCollection
- * @param {string} collection The collection to delete
+ * @param {string} collection The collection to create
  * @param {Object} [options] see [MongoDB driver docs](http://mongodb.github.io/node-mongodb-native/2.2/api/Db.html#createCollection)
  * @param {Function} [callback]
  * @return {Promise}

package/lib/document.js

@@ -4,34 +4,37 @@
  * Module dependencies.
  */
 
-var EventEmitter = require('events').EventEmitter;
-var MongooseError = require('./error');
-var MixedSchema = require('./schema/mixed');
-var Schema = require('./schema');
-var ObjectExpectedError = require('./error/objectExpected');
-var ObjectParameterError = require('./error/objectParameter');
-var StrictModeError = require('./error/strict');
-var ValidatorError = require('./schematype').ValidatorError;
-var VirtualType = require('./virtualtype');
-var utils = require('./utils');
-var clone = utils.clone;
-var isDefiningProjection = require('./services/projection/isDefiningProjection');
-var isExclusive = require('./services/projection/isExclusive');
-var isMongooseObject = utils.isMongooseObject;
-var inspect = require('util').inspect;
-var internalToObjectOptions = require('./options').internalToObjectOptions;
-var ValidationError = MongooseError.ValidationError;
-var InternalCache = require('./internal');
-var cleanModifiedSubpaths = require('./services/document/cleanModifiedSubpaths');
-var compile = require('./services/document/compile').compile;
-var deepEqual = utils.deepEqual;
-var defineKey = require('./services/document/compile').defineKey;
-var DocumentArray;
-var MongooseArray;
-var Embedded;
-var flatten = require('./services/common').flatten;
-var mpath = require('mpath');
-var idGetter = require('./plugins/idGetter');
+const EventEmitter = require('events').EventEmitter;
+const InternalCache = require('./internal');
+const MongooseError = require('./error');
+const MixedSchema = require('./schema/mixed');
+const Schema = require('./schema');
+const ObjectExpectedError = require('./error/objectExpected');
+const ObjectParameterError = require('./error/objectParameter');
+const StrictModeError = require('./error/strict');
+const ValidatorError = require('./schematype').ValidatorError;
+const VirtualType = require('./virtualtype');
+const cleanModifiedSubpaths = require('./services/document/cleanModifiedSubpaths');
+const compile = require('./services/document/compile').compile;
+const defineKey = require('./services/document/compile').defineKey;
+const flatten = require('./services/common').flatten;
+const get = require('lodash.get');
+const idGetter = require('./plugins/idGetter');
+const isDefiningProjection = require('./services/projection/isDefiningProjection');
+const isExclusive = require('./services/projection/isExclusive');
+const inspect = require('util').inspect;
+const internalToObjectOptions = require('./options').internalToObjectOptions;
+const mpath = require('mpath');
+const utils = require('./utils');
+
+const ValidationError = MongooseError.ValidationError;
+const clone = utils.clone;
+const deepEqual = utils.deepEqual;
+const isMongooseObject = utils.isMongooseObject;
+
+let DocumentArray;
+let MongooseArray;
+let Embedded;
 
 /**
  * Document constructor.
@@ -46,9 +49,15 @@ var idGetter = require('./plugins/idGetter');
  */
 
 function Document(obj, fields, skipId, options) {
+  if (typeof skipId === 'object' && skipId != null) {
+    options = skipId;
+    skipId = options.skipId;
+  }
+  options = options || {};
+
   this.$__ = new InternalCache;
   this.$__.emitter = new EventEmitter();
-  this.isNew = true;
+  this.isNew = 'isNew' in options ? options.isNew : true;
   this.errors = undefined;
   this.$__.$options = options || {};
 
@@ -1513,7 +1522,12 @@ function _getPathsToValidate(doc) {
     var path = paths[i];
 
     var _pathType = doc.schema.path(path);
-    if (!_pathType || !_pathType.$isMongooseArray || _pathType.$isMongooseDocumentArray) {
+    if (!_pathType ||
+        !_pathType.$isMongooseArray ||
+        // To avoid potential performance issues, skip doc arrays whose children
+        // are not required. `getPositionalPathType()` may be slow, so avoid
+        // it unless we have a case of #6364
+        (_pathType.$isMongooseDocumentArray && !get(_pathType, 'schemaOptions.required'))) {
       continue;
     }
 
@@ -1802,7 +1816,7 @@ Document.prototype.$markValid = function(path) {
  * ####Example:
  *
  *     product.sold = Date.now();
- *     product.save(function (err, product, numAffected) {
+ *     product.save(function (err, product) {
  *       if (err) ..
  *     })
  *
@@ -1810,7 +1824,6 @@ Document.prototype.$markValid = function(path) {
  *
  * 1. `err` if an error occurred
  * 2. `product` which is the saved `product`
- * 3. `numAffected` will be 1 when the document was successfully persisted to MongoDB, otherwise 0. Unless you tweak mongoose's internals, you don't need to worry about checking this parameter for errors - checking `err` is sufficient to make sure your document was properly saved.
  *
  * As an extra measure of flow control, save will return a Promise.
  * ####Example:
@@ -2138,15 +2151,21 @@ Document.prototype.$toObject = function(options, json) {
     _minimize = this.schema.options.minimize;
   }
 
+  // The original options that will be passed to `clone()`. Important because
+  // `clone()` will recursively call `$toObject()` on embedded docs, so we
+  // need the original options the user passed in, plus `_isNested` and
+  // `_parentOptions` for checking whether we need to depopulate.
   const cloneOptions = Object.assign(utils.clone(options), {
     _isNested: true,
     json: json,
     minimize: _minimize
   });
 
+  const depopulate = options.depopulate ||
+    get(options, '_parentOptions.depopulate', false);
   // _isNested will only be true if this is not the top level document, we
   // should never depopulate
-  if (options.depopulate && options._isNested && this.$__.wasPopulated) {
+  if (depopulate && options._isNested && this.$__.wasPopulated) {
     // populated paths that we set to a document
     return clone(this._id, cloneOptions);
   }
@@ -2157,6 +2176,8 @@ Document.prototype.$toObject = function(options, json) {
   options.json = json;
   options.minimize = _minimize;
 
+  cloneOptions._parentOptions = options;
+
   // remember the root transform function
   // to save it from being overwritten by sub-transform functions
   var originalTransform = options.transform;

package/lib/model.js

@@ -27,6 +27,8 @@ var internalToObjectOptions = require('./options').internalToObjectOptions;
 var isPathSelectedInclusive = require('./services/projection/isPathSelectedInclusive');
 var get = require('lodash.get');
 var getSchemaTypes = require('./services/populate/getSchemaTypes');
+var getVirtual = require('./services/populate/getVirtual');
+var modifiedPaths = require('./services/update/modifiedPaths');
 var mpath = require('mpath');
 var parallel = require('async/parallel');
 var parallelLimit = require('async/parallelLimit');
@@ -57,7 +59,7 @@ function Model(doc, fields, skipId) {
       '**not** a schema. Make sure you\'re calling `mongoose.model()`, not ' +
       '`mongoose.Model()`.');
   }
-  Document.call(this, doc, fields, skipId, true);
+  Document.call(this, doc, fields, skipId);
 }
 
 /*!
@@ -408,8 +410,7 @@ function operand(self, where, delta, data, val, op) {
   // editing the correct array element.
   // only increment the version if an array position changes.
   // modifying elements of an array is ok if position does not change.
-
-  if (op === '$push' || op === '$addToSet') {
+  if (op === '$push' || op === '$addToSet' || op === '$pullAll' || op === '$pull') {
     self.$__.version = VERSION_INC;
   } else if (/^\$p/.test(op)) {
     // potentially changing array positions
@@ -732,7 +733,7 @@ Model.prototype.$__where = function _where(where) {
  *     })
  *
  *
- * As an extra measure of flow control, remove will return a Promise (bound to `fn` if passed) so it could be chained, or hooked to recive errors
+ * As an extra measure of flow control, remove will return a Promise (bound to `fn` if passed) so it could be chained, or hooked to recieve errors
  *
  * ####Example:
  *     product.remove().then(function (product) {
@@ -1724,13 +1725,16 @@ Model.findOneAndUpdate = function(conditions, update, options, callback) {
 
   update = utils.clone(update, {depopulate: 1, _isNested: true});
   if (this.schema.options.versionKey && options && options.upsert) {
-    if (options.overwrite) {
-      update[this.schema.options.versionKey] = 0;
-    } else {
-      if (!update.$setOnInsert) {
-        update.$setOnInsert = {};
+    var updatedPaths = modifiedPaths(update);
+    if (!updatedPaths[this.schema.options.versionKey]) {
+      if (options.overwrite) {
+        update[this.schema.options.versionKey] = 0;
+      } else {
+        if (!update.$setOnInsert) {
+          update.$setOnInsert = {};
+        }
+        update.$setOnInsert[this.schema.options.versionKey] = 0;
       }
-      update.$setOnInsert[this.schema.options.versionKey] = 0;
     }
   }
 
@@ -2617,14 +2621,15 @@ function _update(model, op, conditions, doc, options, callback) {
   }
   options = typeof options === 'function' ? options : utils.clone(options);
 
-  if (model.schema.options.versionKey && options && options.upsert) {
+  var versionKey = get(model, 'schema.options.versionKey', null);
+  if (versionKey && options && options.upsert) {
     if (options.overwrite) {
-      doc[model.schema.options.versionKey] = 0;
+      doc[versionKey] = 0;
     } else {
       if (!doc.$setOnInsert) {
         doc.$setOnInsert = {};
       }
-      doc.$setOnInsert[model.schema.options.versionKey] = 0;
+      doc.$setOnInsert[versionKey] = 0;
     }
   }
 
@@ -2767,10 +2772,10 @@ Model.mapReduce = function mapReduce(o, callback) {
  *     });
  *
  *     // Or use the aggregation pipeline builder.
- *     Users.aggregate()
- *       .group({ _id: null, maxBalance: { $max: '$balance' } })
- *       .select('-id maxBalance')
- *       .exec(function (err, res) {
+ *     Users.aggregate().
+ *       group({ _id: null, maxBalance: { $max: '$balance' } }).
+ *       project('-id maxBalance').
+ *       exec(function (err, res) {
  *         if (err) return handleError(err);
  *         console.log(res); // [ { maxBalance: 98 } ]
  *       });
@@ -3026,8 +3031,8 @@ function _populate(model, docs, paths, cache, callback) {
 /*!
  * Populates `docs`
  */
-var excludeIdReg = /\s?-_id\s?/,
-    excludeIdRegGlobal = /\s?-_id\s?/g;
+const excludeIdReg = /\s?-_id\s?/;
+const excludeIdRegGlobal = /\s?-_id\s?/g;
 
 function populate(model, docs, options, callback) {
   var modelsMap;
@@ -3157,12 +3162,7 @@ function populate(model, docs, options, callback) {
     var val;
 
     // Clone because `assignRawDocsToIdStructure` will mutate the array
-    var allIds = [].concat(mod.allIds.map(function(v) {
-      if (Array.isArray(v)) {
-        return [].concat(v);
-      }
-      return v;
-    }));
+    var allIds = utils.clone(mod.allIds);
 
     // optimization:
     // record the document positions as returned by
@@ -3224,7 +3224,8 @@ function populate(model, docs, options, callback) {
 
     assignVals({
       originalModel: model,
-      rawIds: mod.allIds,
+      // If virtual, make sure to not mutate original field
+      rawIds: mod.isVirtual ? allIds : mod.allIds,
       allIds: allIds,
       localField: mod.localField,
       foreignField: mod.foreignField,
@@ -3253,7 +3254,6 @@ function assignVals(o) {
 
   // now update the original documents being populated using the
   // result structure that contains real documents.
-
   var docs = o.docs;
   var rawIds = o.rawIds;
   var options = o.options;
@@ -3264,7 +3264,7 @@ function assignVals(o) {
 
   for (var i = 0; i < docs.length; ++i) {
     if (utils.getValue(o.path, docs[i]) == null &&
-      !o.originalModel.schema._getVirtual(o.path)) {
+      !getVirtual(o.originalModel.schema, o.path)) {
       continue;
     }
 
@@ -3294,7 +3294,7 @@ function assignVals(o) {
       if (docs[i].$__) {
         docs[i].populated(o.path, o.allIds[i], o.allOptions);
       }
-      utils.setValue(o.path, rawIds[i], docs[i], setValue);
+      utils.setValue(o.path, rawIds[i], docs[i], setValue, false);
     }
   }
 }
@@ -3383,11 +3383,20 @@ function assignRawDocsToIdStructure(rawIds, resultDocs, resultOrder, options, lo
 }
 
 function getModelsMapForPopulate(model, docs, options) {
-  var i, doc, len = docs.length,
-      available = {},
-      map = [],
-      modelNameFromQuery = options.model && options.model.modelName || options.model,
-      schema, refPath, Model, currentOptions, modelNames, modelName, discriminatorKey, modelForFindSchema;
+  let i;
+  let doc;
+  let len = docs.length;
+  let available = {};
+  let map = [];
+  let modelNameFromQuery = options.model && options.model.modelName || options.model;
+  let schema;
+  let refPath;
+  let Model;
+  let currentOptions;
+  let modelNames;
+  let modelName;
+  let discriminatorKey;
+  let modelForFindSchema;
 
   var originalModel = options.model;
   var isVirtual = false;
@@ -3398,23 +3407,21 @@ function getModelsMapForPopulate(model, docs, options) {
     doc = docs[i];
 
     schema = getSchemaTypes(modelSchema, doc, options.path);
-    var isUnderneathDocArray = schema && schema.$isUnderneathDocArray;
-    if (isUnderneathDocArray &&
-        options &&
-        options.options &&
-        options.options.sort) {
+    const isUnderneathDocArray = schema && schema.$isUnderneathDocArray;
+    if (isUnderneathDocArray && get(options, 'options.sort') != null) {
       return new Error('Cannot populate with `sort` on path ' + options.path +
         ' because it is a subproperty of a document array');
     }
 
+    modelNames = null;
     if (Array.isArray(schema)) {
-      for (var j = 0; j < schema.length; ++j) {
+      for (let j = 0; j < schema.length; ++j) {
         var _modelNames = _getModelNames(doc, schema[j]);
         if (!_modelNames) {
           continue;
         }
-        modelNames = (modelNames || []);
-        for (var x = 0; x < _modelNames.length; ++x) {
+        modelNames = modelNames || [];
+        for (let x = 0; x < _modelNames.length; ++x) {
           if (modelNames.indexOf(_modelNames[x]) === -1) {
             modelNames.push(_modelNames[x]);
           }
@@ -3427,10 +3434,10 @@ function getModelsMapForPopulate(model, docs, options) {
       }
     }
 
-    var virtual = model.schema._getVirtual(options.path);
-    var localField;
+    let virtual = getVirtual(model.schema, options.path);
+    let localField;
     if (virtual && virtual.options) {
-      var virtualPrefix = virtual.$nestedSchemaPath ?
+      let virtualPrefix = virtual.$nestedSchemaPath ?
         virtual.$nestedSchemaPath + '.' : '';
       if (typeof virtual.options.localField === 'function') {
         localField = virtualPrefix + virtual.options.localField.call(doc, doc);
@@ -3440,17 +3447,24 @@ function getModelsMapForPopulate(model, docs, options) {
     } else {
       localField = options.path;
     }
-    var foreignField = virtual && virtual.options ?
+    let foreignField = virtual && virtual.options ?
       virtual.options.foreignField :
       '_id';
-    var justOne = true;
+    let justOne = true;
     if (virtual && virtual.options && virtual.options.ref) {
       justOne = virtual.options.justOne;
       isVirtual = true;
+      if (!modelNames) {
+        modelNames = [virtual.options.ref];
+      }
     } else if (schema) {
       justOne = !schema.$isMongooseArray;
     }
 
+    if (!modelNames) {
+      continue;
+    }
+
     if (virtual && (!localField || !foreignField)) {
       throw new Error('If you are populating a virtual, you must set the ' +
         'localField and foreignField options');
@@ -3463,11 +3477,11 @@ function getModelsMapForPopulate(model, docs, options) {
     if (typeof foreignField === 'function') {
       foreignField = foreignField.call(doc);
     }
-    var ret = convertTo_id(utils.getValue(localField, doc));
-    var id = String(utils.getValue(foreignField, doc));
+    const ret = convertTo_id(utils.getValue(localField, doc));
+    const id = String(utils.getValue(foreignField, doc));
     options._docs[id] = Array.isArray(ret) ? ret.slice() : ret;
 
-    var k = modelNames.length;
+    let k = modelNames.length;
     while (k--) {
       modelName = modelNames[k];
       if (modelName == null) {
@@ -3559,7 +3573,7 @@ function getModelsMapForPopulate(model, docs, options) {
         } else {
           schemaForCurrentDoc = schema;
         }
-        var virtual = modelForCurrentDoc.schema._getVirtual(options.path);
+        var virtual = getVirtual(modelForCurrentDoc.schema, options.path);
 
         var ref;
         if ((ref = get(schemaForCurrentDoc, 'options.ref')) != null) {

package/lib/plugins/idGetter.js

@@ -7,10 +7,24 @@
 module.exports = function(schema) {
   // ensure the documents receive an id getter unless disabled
   var autoIdGetter = !schema.paths['id'] &&
-      (!schema.options.noVirtualId && schema.options.id);
-  if (autoIdGetter) {
-    schema.virtual('id').get(idGetter);
+    (!schema.options.noVirtualId && schema.options.id);
+  if (!autoIdGetter) {
+    return;
   }
+
+  schema.virtual('id').get(idGetter);
+
+  if ('toHexString' in schema.methods) {
+    return;
+  }
+
+  // Re: gh-6115, make it easy to get something usable as an ObjectID regardless
+  // of whether a property is populated or not. With this, you can do
+  // `blogPost.author.toHexString()` and get a hex string regardless of whether
+  // `author` is populated
+  schema.methods.toHexString = function() {
+    return this.id;
+  };
 };
 
 /*!

package/lib/query.js

@@ -1,23 +1,26 @@
+'use strict';
+
 /*!
  * Module dependencies.
  */
 
-var CastError = require('./error/cast');
-var ObjectParameterError = require('./error/objectParameter');
-var QueryCursor = require('./cursor/QueryCursor');
-var ReadPreference = require('./drivers').ReadPreference;
-var cast = require('./cast');
-var castUpdate = require('./services/query/castUpdate');
-var hasDollarKeys = require('./services/query/hasDollarKeys');
-var helpers = require('./queryhelpers');
-var isInclusive = require('./services/projection/isInclusive');
-var mquery = require('mquery');
-var selectPopulatedFields = require('./services/query/selectPopulatedFields');
-var setDefaultsOnInsert = require('./services/setDefaultsOnInsert');
-var slice = require('sliced');
-var updateValidators = require('./services/updateValidators');
-var util = require('util');
-var utils = require('./utils');
+const CastError = require('./error/cast');
+const ObjectParameterError = require('./error/objectParameter');
+const QueryCursor = require('./cursor/QueryCursor');
+const ReadPreference = require('./drivers').ReadPreference;
+const cast = require('./cast');
+const castUpdate = require('./services/query/castUpdate');
+const get = require('lodash.get');
+const hasDollarKeys = require('./services/query/hasDollarKeys');
+const helpers = require('./queryhelpers');
+const isInclusive = require('./services/projection/isInclusive');
+const mquery = require('mquery');
+const selectPopulatedFields = require('./services/query/selectPopulatedFields');
+const setDefaultsOnInsert = require('./services/setDefaultsOnInsert');
+const slice = require('sliced');
+const updateValidators = require('./services/updateValidators');
+const util = require('util');
+const utils = require('./utils');
 
 /**
  * Query constructor used for building queries. You do not need
@@ -80,8 +83,9 @@ function Query(conditions, options, model, collection) {
   }
 
   this.options = this.options || {};
-  if (this.schema != null && this.schema.options.collation != null) {
-    this.options.collation = this.schema.options.collation;
+  const collation = get(this, 'schema.options.collation', null);
+  if (collation != null) {
+    this.options.collation = collation;
   }
 }
 
@@ -1163,12 +1167,14 @@ Query.prototype._optionsForExec = function(model) {
     return options;
   }
 
-  if (!('safe' in options) && model.schema.options.safe) {
-    options.safe = model.schema.options.safe;
+  const safe = get(model, 'schema.options.safe', null);
+  if (!('safe' in options) && safe != null) {
+    options.safe = safe;
   }
 
-  if (!('readPreference' in options) && model.schema.options.read) {
-    options.readPreference = model.schema.options.read;
+  const readPreference = get(model, 'schema.options.read');
+  if (!('readPreference' in options) && readPreference) {
+    options.readPreference = readPreference;
   }
 
   if (options.upsert !== void 0) {
@@ -1675,6 +1681,8 @@ Query.prototype.count = function(conditions, callback) {
     conditions = undefined;
   }
 
+  conditions = utils.toObject(conditions);
+
   if (mquery.canMerge(conditions)) {
     this.merge(conditions);
   }
@@ -3209,9 +3217,9 @@ Query.prototype.cast = function(model, obj) {
       upsert: this.options && this.options.upsert,
       strict: (this.options && 'strict' in this.options) ?
         this.options.strict :
-        (model.schema.options && model.schema.options.strict),
+        get(model, 'schema.options.strict', null),
       strictQuery: (this.options && this.options.strictQuery) ||
-        (model.schema.options && model.schema.options.strictQuery)
+        get(model, 'schema.options.strictQuery', null)
     }, this);
   } catch (err) {
     // CastError, assign model

package/lib/queryhelpers.js

@@ -103,7 +103,7 @@ exports.createModel = function createModel(model, doc, fields, userProvidedField
     }
   }
 
-  return new model(undefined, fields, true);
+  return new model(undefined, fields, { skipId: true, isNew: false });
 };
 
 /*!

package/lib/schema/array.js

@@ -70,6 +70,7 @@ function SchemaArray(key, cast, options, schemaOptions) {
       : cast;
 
     this.casterConstructor = caster;
+
     if (typeof caster === 'function' && !caster.$isArraySubdocument) {
       this.caster = new caster(null, castOptions);
     } else {
@@ -140,20 +141,6 @@ SchemaArray.prototype.enum = function() {
   return this;
 };
 
-/**
- * Check if the given value satisfies a `required` validator. The given value
- * must be an array (that is, not `null` or `undefined`).
- * Empty arrays will **not** make `required` validator fail.
- *
- * @param {Any} value
- * @return {Boolean}
- * @api public
- */
-
-SchemaArray.prototype.checkRequired = function(value) {
-  return Array.isArray(value);
-};
-
 /**
  * Overrides the getters application for the population special-case
  *
@@ -273,6 +260,9 @@ SchemaArray.prototype.castForQuery = function($conditional, value) {
     var caster = this.caster;
 
     if (Array.isArray(val)) {
+      this.setters.reverse().forEach(setter => {
+        val = setter.call(this, val, this);
+      });
       val = val.map(function(v) {
         if (utils.isObject(v) && v.$elemMatch) {
           return v;

package/lib/schema/decimal128.js

@@ -124,6 +124,10 @@ Decimal128.prototype.cast = function(value, doc, init) {
     return new Decimal128Type(value);
   }
 
+  if (typeof value === 'number') {
+    return Decimal128Type.fromString(String(value));
+  }
+
   throw new CastError('Decimal128', value, this.path);
 };
 

package/lib/schema/documentarray.js

@@ -1,4 +1,4 @@
-/* eslint no-empty: 1 */
+'use strict';
 
 /*!
  * Module dependencies.
@@ -26,15 +26,16 @@ var Subdocument;
  * @api public
  */
 
-function DocumentArray(key, schema, options) {
-  var EmbeddedDocument = _createConstructor(schema, options);
+function DocumentArray(key, schema, options, schemaOptions) {
+  const EmbeddedDocument = _createConstructor(schema, options);
   EmbeddedDocument.prototype.$basePath = key;
 
   ArrayType.call(this, key, EmbeddedDocument, options);
 
   this.schema = schema;
+  this.schemaOptions = schemaOptions || {};
   this.$isMongooseDocumentArray = true;
-  var fn = this.defaultValue;
+  const fn = this.defaultValue;
 
   if (!('defaultValue' in this) || fn !== void 0) {
     this.default(function() {
@@ -169,7 +170,7 @@ DocumentArray.prototype.doValidate = function(array, fn, scope, options) {
     for (var i = 0, len = count; i < len; ++i) {
       // sidestep sparse entries
       var doc = array[i];
-      if (!doc) {
+      if (doc == null) {
         --count || fn(error);
         continue;
       }

package/lib/schema/number.js

@@ -223,6 +223,9 @@ SchemaNumber.prototype.cast = function(value, doc, init) {
     if (typeof val === 'number') {
       return val;
     }
+    if (!Array.isArray(val) && typeof val.valueOf === 'function') {
+      return Number(val.valueOf());
+    }
     if (val.toString && !Array.isArray(val) && val.toString() == Number(val)) {
       return new Number(val);
     }

package/lib/schema.js

@@ -9,6 +9,7 @@ var utils = require('./utils');
 var MongooseTypes;
 var Kareem = require('kareem');
 var SchemaType = require('./schematype');
+var get = require('lodash.get');
 var mpath = require('mpath');
 
 /**
@@ -235,6 +236,7 @@ Schema.prototype.tree;
 Schema.prototype.clone = function() {
   var s = new Schema({}, this._userProvidedOptions);
   s.obj = this.obj;
+  s.options = utils.clone(this.options);
   s.callQueue = this.callQueue.map(function(f) { return f; });
   s.methods = utils.clone(this.methods);
   s.statics = utils.clone(this.statics);
@@ -248,12 +250,24 @@ Schema.prototype.clone = function() {
   s.nested = utils.clone(this.nested);
   s.subpaths = utils.clone(this.subpaths);
   s.childSchemas = this.childSchemas.slice();
+  s.singleNestedPaths = utils.clone(this.singleNestedPaths);
 
   s.virtuals = utils.clone(this.virtuals);
   s.$globalPluginsApplied = this.$globalPluginsApplied;
   s.$isRootDiscriminator = this.$isRootDiscriminator;
-  s.discriminatorMapping = Object.assign({}, this.discriminatorMapping);
-  s.discriminators = Object.assign({}, this.discriminators);
+
+  if (this.discriminatorMapping != null) {
+    s.discriminatorMapping = Object.assign({}, this.discriminatorMapping);
+  }
+  if (s.discriminators != null) {
+    s.discriminators = Object.assign({}, this.discriminators);
+  }
+
+  s.aliases = Object.assign({}, this.aliases);
+
+  // Bubble up `init` for backwards compat
+  s.on('init', v => this.emit('init', v));
+
   return s;
 };
 
@@ -540,7 +554,7 @@ Schema.interpretAsType = function(path, obj, options) {
     if (cast &&
         cast[options.typeKey] &&
         cast[options.typeKey].instanceOfSchema) {
-      return new MongooseTypes.DocumentArray(path, cast[options.typeKey], cast);
+      return new MongooseTypes.DocumentArray(path, cast[options.typeKey], obj, cast);
     }
 
     if (Array.isArray(cast)) {
@@ -762,11 +776,11 @@ Schema.prototype.setupTimestamp = function(timestamps) {
       var defaultTimestamp = new Date();
       var auto_id = this._id && this._id.auto;
 
-      if (createdAt != null && !this.get(createdAt) && this.isSelected(createdAt)) {
+      if (createdAt && !this.get(createdAt) && this.isSelected(createdAt)) {
         this.set(createdAt, auto_id ? this._id.getTimestamp() : defaultTimestamp);
       }
 
-      if (updatedAt != null && (this.isNew || this.isModified())) {
+      if (updatedAt && (this.isNew || this.isModified())) {
         var ts = defaultTimestamp;
         if (this.isNew) {
           if (createdAt != null) {
@@ -791,10 +805,10 @@ Schema.prototype.setupTimestamp = function(timestamps) {
           updates.$set = {};
           _updates = updates.$set;
         }
-        if (updatedAt != null && !currentUpdate[updatedAt]) {
+        if (updatedAt && !currentUpdate[updatedAt]) {
           _updates[updatedAt] = now;
         }
-        if (createdAt != null && !currentUpdate[createdAt]) {
+        if (createdAt && !currentUpdate[createdAt]) {
           _updates[createdAt] = now;
         }
         return updates;
@@ -802,12 +816,12 @@ Schema.prototype.setupTimestamp = function(timestamps) {
       updates = { $set: {} };
       currentUpdate = currentUpdate || {};
 
-      if (updatedAt != null &&
+      if (updatedAt &&
           (!currentUpdate.$currentDate || !currentUpdate.$currentDate[updatedAt])) {
         updates.$set[updatedAt] = now;
       }
 
-      if (createdAt != null) {
+      if (createdAt) {
         if (currentUpdate[createdAt]) {
           delete currentUpdate[createdAt];
         }
@@ -823,10 +837,10 @@ Schema.prototype.setupTimestamp = function(timestamps) {
     };
 
     this.methods.initializeTimestamps = function() {
-      if (!this.get(createdAt)) {
+      if (createdAt && !this.get(createdAt)) {
         this.set(createdAt, new Date());
       }
-      if (!this.get(updatedAt)) {
+      if (updatedAt && !this.get(updatedAt)) {
         this.set(updatedAt, new Date());
       }
       return this;
@@ -980,7 +994,9 @@ function getPositionalPathType(self, path) {
     if (i === last && val && !/\D/.test(subpath)) {
       if (val.$isMongooseDocumentArray) {
         var oldVal = val;
-        val = new SchemaType(subpath);
+        val = new SchemaType(subpath, {
+          required: get(val, 'schemaOptions.required', false)
+        });
         val.cast = function(value, doc, init) {
           return oldVal.cast(value, doc, init)[0];
         };
@@ -1341,7 +1357,8 @@ Schema.prototype.indexes = function() {
       path = schema.paths[key];
 
       if ((path instanceof MongooseTypes.DocumentArray) || path.$isSingleNested) {
-        if (path.options.excludeIndexes !== true) {
+        if (get(path, 'options.excludeIndexes') !== true &&
+            get(path, 'schemaOptions.excludeIndexes') !== true) {
           collectIndexes(path.schema, prefix + key + '.');
         }
 
@@ -1526,43 +1543,6 @@ Schema.prototype.virtual = function(name, options) {
   return virtuals[name];
 };
 
-/*!
- * ignore
- */
-
-Schema.prototype._getVirtual = function(name) {
-  return _getVirtual(this, name);
-};
-
-/*!
- * ignore
- */
-
-function _getVirtual(schema, name) {
-  if (schema.virtuals[name]) {
-    return schema.virtuals[name];
-  }
-  var parts = name.split('.');
-  var cur = '';
-  var nestedSchemaPath = '';
-  for (var i = 0; i < parts.length; ++i) {
-    cur += (cur.length > 0 ? '.' : '') + parts[i];
-    if (schema.virtuals[cur]) {
-      if (i === parts.length - 1) {
-        schema.virtuals[cur].$nestedSchemaPath = nestedSchemaPath;
-        return schema.virtuals[cur];
-      }
-      continue;
-    } else if (schema.paths[cur] && schema.paths[cur].schema) {
-      schema = schema.paths[cur].schema;
-      nestedSchemaPath += (nestedSchemaPath.length > 0 ? '.' : '') + cur;
-      cur = '';
-    } else {
-      return null;
-    }
-  }
-}
-
 /**
  * Returns the virtual type with the given `name`.
  *
@@ -1638,7 +1618,7 @@ Schema.prototype.remove = function(path) {
  * ```
  *
  * @param {Function} model
- * @param {Boolean} [virtualsOny] if truthy, only pulls virtuals from the class, not methods or statics
+ * @param {Boolean} [virtualsOnly] if truthy, only pulls virtuals from the class, not methods or statics
  */
 Schema.prototype.loadClass = function(model, virtualsOnly) {
   if (model === Object.prototype ||

package/lib/services/model/applyHooks.js

@@ -40,6 +40,9 @@ function applyHooks(model, schema, options) {
     }
   }
 
+  // Built-in hooks rely on hooking internal functions in order to support
+  // promises and make it so that `doc.save.toString()` provides meaningful
+  // information.
   objToDecorate.$__save = schema.s.hooks.
     createWrapper('save', objToDecorate.$__save, null, kareemOptions);
   objToDecorate.$__validate = schema.s.hooks.
@@ -48,4 +51,17 @@ function applyHooks(model, schema, options) {
     createWrapper('remove', objToDecorate.$__remove, null, kareemOptions);
   objToDecorate.$__init = schema.s.hooks.
     createWrapperSync('init', objToDecorate.$__init, null, kareemOptions);
+
+  // Support hooks for custom methods
+  const customMethods = Object.keys(schema.methods);
+  for (const method of customMethods) {
+    if (!schema.s.hooks.hasHooks(method)) {
+      // Don't wrap if there are no hooks for the custom method to avoid
+      // surprises. Also, `createWrapper()` enforces consistent async,
+      // so wrapping a sync method would break it.
+      continue;
+    }
+    objToDecorate[method] = schema.s.hooks.
+      createWrapper(method, objToDecorate[method], null, kareemOptions);
+  }
 }

package/lib/services/model/discriminator.js

@@ -136,7 +136,7 @@ module.exports = function discriminator(model, name, schema, tiedValue) {
 
     schema.plugins = Array.prototype.slice(baseSchema.plugins);
     schema.callQueue = baseSchema.callQueue.concat(schema.callQueue);
-    schema._requiredpaths = undefined; // reset just in case Schema#requiredPaths() was called on either schema
+    delete schema._requiredpaths; // reset just in case Schema#requiredPaths() was called on either schema
   }
 
   // merges base schema into new discriminator schema and sets new type field.
@@ -148,6 +148,8 @@ module.exports = function discriminator(model, name, schema, tiedValue) {
 
   if (!model.schema.discriminatorMapping) {
     model.schema.discriminatorMapping = {key: key, value: null, isRoot: true};
+  }
+  if (!model.schema.discriminators) {
     model.schema.discriminators = {};
   }
 

package/lib/services/populate/getSchemaTypes.js

@@ -14,16 +14,16 @@ var mpath = require('mpath');
  */
 
 module.exports = function getSchemaTypes(schema, doc, path) {
-  var pathschema = schema.path(path);
+  const pathschema = schema.path(path);
 
   if (pathschema) {
     return pathschema;
   }
 
   function search(parts, schema) {
-    var p = parts.length + 1;
-    var foundschema;
-    var trypath;
+    let p = parts.length + 1;
+    let foundschema;
+    let trypath;
 
     while (p--) {
       trypath = parts.slice(0, p).join('.');
@@ -35,10 +35,10 @@ module.exports = function getSchemaTypes(schema, doc, path) {
             return foundschema.caster;
           }
 
-          var schemas = null;
+          let schemas = null;
           if (doc != null && foundschema.schema != null && foundschema.schema.discriminators != null) {
-            var discriminators = foundschema.schema.discriminators;
-            var keys = mpath.get(trypath + '.' + foundschema.schema.options.discriminatorKey,
+            const discriminators = foundschema.schema.discriminators;
+            const keys = mpath.get(trypath + '.' + foundschema.schema.options.discriminatorKey,
               doc);
             schemas = Object.keys(discriminators).
               reduce(function(cur, discriminator) {
@@ -56,7 +56,7 @@ module.exports = function getSchemaTypes(schema, doc, path) {
           // If there is no foundschema.schema we are dealing with
           // a path like array.$
           if (p !== parts.length && foundschema.schema) {
-            var ret;
+            let ret;
             if (parts[p] === '$') {
               if (p + 1 === parts.length) {
                 // comments.$
@@ -74,7 +74,7 @@ module.exports = function getSchemaTypes(schema, doc, path) {
             if (schemas != null && schemas.length > 0) {
               ret = [];
               for (var i = 0; i < schemas.length; ++i) {
-                var _ret = search(parts.slice(p), schemas[i]);
+                let _ret = search(parts.slice(p), schemas[i]);
                 if (_ret != null) {
                   _ret.$isUnderneathDocArray = _ret.$isUnderneathDocArray ||
                     !foundschema.schema.$isSingleNested;
@@ -104,8 +104,8 @@ module.exports = function getSchemaTypes(schema, doc, path) {
   }
 
   // look for arrays
-  var parts = path.split('.');
-  for (var i = 0; i < parts.length; ++i) {
+  const parts = path.split('.');
+  for (let i = 0; i < parts.length; ++i) {
     if (parts[i] === '$') {
       // Re: gh-5628, because `schema.path()` doesn't take $ into account.
       parts[i] = '0';

package/lib/services/populate/getVirtual.js

@@ -0,0 +1,51 @@
+'use strict';
+
+module.exports = getVirtual;
+
+/*!
+ * ignore
+ */
+
+function getVirtual(schema, name) {
+  if (schema.virtuals[name]) {
+    return schema.virtuals[name];
+  }
+  const parts = name.split('.');
+  let cur = '';
+  let nestedSchemaPath = '';
+  for (let i = 0; i < parts.length; ++i) {
+    cur += (cur.length > 0 ? '.' : '') + parts[i];
+    if (schema.virtuals[cur]) {
+      if (i === parts.length - 1) {
+        schema.virtuals[cur].$nestedSchemaPath = nestedSchemaPath;
+        return schema.virtuals[cur];
+      }
+      continue;
+    }
+
+    if (schema.paths[cur] && schema.paths[cur].schema) {
+      schema = schema.paths[cur].schema;
+
+      if (i === parts.length - 2 && schema.discriminators) {
+        // Check for embedded discriminators, don't currently support populating
+        // nested virtuals underneath embedded discriminators because that will
+        // require substantial refactoring.
+        for (let key of Object.keys(schema.discriminators)) {
+          const discriminatorSchema = schema.discriminators[key];
+          let _cur = parts[i + 1];
+          if (discriminatorSchema.virtuals[_cur]) {
+            discriminatorSchema.virtuals[_cur].$nestedSchemaPath =
+              (nestedSchemaPath.length > 0 ? '.' : '') + cur;
+            return discriminatorSchema.virtuals[_cur];
+          }
+        }
+      }
+
+      nestedSchemaPath += (nestedSchemaPath.length > 0 ? '.' : '') + cur;
+      cur = '';
+      continue;
+    }
+
+    return null;
+  }
+}

package/lib/services/update/modifiedPaths.js

@@ -0,0 +1,27 @@
+'use strict';
+
+const _modifiedPaths = require('../common').modifiedPaths;
+
+/**
+ * Given an update document with potential update operators (`$set`, etc.)
+ * returns an object whose keys are the directly modified paths
+ *
+ * @param {Object} update
+ * @return {Object} modified
+ */
+
+module.exports = function modifiedPaths(update) {
+  const keys = Object.keys(update);
+  const hasDollarKey = keys.filter(key => key.startsWith('$')).length > 0;
+
+  const res = {};
+  if (hasDollarKey) {
+    for (const key of keys) {
+      _modifiedPaths(update[key], '', res);
+    }
+  } else {
+    _modifiedPaths(update, '', res);
+  }
+
+  return res;
+};

package/lib/types/array.js

@@ -6,6 +6,7 @@ const EmbeddedDocument = require('./embedded');
 const Document = require('../document');
 const ObjectId = require('./objectid');
 const cleanModifiedSubpaths = require('../services/document/cleanModifiedSubpaths');
+const get = require('lodash.get');
 const internalToObjectOptions = require('../options').internalToObjectOptions;
 const utils = require('../utils');
 
@@ -514,10 +515,10 @@ MongooseArray.mixin = {
    */
 
   pull: function() {
-    var values = [].map.call(arguments, this._cast, this),
-        cur = this._parent.get(this._path),
-        i = cur.length,
-        mem;
+    var values = [].map.call(arguments, this._cast, this);
+    var cur = this._parent.get(this._path);
+    var i = cur.length;
+    var mem;
 
     while (i--) {
       mem = cur[i];
@@ -822,7 +823,7 @@ function _isAllSubdocs(docs, ref) {
  */
 
 function _checkManualPopulation(arr, docs) {
-  var ref = arr._schema.caster.options && arr._schema.caster.options.ref;
+  var ref = get(arr, '_schema.caster.options.ref', null);
   if (arr.length === 0 &&
       docs.length > 0) {
     if (_isAllSubdocs(docs, ref)) {

package/lib/types/documentarray.js

@@ -64,6 +64,13 @@ function MongooseDocumentArray(values, path, doc) {
   if (doc && doc instanceof Document) {
     arr._parent = doc;
     arr._schema = doc.schema.path(path);
+
+    // Tricky but this may be a document array embedded in a normal array,
+    // in which case `path` would point to the embedded array. See #6405, #6398
+    if (arr._schema && !arr._schema.$isMongooseDocumentArray) {
+      arr._schema = arr._schema.casterConstructor;
+    }
+
     arr._handlers = {
       isNew: arr.notify('isNew'),
       save: arr.notify('save')

package/lib/utils.js

@@ -538,7 +538,22 @@ exports.populate = function populate(path, select, model, match, options, subPop
     throw new TypeError('utils.populate: invalid path. Expected string. Got typeof `' + typeof path + '`');
   }
 
-  if (typeof subPopulate === 'object') {
+  if (Array.isArray(subPopulate)) {
+    let ret = [];
+    subPopulate.forEach(function(obj) {
+      if (/[\s]/.test(obj.path)) {
+        let copy = Object.assign({}, obj);
+        let paths = copy.path.split(' ');
+        paths.forEach(function(p) {
+          copy.path = p;
+          ret.push(exports.populate(copy)[0]);
+        });
+      } else {
+        ret.push(exports.populate(obj)[0]);
+      }
+    });
+    subPopulate = exports.populate(ret);
+  } else if (typeof subPopulate === 'object') {
     subPopulate = exports.populate(subPopulate);
   }
 
@@ -571,8 +586,8 @@ exports.getValue = function(path, obj, map) {
  * @param {Object} obj
  */
 
-exports.setValue = function(path, val, obj, map) {
-  mpath.set(path, val, obj, '_doc', map);
+exports.setValue = function(path, val, obj, map, _copying) {
+  mpath.set(path, val, obj, '_doc', map, _copying);
 };
 
 /*!

package/migrating_to_5.md

@@ -102,7 +102,43 @@ const cursorWithOptions = MyModel.
 
 `Model.geoNear()` has been removed because the [MongoDB driver no longer supports it](https://github.com/mongodb/node-mongodb-native/blob/3.0.0/CHANGES_3.0.0.md#geonear-command-helper)
 
-### Domain sockets
+### Required URI encoding of connection strings
+
+Due to changes in the MongoDB driver, connection strings must be URI encoded.
+
+If they are not, connections may fail with an illegeal character message.
+
+#### Passwords which contain certain characters
+
+See a [full list of affected characters](https://developer.mozilla.org/en-US/docs/Glossary/percent-encoding).
+
+If your app is used by a lot of different connection strings, it's possible
+that your test cases will pass, but production passwords will fail. Encode all your connection
+strings to be safe.
+
+If you want to continue to use unencoded connection strings, the easiest fix is to use
+the `mongodb-uri` module to parse the connection strings, and then produce the properly encoded
+versions. You can use a function like this:
+
+```javascript
+const uriFormat = require('mongodb-uri')
+function encodeMongoURI (urlString) {
+    if (urlString) {
+      let parsed = uriFormat.parse(urlString)
+      urlString = uriFormat.format(parsed);
+    }
+    return urlString;
+  }
+}
+
+// Your un-encoded string.
+const mongodbConnectString = "mongodb://...";
+mongoose.connect(encodeMongoURI(mongodbConnectString)
+```
+
+The function above is safe to use whether the existing string is already encoded or not.
+
+#### Domain sockets
 
 Domain sockets must be URI encoded. For example:
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mongoose",
   "description": "Mongoose MongoDB ODM",
-  "version": "5.0.13",
+  "version": "5.0.17",
   "author": "Guillermo Rauch <guillermo@learnboost.com>",
   "keywords": [
     "mongodb",
@@ -21,11 +21,11 @@
   "dependencies": {
     "async": "2.1.4",
     "bson": "~1.0.4",
-    "kareem": "2.0.6",
+    "kareem": "2.0.7",
     "lodash.get": "4.4.2",
-    "mongodb": "3.0.4",
+    "mongodb": "3.0.7",
     "mongoose-legacy-pluralize": "1.0.2",
-    "mpath": "0.3.0",
+    "mpath": "0.4.1",
     "mquery": "3.0.0",
     "ms": "2.0.0",
     "regexp-clone": "0.0.1",