mongoose 5.0.17 → 5.1.2

package/lib/model.js

@@ -11,6 +11,7 @@ var DocumentNotFoundError = require('./error').DocumentNotFoundError;
 var DivergentArrayError = require('./error').DivergentArrayError;
 var Error = require('./error');
 var EventEmitter = require('events').EventEmitter;
+var MongooseMap = require('./types/map');
 var OverwriteModelError = require('./error').OverwriteModelError;
 var PromiseProvider = require('./promise_provider');
 var Query = require('./query');
@@ -35,9 +36,9 @@ var parallelLimit = require('async/parallelLimit');
 var setDefaultsOnInsert = require('./services/setDefaultsOnInsert');
 var utils = require('./utils');
 
-var VERSION_WHERE = 1,
-    VERSION_INC = 2,
-    VERSION_ALL = VERSION_WHERE | VERSION_INC;
+const VERSION_WHERE = 1;
+const VERSION_INC = 2;
+const VERSION_ALL = VERSION_WHERE | VERSION_INC;
 
 /**
  * Model constructor
@@ -129,21 +130,27 @@ Model.prototype.baseModelName;
  */
 
 Model.prototype.$__handleSave = function(options, callback) {
-  var _this = this;
-  var i;
-  var keys;
-  var len;
+  const _this = this;
+  let i;
+  let keys;
+  let len;
   if (!options.safe && this.schema.options.safe) {
     options.safe = this.schema.options.safe;
   }
   if (typeof options.safe === 'boolean') {
     options.safe = null;
   }
-  var safe = options.safe ? utils.clone(options.safe) : options.safe;
+  let safe = options.safe ? utils.clone(options.safe) : options.safe;
+
+  const session = 'session' in options ? options.session : this.$session();
+  if (session != null) {
+    safe = typeof safe === 'object' && safe != null ? safe : {};
+    safe.session = session;
+  }
 
   if (this.isNew) {
     // send entire doc
-    var obj = this.toObject(internalToObjectOptions);
+    const obj = this.toObject(internalToObjectOptions);
 
     if ((obj || {})._id === void 0) {
       // documents must have an _id else mongoose won't know
@@ -158,7 +165,7 @@ Model.prototype.$__handleSave = function(options, callback) {
     }
 
     this.$__version(true, obj);
-    this.collection.insert(obj, safe, function(err, ret) {
+    this.collection.insertOne(obj, safe, function(err, ret) {
       if (err) {
         _this.isNew = true;
         _this.emit('isNew', true);
@@ -181,7 +188,7 @@ Model.prototype.$__handleSave = function(options, callback) {
     // since it already exists
     this.$__.inserting = false;
 
-    var delta = this.$__delta();
+    const delta = this.$__delta();
 
     if (delta) {
       if (delta instanceof Error) {
@@ -189,7 +196,7 @@ Model.prototype.$__handleSave = function(options, callback) {
         return;
       }
 
-      var where = this.$__where(delta[0]);
+      const where = this.$__where(delta[0]);
       if (where instanceof Error) {
         callback(where);
         return;
@@ -203,7 +210,7 @@ Model.prototype.$__handleSave = function(options, callback) {
         }
       }
 
-      this.collection.update(where, delta[1], safe, function(err, ret) {
+      this.collection.updateOne(where, delta[1], safe, function(err, ret) {
         if (err) {
           callback(err);
           return;
@@ -234,6 +241,9 @@ Model.prototype.$__save = function(options, callback) {
       });
     }
 
+    // store the modified paths before the document is reset
+    const modifiedPaths = this.modifiedPaths();
+
     this.$__reset();
 
     let numAffected = 0;
@@ -257,16 +267,17 @@ Model.prototype.$__save = function(options, callback) {
         let doIncrement = VERSION_INC === (VERSION_INC & this.$__.version);
         this.$__.version = undefined;
 
+        let key = this.schema.options.versionKey;
+        let version = this.getValue(key) || 0;
+
         if (numAffected <= 0) {
           // the update failed. pass an error back
-          let err = new VersionError(this);
+          let err = new VersionError(this, version, modifiedPaths);
           return callback(err);
         }
 
         // increment version if was successful
         if (doIncrement) {
-          let key = this.schema.options.versionKey;
-          let version = this.getValue(key) | 0;
           this.setValue(key, version + 1);
         }
       }
@@ -321,7 +332,9 @@ Model.prototype.save = function(options, fn) {
     options = undefined;
   }
 
-  if (!options) {
+  if (options != null) {
+    options = utils.clone(options);
+  } else {
     options = {};
   }
 
@@ -505,36 +518,40 @@ function handleAtomics(self, where, delta, data, value) {
  */
 
 Model.prototype.$__delta = function() {
-  var dirty = this.$__dirty();
-  if (!dirty.length && VERSION_ALL !== this.$__.version) return;
+  const dirty = this.$__dirty();
+  if (!dirty.length && VERSION_ALL !== this.$__.version) {
+    return;
+  }
 
-  var where = {},
-      delta = {},
-      len = dirty.length,
-      divergent = [],
-      d = 0;
+  let where = {};
+  let delta = {};
+  const len = dirty.length;
+  const divergent = [];
+  let d = 0;
 
   where._id = this._doc._id;
-  if (where._id.toObject) {
+  // If `_id` is an object, need to depopulate, but also need to be careful
+  // because `_id` can technically be null (see gh-6406)
+  if (get(where, '_id.$__', null) != null) {
     where._id = where._id.toObject({ transform: false, depopulate: true });
   }
 
   for (; d < len; ++d) {
-    var data = dirty[d];
-    var value = data.value;
+    const data = dirty[d];
+    let value = data.value;
 
-    var match = checkDivergentArray(this, data.path, value);
+    const match = checkDivergentArray(this, data.path, value);
     if (match) {
       divergent.push(match);
       continue;
     }
 
-    var pop = this.populated(data.path, true);
+    const pop = this.populated(data.path, true);
     if (!pop && this.$__.selected) {
       // If any array was selected using an $elemMatch projection, we alter the path and where clause
       // NOTE: MongoDB only supports projected $elemMatch on top level array.
-      var pathSplit = data.path.split('.');
-      var top = pathSplit[0];
+      const pathSplit = data.path.split('.');
+      const top = pathSplit[0];
       if (this.$__.selected[top] && this.$__.selected[top].$elemMatch) {
         // If the selected array entry was modified
         if (pathSplit.length > 1 && pathSplit[1] == 0 && typeof where[top] === 'undefined') {
@@ -714,7 +731,7 @@ Model.prototype.$__where = function _where(where) {
     where._id = this._doc._id;
   }
 
-  if (this._doc._id == null) {
+  if (this._doc._id === void 0) {
     return new Error('No _id found on document!');
   }
 
@@ -1064,8 +1081,8 @@ function _ensureIndexes(model, options, callback) {
     var index = indexes.shift();
     if (!index) return done();
 
-    var indexFields = index[0];
-    var indexOptions = index[1];
+    var indexFields = utils.clone(index[0]);
+    var indexOptions = utils.clone(index[1]);
     _handleSafe(options);
 
     indexSingleStart(indexFields, options);
@@ -1843,6 +1860,126 @@ Model.findByIdAndUpdate = function(id, update, options, callback) {
   return this.findOneAndUpdate.call(this, {_id: id}, update, options, callback);
 };
 
+/**
+ * Issue a MongoDB `findOneAndDelete()` command.
+ *
+ * Finds a matching document, removes it, and passes the found document
+ * (if any) to the callback.
+ *
+ * Executes immediately if `callback` is passed else a Query object is returned.
+ *
+ * This function triggers the following middleware.
+ *
+ * - `findOneAndDelete()`
+ *
+ * This function differs slightly from `Model.findOneAndRemove()` in that
+ * `findOneAndRemove()` becomes a [MongoDB `findAndModify()` command](https://docs.mongodb.com/manual/reference/method/db.collection.findAndModify/),
+ * as opposed to a `findOneAndDelete()` command. For most mongoose use cases,
+ * this distinction is purely pedantic. You should use `findOneAndDelete()`
+ * unless you have a good reason not to.
+ *
+ * ####Options:
+ *
+ * - `sort`: if multiple docs are found by the conditions, sets the sort order to choose which doc to update
+ * - `maxTimeMS`: puts a time limit on the query - requires mongodb >= 2.6.0
+ * - `select`: sets the document fields to return
+ * - `rawResult`: if true, returns the [raw result from the MongoDB driver](http://mongodb.github.io/node-mongodb-native/2.0/api/Collection.html#findAndModify)
+ * - `strict`: overwrites the schema's [strict mode option](http://mongoosejs.com/docs/guide.html#strict) for this update
+ *
+ * ####Examples:
+ *
+ *     A.findOneAndDelete(conditions, options, callback) // executes
+ *     A.findOneAndDelete(conditions, options)  // return Query
+ *     A.findOneAndDelete(conditions, callback) // executes
+ *     A.findOneAndDelete(conditions) // returns Query
+ *     A.findOneAndDelete()           // returns Query
+ *
+ * Values are cast to their appropriate types when using the findAndModify helpers.
+ * However, the below are not executed by default.
+ *
+ * - defaults. Use the `setDefaultsOnInsert` option to override.
+ *
+ * `findAndModify` helpers support limited validation. You can
+ * enable these by setting the `runValidators` options,
+ * respectively.
+ *
+ * If you need full-fledged validation, use the traditional approach of first
+ * retrieving the document.
+ *
+ *     Model.findById(id, function (err, doc) {
+ *       if (err) ..
+ *       doc.name = 'jason bourne';
+ *       doc.save(callback);
+ *     });
+ *
+ * @param {Object} conditions
+ * @param {Object} [options] optional see [`Query.prototype.setOptions()`](http://mongoosejs.com/docs/api.html#query_Query-setOptions)
+ * @param {Function} [callback]
+ * @return {Query}
+ * @api public
+ */
+
+Model.findOneAndDelete = function(conditions, options, callback) {
+  if (arguments.length === 1 && typeof conditions === 'function') {
+    var msg = 'Model.findOneAndDelete(): First argument must not be a function.\n\n'
+        + '  ' + this.modelName + '.findOneAndDelete(conditions, callback)\n'
+        + '  ' + this.modelName + '.findOneAndDelete(conditions)\n'
+        + '  ' + this.modelName + '.findOneAndDelete()\n';
+    throw new TypeError(msg);
+  }
+
+  if (typeof options === 'function') {
+    callback = options;
+    options = undefined;
+  }
+  if (callback) {
+    callback = this.$wrapCallback(callback);
+  }
+
+  var fields;
+  if (options) {
+    fields = options.select;
+    options.select = undefined;
+  }
+
+  var mq = new this.Query({}, {}, this, this.collection);
+  mq.select(fields);
+
+  return mq.findOneAndDelete(conditions, options, callback);
+};
+
+/**
+ * Issue a MongoDB `findOneAndDelete()` command by a document's _id field.
+ * In other words, `findByIdAndDelete(id)` is a shorthand for
+ * `findOneAndDelete({ _id: id })`.
+ *
+ * This function triggers the following middleware.
+ *
+ * - `findOneAndDelete()`
+ *
+ * @param {Object|Number|String} id value of `_id` to query by
+ * @param {Object} [options] optional see [`Query.prototype.setOptions()`](http://mongoosejs.com/docs/api.html#query_Query-setOptions)
+ * @param {Function} [callback]
+ * @return {Query}
+ * @see Model.findOneAndRemove #model_Model.findOneAndRemove
+ * @see mongodb http://www.mongodb.org/display/DOCS/findAndModify+Command
+ */
+
+Model.findByIdAndDelete = function(id, options, callback) {
+  if (arguments.length === 1 && typeof id === 'function') {
+    var msg = 'Model.findByIdAndDelete(): First argument must not be a function.\n\n'
+        + '  ' + this.modelName + '.findByIdAndDelete(id, callback)\n'
+        + '  ' + this.modelName + '.findByIdAndDelete(id)\n'
+        + '  ' + this.modelName + '.findByIdAndDelete()\n';
+    throw new TypeError(msg);
+  }
+  if (callback) {
+    callback = this.$wrapCallback(callback);
+  }
+
+  return this.findOneAndDelete({_id: id}, options, callback);
+};
+
 /**
  * Issue a mongodb findAndModify remove command.
  *
@@ -2013,15 +2150,15 @@ Model.findByIdAndRemove = function(id, options, callback) {
  */
 
 Model.create = function create(doc, callback) {
-  var args;
-  var cb;
-  var discriminatorKey = this.schema.options.discriminatorKey;
+  let args;
+  let cb;
+  const discriminatorKey = this.schema.options.discriminatorKey;
 
   if (Array.isArray(doc)) {
     args = doc;
     cb = callback;
   } else {
-    var last = arguments[arguments.length - 1];
+    let last = arguments[arguments.length - 1];
     // Handle falsy callbacks re: #5061
     if (typeof last === 'function' || !last) {
       cb = last;
@@ -2040,15 +2177,19 @@ Model.create = function create(doc, callback) {
       return cb(null);
     }
 
-    var toExecute = [];
-    var firstError;
+    const toExecute = [];
+    let firstError;
     args.forEach(doc => {
       toExecute.push(callback => {
-        var Model = this.discriminators && doc[discriminatorKey] ?
+        const Model = this.discriminators && doc[discriminatorKey] != null ?
           this.discriminators[doc[discriminatorKey]] || getDiscriminatorByValue(this, doc[discriminatorKey]) :
           this;
-        var toSave = doc;
-        var callbackWrapper = (error, doc) => {
+        if (Model == null) {
+          throw new Error(`Discriminator "${doc[discriminatorKey]}" not ` +
+            `found for model "${this.modelName}"`);
+        }
+        let toSave = doc;
+        const callbackWrapper = (error, doc) => {
           if (error) {
             if (!firstError) {
               firstError = error;
@@ -2071,9 +2212,9 @@ Model.create = function create(doc, callback) {
     });
 
     parallel(toExecute, (error, res) => {
-      var savedDocs = [];
-      var len = res.length;
-      for (var i = 0; i < len; ++i) {
+      const savedDocs = [];
+      const len = res.length;
+      for (let i = 0; i < len; ++i) {
         if (res[i].doc) {
           savedDocs.push(res[i].doc);
         }
@@ -2113,7 +2254,7 @@ Model.create = function create(doc, callback) {
  *
  * @param {Array} [pipeline]
  * @param {Object} [options] see the [mongodb driver options](http://mongodb.github.io/node-mongodb-native/3.0/api/Collection.html#watch)
- * @return {ChangeStream} mongoose-specific change stream wrapper
+ * @return {ChangeStream} mongoose-specific change stream wrapper, inherits from EventEmitter
  * @api public
  */
 
@@ -2121,6 +2262,41 @@ Model.watch = function(pipeline, options) {
   return new ChangeStream(this, pipeline, options);
 };
 
+/**
+ * _Requires MongoDB >= 3.6.0._ Starts a [MongoDB session](https://docs.mongodb.com/manual/release-notes/3.6/#client-sessions)
+ * for benefits like causal consistency and [retryable writes](https://docs.mongodb.com/manual/core/retryable-writes/).
+ *
+ * This function does not trigger any middleware.
+ *
+ * ####Example:
+ *
+ *     const session = await Person.startSession();
+ *     let doc = await Person.findOne({ name: 'Ned Stark' }, { session });
+ *     await doc.remove();
+ *     // `doc` will always be null, even if reading from a replica set
+ *     // secondary. Without causal consistency, it is possible to
+ *     // get a doc back from the below query if the query reads from a
+ *     // secondary that is experiencing replication lag.
+ *     doc = await Person.findOne({ name: 'Ned Stark' }, { readPreference: 'secondary' });
+ *
+ * @param {Object} [options] see the [mongodb driver options](http://mongodb.github.io/node-mongodb-native/3.0/api/MongoClient.html#startSession)
+ * @param {Boolean} [options.causalConsistency=true] set to false to disable causal consistency
+ * @param {Function} [callback]
+ * @return {Promise<ClientSession>} promise that resolves to a MongoDB driver `ClientSession`
+ * @api public
+ */
+
+Model.startSession = function(options, callback) {
+  return utils.promiseOrCallback(callback, cb => {
+    if (this.collection.buffer) {
+      return this.collection.addQueue(() => {
+        cb(null, this.db.client.startSession(options));
+      });
+    }
+    return cb(null, this.db.client.startSession(options));
+  });
+};
+
 /**
  * Shortcut for validating an array of documents and inserting them into
  * MongoDB if they're all valid. This function is faster than `.create()`
@@ -2189,7 +2365,9 @@ Model.$__insertMany = function(arr, options, callback) {
   var validationErrors = [];
   arr.forEach(function(doc) {
     toExecute.push(function(callback) {
-      doc = new _this(doc);
+      if (!(doc instanceof _this)) {
+        doc = new _this(doc);
+      }
       doc.validate({ __noPromise: true }, function(error) {
         if (error) {
           // Option `ordered` signals that insert should be continued after reaching
@@ -2646,8 +2824,10 @@ function _update(model, op, conditions, doc, options, callback) {
  * ####Example:
  *
  *     var o = {};
- *     o.map = function () { emit(this.name, 1) }
- *     o.reduce = function (k, vals) { return vals.length }
+ *     // `map()` and `reduce()` are run on the MongoDB server, not Node.js,
+ *     // these functions are converted to strings
+ *     o.map = function () { emit(this.name, 1) };
+ *     o.reduce = function (k, vals) { return vals.length };
  *     User.mapReduce(o, function (err, results) {
  *       console.log(results)
  *     })
@@ -2677,8 +2857,10 @@ function _update(model, op, conditions, doc, options, callback) {
  * ####Example:
  *
  *     var o = {};
- *     o.map = function () { emit(this.name, 1) }
- *     o.reduce = function (k, vals) { return vals.length }
+ *     // You can also define `map()` and `reduce()` as strings if your
+ *     // linter complains about `emit()` not being defined
+ *     o.map = 'function () { emit(this.name, 1) }';
+ *     o.reduce = 'function (k, vals) { return vals.length }';
  *     o.out = { replace: 'createdCollectionNameForResults' }
  *     o.verbose = true;
  *
@@ -2969,6 +3151,7 @@ Model.geoSearch = function(conditions, options, callback) {
  *
  * @param {Document|Array} docs Either a single document or array of documents to populate.
  * @param {Object} options A hash of key/val (path, options) used for population.
+ * @param {Boolean} [options.retainNullValues=false] by default, mongoose removes null and undefined values from populated arrays. Use this option to make `populate()` retain `null` and `undefined` array entries.
  * @param {Function} [callback(err,doc)] Optional callback, executed upon completion. Receives `err` and the `doc(s)`.
  * @return {Promise}
  * @api public
@@ -3035,8 +3218,6 @@ const excludeIdReg = /\s?-_id\s?/;
 const excludeIdRegGlobal = /\s?-_id\s?/g;
 
 function populate(model, docs, options, callback) {
-  var modelsMap;
-
   // normalize single / multiple docs passed
   if (!Array.isArray(docs)) {
     docs = [docs];
@@ -3046,15 +3227,20 @@ function populate(model, docs, options, callback) {
     return callback();
   }
 
-  modelsMap = getModelsMapForPopulate(model, docs, options);
+  const modelsMap = getModelsMapForPopulate(model, docs, options);
+
   if (modelsMap instanceof Error) {
     return utils.immediate(function() {
       callback(modelsMap);
     });
   }
 
-  var i, len = modelsMap.length,
-      mod, match, select, vals = [];
+  let i;
+  const len = modelsMap.length;
+  let mod;
+  let match;
+  let select;
+  let vals = [];
 
   function flatten(item) {
     // no need to include undefined values in our query
@@ -3073,7 +3259,7 @@ function populate(model, docs, options, callback) {
       match = {};
     }
 
-    var ids = utils.array.flatten(mod.ids, flatten);
+    let ids = utils.array.flatten(mod.ids, flatten);
     ids = utils.array.unique(ids);
 
     if (ids.length === 0 || ids.every(utils.isNullOrUndefined)) {
@@ -3247,24 +3433,31 @@ function populate(model, docs, options, callback) {
  */
 
 function assignVals(o) {
+  // Glob all options together because `populateOptions` is confusing
+  const retainNullValues = get(o, 'allOptions.options.options.retainNullValues', false);
+  const populateOptions = Object.assign({}, o.options, {
+    justOne: o.justOne,
+    retainNullValues: retainNullValues
+  });
+
   // replace the original ids in our intermediate _ids structure
   // with the documents found by query
-  assignRawDocsToIdStructure(o.rawIds, o.rawDocs, o.rawOrder, o.options,
+  assignRawDocsToIdStructure(o.rawIds, o.rawDocs, o.rawOrder, populateOptions,
     o.localField, o.foreignField);
 
   // now update the original documents being populated using the
   // result structure that contains real documents.
-  var docs = o.docs;
-  var rawIds = o.rawIds;
-  var options = o.options;
+  const docs = o.docs;
+  const rawIds = o.rawIds;
+  const options = o.options;
 
   function setValue(val) {
-    return valueFilter(val, options, o.justOne);
+    return valueFilter(val, options, populateOptions);
   }
 
-  for (var i = 0; i < docs.length; ++i) {
-    if (utils.getValue(o.path, docs[i]) == null &&
-      !getVirtual(o.originalModel.schema, o.path)) {
+  for (let i = 0; i < docs.length; ++i) {
+    const existingVal = utils.getValue(o.path, docs[i]);
+    if (existingVal == null && !getVirtual(o.originalModel.schema, o.path)) {
       continue;
     }
 
@@ -3278,10 +3471,27 @@ function assignVals(o) {
       rawIds[i] = rawIds[i][0];
     }
 
+
+    // If we're populating a map, the existing value will be an object, so
+    // we need to transform again
+    const isMap = docs[i].constructor.name === 'model' ?
+      existingVal instanceof Map :
+      existingVal != null && existingVal.constructor.name === 'Object';
+    if (!o.isVirtual && isMap) {
+      const _keys = existingVal instanceof Map ?
+        Array.from(existingVal.keys()) :
+        Object.keys(existingVal);
+      rawIds[i] = rawIds[i].reduce((cur, v, i) => {
+        // Avoid casting because that causes infinite recursion
+        cur.$init(_keys[i], v);
+        return cur;
+      }, new MongooseMap({}, docs[i]));
+    }
+
     if (o.isVirtual && docs[i].constructor.name === 'model') {
       // If virtual populate and doc is already init-ed, need to walk through
       // the actual doc to set rather than setting `_doc` directly
-      mpath.set(o.path, rawIds[i], docs[i]);
+      mpath.set(o.path, rawIds[i], docs[i], setValue);
     } else {
       var parts = o.path.split('.');
       var cur = docs[i];
@@ -3294,6 +3504,7 @@ function assignVals(o) {
       if (docs[i].$__) {
         docs[i].populated(o.path, o.allIds[i], o.allOptions);
       }
+
       utils.setValue(o.path, rawIds[i], docs[i], setValue, false);
     }
   }
@@ -3374,9 +3585,6 @@ function assignRawDocsToIdStructure(rawIds, resultDocs, resultOrder, options, lo
     // can safely use this to our advantage dealing with sorted
     // result sets too.
     newOrder.forEach(function(doc, i) {
-      if (!doc) {
-        return;
-      }
       rawIds[i] = doc;
     });
   }
@@ -3477,6 +3685,7 @@ function getModelsMapForPopulate(model, docs, options) {
     if (typeof foreignField === 'function') {
       foreignField = foreignField.call(doc);
     }
+
     const ret = convertTo_id(utils.getValue(localField, doc));
     const id = String(utils.getValue(foreignField, doc));
     options._docs[id] = Array.isArray(ret) ? ret.slice() : ret;
@@ -3539,6 +3748,9 @@ function getModelsMapForPopulate(model, docs, options) {
     if (schema && schema.caster) {
       schema = schema.caster;
     }
+    if (schema && schema.$isSchemaMap) {
+      schema = schema.$__schemaType;
+    }
 
     if (!schema && model.discriminators) {
       discriminatorKey = model.schema.discriminatorMapping.key;
@@ -3548,7 +3760,11 @@ function getModelsMapForPopulate(model, docs, options) {
 
     if (refPath) {
       modelNames = utils.getValue(refPath, doc);
-      isRefPathArray = Array.isArray(modelNames);
+      isRefPathArray = false;
+      if (Array.isArray(modelNames)) {
+        isRefPathArray = true;
+        modelNames = utils.array.flatten(modelNames);
+      }
     } else {
       if (!modelNameFromQuery) {
         var modelForCurrentDoc = model;
@@ -3627,18 +3843,33 @@ function convertTo_id(val) {
   if (val instanceof Model) return val._id;
 
   if (Array.isArray(val)) {
-    for (var i = 0; i < val.length; ++i) {
+    for (let i = 0; i < val.length; ++i) {
       if (val[i] instanceof Model) {
         val[i] = val[i]._id;
       }
     }
-    if (val.isMongooseArray) {
+    if (val.isMongooseArray && val._schema) {
       return val._schema.cast(val, val._parent);
     }
 
     return [].concat(val);
   }
 
+  // `populate('map')` may be an object if populating on a doc that hasn't
+  // been hydrated yet
+  if (val != null && val.constructor.name === 'Object') {
+    const ret = [];
+    for (const key of Object.keys(val)) {
+      ret.push(val[key]);
+    }
+    return ret;
+  }
+  // If doc has already been hydrated, e.g. `doc.populate('map').execPopulate()`
+  // then `val` will already be a map
+  if (val instanceof Map) {
+    return Array.from(val.values());
+  }
+
   return val;
 }
 
@@ -3660,14 +3891,16 @@ function convertTo_id(val) {
  * that population mapping can occur.
  */
 
-function valueFilter(val, assignmentOpts, justOne) {
+function valueFilter(val, assignmentOpts, populateOptions) {
   if (Array.isArray(val)) {
     // find logic
-    var ret = [];
-    var numValues = val.length;
-    for (var i = 0; i < numValues; ++i) {
+    const ret = [];
+    const numValues = val.length;
+    for (let i = 0; i < numValues; ++i) {
       var subdoc = val[i];
-      if (!isDoc(subdoc)) continue;
+      if (!isDoc(subdoc) && (!populateOptions.retainNullValues || subdoc != null)) {
+        continue;
+      }
       maybeRemoveId(subdoc, assignmentOpts);
       ret.push(subdoc);
       if (assignmentOpts.originalLimit &&
@@ -3681,7 +3914,7 @@ function valueFilter(val, assignmentOpts, justOne) {
     while (val.length > ret.length) {
       Array.prototype.pop.apply(val, []);
     }
-    for (i = 0; i < ret.length; ++i) {
+    for (let i = 0; i < ret.length; ++i) {
       val[i] = ret[i];
     }
     return val;
@@ -3693,7 +3926,7 @@ function valueFilter(val, assignmentOpts, justOne) {
     return val;
   }
 
-  return justOne ? null : [];
+  return populateOptions.justOne ? (val == null ? val : null) : [];
 }
 
 /*!
@@ -3870,24 +4103,24 @@ function applyQueryMiddleware(Query, model) {
     nullResultByDefault: true
   };
 
-  Query.prototype._count = model.hooks.createWrapper('count',
-    Query.prototype._count, null, kareemOptions);
+  // `update()` thunk has a different name because `_update` was already taken
   Query.prototype._execUpdate = model.hooks.createWrapper('update',
     Query.prototype._execUpdate, null, kareemOptions);
-  Query.prototype._find = model.hooks.createWrapper('find',
-    Query.prototype._find, null, kareemOptions);
-  Query.prototype._findOne = model.hooks.createWrapper('findOne',
-    Query.prototype._findOne, null, kareemOptions);
-  Query.prototype._findOneAndRemove = model.hooks.createWrapper('findOneAndRemove',
-    Query.prototype._findOneAndRemove, null, kareemOptions);
-  Query.prototype._findOneAndUpdate = model.hooks.createWrapper('findOneAndUpdate',
-    Query.prototype._findOneAndUpdate, null, kareemOptions);
-  Query.prototype._replaceOne = model.hooks.createWrapper('replaceOne',
-    Query.prototype._replaceOne, null, kareemOptions);
-  Query.prototype._updateMany = model.hooks.createWrapper('updateMany',
-    Query.prototype._updateMany, null, kareemOptions);
-  Query.prototype._updateOne = model.hooks.createWrapper('updateOne',
-    Query.prototype._updateOne, null, kareemOptions);
+
+  [
+    'count',
+    'find',
+    'findOne',
+    'findOneAndDelete',
+    'findOneAndRemove',
+    'findOneAndUpdate',
+    'replaceOne',
+    'updateMany',
+    'updateOne'
+  ].forEach(fn => {
+    Query.prototype[`_${fn}`] = model.hooks.createWrapper(fn,
+      Query.prototype[`_${fn}`], null, kareemOptions);
+  });
 }
 
 /*!