mongodb 3.3.0-beta1 → 3.3.0-beta2

package/lib/aggregation_cursor.js

@@ -56,10 +56,12 @@ const MongoDBNamespace = require('./utils').MongoDBNamespace;
  * @fires AggregationCursor#readable
  * @return {AggregationCursor} an AggregationCursor instance.
  */
-var AggregationCursor = function(bson, ns, cmd, options, topology, topologyOptions) {
+var AggregationCursor = function(topology, ns, cmd, options) {
   CoreCursor.apply(this, Array.prototype.slice.call(arguments, 0));
   var state = AggregationCursor.INIT;
   var streamOptions = {};
+  const bson = topology.s.bson;
+  const topologyOptions = topology.s.options;
 
   // MaxTimeMS
   var maxTimeMS = null;

package/lib/change_stream.js

@@ -5,8 +5,14 @@ const isResumableError = require('./error').isResumableError;
 const MongoError = require('./core').MongoError;
 const ReadConcern = require('./read_concern');
 const MongoDBNamespace = require('./utils').MongoDBNamespace;
+const Cursor = require('./cursor');
+const relayEvents = require('./core/utils').relayEvents;
+const maxWireVersion = require('./core/utils').maxWireVersion;
 
-var cursorOptionNames = ['maxAwaitTimeMS', 'collation', 'readPreference'];
+const CHANGE_STREAM_OPTIONS = ['resumeAfter', 'startAfter', 'startAtOperationTime', 'fullDocument'];
+const CURSOR_OPTIONS = ['batchSize', 'maxAwaitTimeMS', 'collation', 'readPreference'].concat(
+  CHANGE_STREAM_OPTIONS
+);
 
 const CHANGE_DOMAIN_TYPES = {
   COLLECTION: Symbol('Collection'),
@@ -14,26 +20,45 @@ const CHANGE_DOMAIN_TYPES = {
   CLUSTER: Symbol('Cluster')
 };
 
+/**
+ * @typedef ResumeToken
+ * @description Represents the logical starting point for a new or resuming {@link ChangeStream} on the server.
+ * @see https://docs.mongodb.com/master/changeStreams/#change-stream-resume-token
+ */
+
+/**
+ * @typedef OperationTime
+ * @description Represents a specific point in time on a server. Can be retrieved by using {@link Db#command}
+ * @see https://docs.mongodb.com/manual/reference/method/db.runCommand/#response
+ */
+
+/**
+ * @typedef ChangeStreamOptions
+ * @description Options that can be passed to a ChangeStream. Note that startAfter, resumeAfter, and startAtOperationTime are all mutually exclusive, and the server will error if more than one is specified.
+ * @property {string} [fullDocument='default'] Allowed values: ‘default’, ‘updateLookup’. When set to ‘updateLookup’, the change stream will include both a delta describing the changes to the document, as well as a copy of the entire document that was changed from some time after the change occurred.
+ * @property {number} [maxAwaitTimeMS] The maximum amount of time for the server to wait on new documents to satisfy a change stream query.
+ * @property {ResumeToken} [resumeAfter] Allows you to start a changeStream after a specified event. See {@link https://docs.mongodb.com/master/changeStreams/#resumeafter-for-change-streams|ChangeStream documentation}.
+ * @property {ResumeToken} [startAfter] Similar to resumeAfter, but will allow you to start after an invalidated event. See {@link https://docs.mongodb.com/master/changeStreams/#startafter-for-change-streams|ChangeStream documentation}.
+ * @property {OperationTime} [startAtOperationTime] Will start the changeStream after the specified operationTime.
+ * @property {number} [batchSize] The number of documents to return per batch. See {@link https://docs.mongodb.com/manual/reference/command/aggregate|aggregation documentation}.
+ * @property {object} [collation] Specify collation settings for operation. See {@link https://docs.mongodb.com/manual/reference/command/aggregate|aggregation documentation}.
+ * @property {ReadPreference} [readPreference] The read preference. Defaults to the read preference of the database or collection. See {@link https://docs.mongodb.com/manual/reference/read-preference|read preference documentation}.
+ */
+
 /**
  * Creates a new Change Stream instance. Normally created using {@link Collection#watch|Collection.watch()}.
  * @class ChangeStream
  * @since 3.0.0
  * @param {(MongoClient|Db|Collection)} changeDomain The domain against which to create the change stream
  * @param {Array} pipeline An array of {@link https://docs.mongodb.com/manual/reference/operator/aggregation-pipeline/|aggregation pipeline stages} through which to pass change stream documents
- * @param {object} [options] Optional settings
- * @param {string} [options.fullDocument='default'] Allowed values: ‘default’, ‘updateLookup’. When set to ‘updateLookup’, the change stream will include both a delta describing the changes to the document, as well as a copy of the entire document that was changed from some time after the change occurred.
- * @param {number} [options.maxAwaitTimeMS] The maximum amount of time for the server to wait on new documents to satisfy a change stream query
- * @param {object} [options.resumeAfter] Specifies the logical starting point for the new change stream. This should be the _id field from a previously returned change stream document.
- * @param {number} [options.batchSize] The number of documents to return per batch. See {@link https://docs.mongodb.com/manual/reference/command/aggregate|aggregation documentation}.
- * @param {object} [options.collation] Specify collation settings for operation. See {@link https://docs.mongodb.com/manual/reference/command/aggregate|aggregation documentation}.
- * @param {ReadPreference} [options.readPreference] The read preference. Defaults to the read preference of the database or collection. See {@link https://docs.mongodb.com/manual/reference/read-preference|read preference documentation}.
+ * @param {ChangeStreamOptions} [options] Optional settings
  * @fires ChangeStream#close
  * @fires ChangeStream#change
  * @fires ChangeStream#end
  * @fires ChangeStream#error
+ * @fires ChangeStream#resumeTokenChanged
  * @return {ChangeStream} a ChangeStream instance.
  */
-
 class ChangeStream extends EventEmitter {
   constructor(changeDomain, pipeline, options) {
     super();
@@ -69,16 +94,8 @@ class ChangeStream extends EventEmitter {
       this.options.readPreference = changeDomain.s.readPreference;
     }
 
-    // We need to get the operationTime as early as possible
-    const isMaster = this.topology.lastIsMaster();
-    if (!isMaster) {
-      throw new MongoError('Topology does not have an ismaster yet.');
-    }
-
-    this.operationTime = isMaster.operationTime;
-
     // Create contained Change Stream cursor
-    this.cursor = createChangeStreamCursor(this);
+    this.cursor = createChangeStreamCursor(this, options);
 
     // Listen for any `change` listeners being added to ChangeStream
     this.on('newListener', eventName => {
@@ -97,6 +114,15 @@ class ChangeStream extends EventEmitter {
     });
   }
 
+  /**
+   * @property {ResumeToken} resumeToken
+   * The cached resume token that will be used to resume
+   * after the most recently returned change.
+   */
+  get resumeToken() {
+    return this.cursor.resumeToken;
+  }
+
   /**
    * Check if there is any document still available in the Change Stream
    * @function ChangeStream.prototype.hasNext
@@ -215,13 +241,114 @@ class ChangeStream extends EventEmitter {
   }
 }
 
-// Create a new change stream cursor based on self's configuration
-var createChangeStreamCursor = function(self) {
-  if (self.resumeToken) {
-    self.options.resumeAfter = self.resumeToken;
+class ChangeStreamCursor extends Cursor {
+  constructor(topology, ns, cmd, options) {
+    // TODO: spread will help a lot here
+    super(topology, ns, cmd, options);
+
+    options = options || {};
+    this._resumeToken = null;
+    this.startAtOperationTime = options.startAtOperationTime;
+
+    if (options.startAfter) {
+      this.resumeToken = options.startAfter;
+    } else if (options.resumeAfter) {
+      this.resumeToken = options.resumeAfter;
+    }
+  }
+
+  set resumeToken(token) {
+    this._resumeToken = token;
+    this.emit('resumeTokenChanged', token);
+  }
+
+  get resumeToken() {
+    return this._resumeToken;
+  }
+
+  get resumeOptions() {
+    const result = {};
+    for (const optionName of CURSOR_OPTIONS) {
+      if (this.options[optionName]) result[optionName] = this.options[optionName];
+    }
+
+    if (this.resumeToken || this.startAtOperationTime) {
+      ['resumeAfter', 'startAfter', 'startAtOperationTime'].forEach(key => delete result[key]);
+
+      if (this.resumeToken) {
+        result.resumeAfter = this.resumeToken;
+      } else if (this.startAtOperationTime && maxWireVersion(this.server) >= 7) {
+        result.startAtOperationTime = this.startAtOperationTime;
+      }
+    }
+
+    return result;
   }
 
-  var changeStreamCursor = buildChangeStreamAggregationCommand(self);
+  _initializeCursor(callback) {
+    super._initializeCursor((err, result) => {
+      if (err) {
+        callback(err, null);
+        return;
+      }
+
+      const response = result.documents[0];
+
+      if (
+        this.startAtOperationTime == null &&
+        this.resumeAfter == null &&
+        this.startAfter == null &&
+        maxWireVersion(this.server) >= 7
+      ) {
+        this.startAtOperationTime = response.operationTime;
+      }
+
+      const cursor = response.cursor;
+      if (cursor.postBatchResumeToken) {
+        this.cursorState.postBatchResumeToken = cursor.postBatchResumeToken;
+
+        if (cursor.firstBatch.length === 0) {
+          this.resumeToken = cursor.postBatchResumeToken;
+        }
+      }
+
+      this.emit('response');
+      callback(err, result);
+    });
+  }
+
+  _getMore(callback) {
+    super._getMore((err, response) => {
+      if (err) {
+        callback(err, null);
+        return;
+      }
+
+      const cursor = response.cursor;
+      if (cursor.postBatchResumeToken) {
+        this.cursorState.postBatchResumeToken = cursor.postBatchResumeToken;
+
+        if (cursor.nextBatch.length === 0) {
+          this.resumeToken = cursor.postBatchResumeToken;
+        }
+      }
+
+      this.emit('response');
+      callback(err, response);
+    });
+  }
+}
+
+/**
+ * @event ChangeStreamCursor#response
+ * internal event DO NOT USE
+ * @ignore
+ */
+
+// Create a new change stream cursor based on self's configuration
+var createChangeStreamCursor = function(self, options) {
+  const changeStreamCursor = buildChangeStreamAggregationCommand(self, options);
+  relayEvents(changeStreamCursor, self, ['resumeTokenChanged', 'end', 'close']);
 
   /**
    * Fired for each new matching change in the specified namespace. Attaching a `change`
@@ -243,9 +370,6 @@ var createChangeStreamCursor = function(self) {
    * @event ChangeStream#close
    * @type {null}
    */
-  changeStreamCursor.on('close', function() {
-    self.emit('close');
-  });
 
   /**
    * Change stream end event
@@ -253,9 +377,13 @@ var createChangeStreamCursor = function(self) {
    * @event ChangeStream#end
    * @type {null}
    */
-  changeStreamCursor.on('end', function() {
-    self.emit('end');
-  });
+
+  /**
+   * Emitted each time the change stream stores a new resume token.
+   *
+   * @event ChangeStream#resumeTokenChanged
+   * @type {ResumeToken}
+   */
 
   /**
    * Fired when the stream encounters an error.
@@ -277,44 +405,26 @@ var createChangeStreamCursor = function(self) {
   return changeStreamCursor;
 };
 
-function getResumeToken(self) {
-  return self.resumeToken || self.options.resumeAfter;
-}
-
-function getStartAtOperationTime(self) {
-  const isMaster = self.topology.lastIsMaster() || {};
-  return (
-    isMaster.maxWireVersion && isMaster.maxWireVersion >= 7 && self.options.startAtOperationTime
-  );
+function applyKnownOptions(target, source, optionNames) {
+  optionNames.forEach(name => {
+    if (source[name]) {
+      target[name] = source[name];
+    }
+  });
 }
 
-var buildChangeStreamAggregationCommand = function(self) {
+var buildChangeStreamAggregationCommand = function(self, options) {
+  options = options || {};
   const topology = self.topology;
   const namespace = self.namespace;
   const pipeline = self.pipeline;
-  const options = self.options;
 
-  var changeStreamStageOptions = {
-    fullDocument: options.fullDocument || 'default'
-  };
-
-  const resumeToken = getResumeToken(self);
-  const startAtOperationTime = getStartAtOperationTime(self);
-  if (resumeToken) {
-    changeStreamStageOptions.resumeAfter = resumeToken;
-  }
-
-  if (startAtOperationTime) {
-    changeStreamStageOptions.startAtOperationTime = startAtOperationTime;
-  }
+  const changeStreamStageOptions = { fullDocument: options.fullDocument || 'default' };
+  applyKnownOptions(changeStreamStageOptions, options, CHANGE_STREAM_OPTIONS);
 
   // Map cursor options
-  var cursorOptions = {};
-  cursorOptionNames.forEach(function(optionName) {
-    if (options[optionName]) {
-      cursorOptions[optionName] = options[optionName];
-    }
-  });
+  const cursorOptions = { cursorFactory: ChangeStreamCursor };
+  applyKnownOptions(cursorOptions, options, CURSOR_OPTIONS);
 
   if (self.type === CHANGE_DOMAIN_TYPES.CLUSTER) {
     changeStreamStageOptions.allChangesForCluster = true;
@@ -377,6 +487,7 @@ function processNewChange(args) {
       : changeStream.promiseLibrary.reject(error);
   }
 
+  const cursor = changeStream.cursor;
   const topology = changeStream.topology;
   const options = changeStream.cursor.options;
 
@@ -384,11 +495,6 @@ function processNewChange(args) {
     if (isResumableError(error) && !changeStream.attemptingResume) {
       changeStream.attemptingResume = true;
 
-      if (!(getResumeToken(changeStream) || getStartAtOperationTime(changeStream))) {
-        const startAtOperationTime = changeStream.cursor.cursorState.operationTime;
-        changeStream.options = Object.assign({ startAtOperationTime }, changeStream.options);
-      }
-
       // stop listening to all events from old cursor
       ['data', 'close', 'end', 'error'].forEach(event =>
         changeStream.cursor.removeAllListeners(event)
@@ -401,7 +507,7 @@ function processNewChange(args) {
       if (eventEmitter) {
         waitForTopologyConnected(topology, { readPreference: options.readPreference }, err => {
           if (err) return changeStream.emit('error', err);
-          changeStream.cursor = createChangeStreamCursor(changeStream);
+          changeStream.cursor = createChangeStreamCursor(changeStream, cursor.resumeOptions);
         });
 
         return;
@@ -411,7 +517,7 @@ function processNewChange(args) {
         waitForTopologyConnected(topology, { readPreference: options.readPreference }, err => {
           if (err) return callback(err, null);
 
-          changeStream.cursor = createChangeStreamCursor(changeStream);
+          changeStream.cursor = createChangeStreamCursor(changeStream, cursor.resumeOptions);
           changeStream.next(callback);
         });
 
@@ -424,7 +530,9 @@ function processNewChange(args) {
           resolve();
         });
       })
-        .then(() => (changeStream.cursor = createChangeStreamCursor(changeStream)))
+        .then(
+          () => (changeStream.cursor = createChangeStreamCursor(changeStream, cursor.resumeOptions))
+        )
         .then(() => changeStream.next());
     }
 
@@ -435,9 +543,8 @@ function processNewChange(args) {
 
   changeStream.attemptingResume = false;
 
-  // Cache the resume token if it is present. If it is not present return an error.
-  if (!change || !change._id) {
-    var noResumeTokenError = new Error(
+  if (change && !change._id) {
+    const noResumeTokenError = new Error(
       'A change stream document has been received that lacks a resume token (_id).'
     );
 
@@ -446,7 +553,12 @@ function processNewChange(args) {
     return changeStream.promiseLibrary.reject(noResumeTokenError);
   }
 
-  changeStream.resumeToken = change._id;
+  // cache the resume token
+  if (cursor.bufferedCount() === 0 && cursor.cursorState.postBatchResumeToken) {
+    cursor.resumeToken = cursor.cursorState.postBatchResumeToken;
+  } else {
+    cursor.resumeToken = change._id;
+  }
 
   // Return the change
   if (eventEmitter) return changeStream.emit('change', change);

package/lib/collection.js

@@ -526,6 +526,8 @@ Collection.prototype.insertMany = function(docs, options, callback) {
  *
  *  { updateMany: { filter: {a:2}, update: {$set: {a:2}}, upsert:true } }
  *
+ *  { updateMany: { filter: {}, update: {$set: {"a.$[i].x": 5}}, arrayFilters: [{ "i.x": 5 }]} }
+ *
  *  { deleteOne: { filter: {c:1} } }
  *
  *  { deleteMany: { filter: {c:1} } }
@@ -539,6 +541,7 @@ Collection.prototype.insertMany = function(docs, options, callback) {
  * @method
  * @param {object[]} operations Bulk operations to perform.
  * @param {object} [options] Optional settings.
+ * @param {object[]} [options.arrayFilters] Determines which array elements to modify for update operation in MongoDB 3.6 or higher.
  * @param {(number|string)} [options.w] The write concern.
  * @param {number} [options.wtimeout] The write concern timeout.
  * @param {boolean} [options.j=false] Specify a journal write concern.
@@ -686,15 +689,7 @@ Collection.prototype.updateOne = function(filter, update, options, callback) {
   if (typeof options === 'function') (callback = options), (options = {});
   options = options || {};
 
-  let err;
-  if (Array.isArray(update)) {
-    for (let i = 0; !err && i < update.length; i++) {
-      err = checkForAtomicOperators(update[i]);
-    }
-  } else {
-    err = checkForAtomicOperators(update);
-  }
-
+  const err = checkForAtomicOperators(update);
   if (err) {
     if (typeof callback === 'function') return callback(err);
     return this.s.promiseLibrary.reject(err);
@@ -1056,6 +1051,10 @@ Collection.prototype.rename = function(newName, options, callback) {
  *
  * @method
  * @param {object} [options] Optional settings.
+ * @param {WriteConcern} [options.writeConcern] A full WriteConcern object
+ * @param {(number|string)} [options.w] The write concern
+ * @param {number} [options.wtimeout] The write concern timeout
+ * @param {boolean} [options.j] The journal write concern
  * @param {ClientSession} [options.session] optional session to use for this operation
  * @param {Collection~resultCallback} [callback] The results callback
  * @return {Promise} returns Promise if no callback passed

package/lib/command_cursor.js

@@ -55,10 +55,12 @@ const MongoDBNamespace = require('./utils').MongoDBNamespace;
  * @fires CommandCursor#readable
  * @return {CommandCursor} an CommandCursor instance.
  */
-var CommandCursor = function(bson, ns, cmd, options, topology, topologyOptions) {
+var CommandCursor = function(topology, ns, cmd, options) {
   CoreCursor.apply(this, Array.prototype.slice.call(arguments, 0));
   var state = CommandCursor.INIT;
   var streamOptions = {};
+  const bson = topology.s.bson;
+  const topologyOptions = topology.s.options;
 
   // MaxTimeMS
   var maxTimeMS = null;
@@ -150,14 +152,14 @@ var methodsToInherit = [
   'kill',
   'setCursorBatchSize',
   '_find',
-  '_getmore',
+  '_initializeCursor',
+  '_getMore',
   '_killcursor',
   'isDead',
   'explain',
   'isNotified',
   'isKilled',
-  '_endSession',
-  '_initImplicitSession'
+  '_endSession'
 ];
 
 // Only inherit the types we need

package/lib/core/connection/connect.js

@@ -278,7 +278,7 @@ function makeConnection(family, options, _callback) {
   socket.setTimeout(connectionTimeout);
   socket.setNoDelay(noDelay);
 
-  const errorEvents = ['error', 'close', 'timeout', 'parseError', 'connect'];
+  const errorEvents = ['error', 'close', 'timeout', 'parseError'];
   function errorHandler(eventName) {
     return err => {
       errorEvents.forEach(event => socket.removeAllListeners(event));

package/lib/core/connection/pool.js

@@ -19,6 +19,7 @@ const apm = require('./apm');
 const Buffer = require('safe-buffer').Buffer;
 const connect = require('./connect');
 const updateSessionFromResponse = require('../sessions').updateSessionFromResponse;
+const eachAsync = require('../utils').eachAsync;
 
 var DISCONNECTED = 'disconnected';
 var CONNECTING = 'connecting';
@@ -26,6 +27,15 @@ var CONNECTED = 'connected';
 var DESTROYING = 'destroying';
 var DESTROYED = 'destroyed';
 
+const CONNECTION_EVENTS = new Set([
+  'error',
+  'close',
+  'timeout',
+  'parseError',
+  'connect',
+  'message'
+]);
+
 var _id = 0;
 
 /**
@@ -198,6 +208,19 @@ Object.defineProperty(Pool.prototype, 'socketTimeout', {
   }
 });
 
+// clears all pool state
+function resetPoolState(pool) {
+  pool.inUseConnections = [];
+  pool.availableConnections = [];
+  pool.connectingConnections = 0;
+  pool.executing = false;
+  pool.reconnectConnection = null;
+  pool.numberOfConsecutiveTimeouts = 0;
+  pool.connectionIndex = 0;
+  pool.retriesLeft = pool.options.reconnectTries;
+  pool.reconnectId = null;
+}
+
 function stateTransition(self, newState) {
   var legalTransitions = {
     disconnected: [CONNECTING, DESTROYING, DISCONNECTED],
@@ -631,43 +654,30 @@ Pool.prototype.unref = function() {
   });
 };
 
-// Events
-var events = ['error', 'close', 'timeout', 'parseError', 'connect', 'message'];
-
 // Destroy the connections
 function destroy(self, connections, options, callback) {
-  let connectionCount = connections.length;
-  function connectionDestroyed() {
-    connectionCount--;
-    if (connectionCount > 0) {
-      return;
-    }
-
-    // Zero out all connections
-    self.inUseConnections = [];
-    self.availableConnections = [];
-    self.connectingConnections = 0;
+  eachAsync(
+    connections,
+    (conn, cb) => {
+      for (const eventName of CONNECTION_EVENTS) {
+        conn.removeAllListeners(eventName);
+      }
 
-    // Set state to destroyed
-    stateTransition(self, DESTROYED);
-    if (typeof callback === 'function') {
-      callback(null, null);
-    }
-  }
+      conn.destroy(options, cb);
+    },
+    err => {
+      if (err) {
+        if (typeof callback === 'function') callback(err, null);
+        return;
+      }
 
-  if (connectionCount === 0) {
-    connectionDestroyed();
-    return;
-  }
+      resetPoolState(self);
+      self.queue = [];
 
-  // Destroy all connections
-  connections.forEach(conn => {
-    for (var i = 0; i < events.length; i++) {
-      conn.removeAllListeners(events[i]);
+      stateTransition(self, DESTROYED);
+      if (typeof callback === 'function') callback(null, null);
     }
-
-    conn.destroy(options, connectionDestroyed);
-  });
+  );
 }
 
 /**
@@ -751,19 +761,34 @@ Pool.prototype.destroy = function(force, callback) {
  * @param {function} [callback]
  */
 Pool.prototype.reset = function(callback) {
-  // this.destroy(true, err => {
-  //   if (err && typeof callback === 'function') {
-  //     callback(err, null);
-  //     return;
-  //   }
+  const connections = this.availableConnections.concat(this.inUseConnections);
+  eachAsync(
+    connections,
+    (conn, cb) => {
+      for (const eventName of CONNECTION_EVENTS) {
+        conn.removeAllListeners(eventName);
+      }
+
+      conn.destroy({ force: true }, cb);
+    },
+    err => {
+      if (err) {
+        if (typeof callback === 'function') {
+          callback(err, null);
+          return;
+        }
+      }
 
-  //   stateTransition(this, DISCONNECTED);
-  //   this.connect();
+      resetPoolState(this);
 
-  //   if (typeof callback === 'function') callback(null, null);
-  // });
+      // create an initial connection, and kick off execution again
+      _createConnection(this);
 
-  if (typeof callback === 'function') callback();
+      if (typeof callback === 'function') {
+        callback(null, null);
+      }
+    }
+  );
 };
 
 // Prepare the buffer that Pool.prototype.write() uses to send to the server