mongodb 3.5.7 → 3.5.11

package/HISTORY.md

@@ -2,6 +2,50 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+<a name="3.5.11"></a>
+## [3.5.11](https://github.com/mongodb/node-mongodb-native/compare/v3.5.10...v3.5.11) (2020-09-10)
+
+
+### Bug Fixes
+
+* add host/port to cmap connection ([07b2b0d](https://github.com/mongodb/node-mongodb-native/commit/07b2b0d))
+
+
+
+<a name="3.5.9"></a>
+## [3.5.9](https://github.com/mongodb/node-mongodb-native/compare/v3.5.8...v3.5.9) (2020-06-12)
+
+
+### Bug Fixes
+
+* don't try to calculate sMax if there are no viable servers ([be51347](https://github.com/mongodb/node-mongodb-native/commit/be51347))
+* use async interruptable interval for server monitoring ([1f855a4](https://github.com/mongodb/node-mongodb-native/commit/1f855a4))
+* use duration of handshake if no previous roundTripTime exists ([ddfa41b](https://github.com/mongodb/node-mongodb-native/commit/ddfa41b))
+
+
+### Features
+
+* introduce an interruptable async interval timer ([9e12cd5](https://github.com/mongodb/node-mongodb-native/commit/9e12cd5))
+
+
+
+<a name="3.5.8"></a>
+## [3.5.8](https://github.com/mongodb/node-mongodb-native/compare/v3.5.7...v3.5.8) (2020-05-28)
+
+
+### Bug Fixes
+
+* always clear cancelled wait queue members during processing ([0394f9d](https://github.com/mongodb/node-mongodb-native/commit/0394f9d))
+* always include `writeErrors` on a `BulkWriteError` instance ([58b4f94](https://github.com/mongodb/node-mongodb-native/commit/58b4f94))
+* ensure implicit sessions are ended consistently ([5c6fda1](https://github.com/mongodb/node-mongodb-native/commit/5c6fda1))
+* filter servers before applying reducers ([4faf9f5](https://github.com/mongodb/node-mongodb-native/commit/4faf9f5))
+* unordered bulk write should attempt to execute all batches ([6cee96b](https://github.com/mongodb/node-mongodb-native/commit/6cee96b))
+* **ChangeStream:** should resume from errors when iterating ([5ecf18e](https://github.com/mongodb/node-mongodb-native/commit/5ecf18e))
+* honor journal=true in connection string ([#2359](https://github.com/mongodb/node-mongodb-native/issues/2359)) ([246669f](https://github.com/mongodb/node-mongodb-native/commit/246669f))
+* **ChangeStream:** whitelist resumable errors ([#2337](https://github.com/mongodb/node-mongodb-native/issues/2337)) ([a9d3965](https://github.com/mongodb/node-mongodb-native/commit/a9d3965)), closes [#17](https://github.com/mongodb/node-mongodb-native/issues/17) [#18](https://github.com/mongodb/node-mongodb-native/issues/18)
+
+
+
 <a name="3.5.7"></a>
 ## [3.5.7](https://github.com/mongodb/node-mongodb-native/compare/v3.5.6...v3.5.7) (2020-04-29)
 

package/lib/bulk/common.js

@@ -1200,19 +1200,10 @@ class BulkOperationBase {
    * @ignore
    * @param {function} callback
    * @param {BulkWriteResult} writeResult
-   * @param {class} self either OrderedBulkOperation or UnorderdBulkOperation
+   * @param {class} self either OrderedBulkOperation or UnorderedBulkOperation
    */
   handleWriteError(callback, writeResult) {
     if (this.s.bulkResult.writeErrors.length > 0) {
-      if (this.s.bulkResult.writeErrors.length === 1) {
-        handleCallback(
-          callback,
-          new BulkWriteError(toError(this.s.bulkResult.writeErrors[0]), writeResult),
-          null
-        );
-        return true;
-      }
-
       const msg = this.s.bulkResult.writeErrors[0].errmsg
         ? this.s.bulkResult.writeErrors[0].errmsg
         : 'write operation failed';
@@ -1230,7 +1221,9 @@ class BulkOperationBase {
         null
       );
       return true;
-    } else if (writeResult.getWriteConcernError()) {
+    }
+
+    if (writeResult.getWriteConcernError()) {
       handleCallback(
         callback,
         new BulkWriteError(toError(writeResult.getWriteConcernError()), writeResult),

package/lib/bulk/unordered.js

@@ -108,6 +108,14 @@ class UnorderedBulkOperation extends BulkOperationBase {
 
     super(topology, collection, options, false);
   }
+
+  handleWriteError(callback, writeResult) {
+    if (this.s.batches.length) {
+      return false;
+    }
+
+    return super.handleWriteError(callback, writeResult);
+  }
 }
 
 /**

package/lib/change_stream.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const Denque = require('denque');
 const EventEmitter = require('events');
 const isResumableError = require('./error').isResumableError;
 const MongoError = require('./core').MongoError;
@@ -7,8 +8,12 @@ const Cursor = require('./cursor');
 const relayEvents = require('./core/utils').relayEvents;
 const maxWireVersion = require('./core/utils').maxWireVersion;
 const maybePromise = require('./utils').maybePromise;
+const now = require('./utils').now;
+const calculateDurationInMs = require('./utils').calculateDurationInMs;
 const AggregateOperation = require('./operations/aggregate');
 
+const kResumeQueue = Symbol('resumeQueue');
+
 const CHANGE_STREAM_OPTIONS = ['resumeAfter', 'startAfter', 'startAtOperationTime', 'fullDocument'];
 const CURSOR_OPTIONS = ['batchSize', 'maxAwaitTimeMS', 'collation', 'readPreference'].concat(
   CHANGE_STREAM_OPTIONS
@@ -91,15 +96,17 @@ class ChangeStream extends EventEmitter {
       this.options.readPreference = parent.s.readPreference;
     }
 
+    this[kResumeQueue] = new Denque();
+
     // Create contained Change Stream cursor
     this.cursor = createChangeStreamCursor(this, options);
 
+    this.closed = false;
+
     // Listen for any `change` listeners being added to ChangeStream
     this.on('newListener', eventName => {
       if (eventName === 'change' && this.cursor && this.listenerCount('change') === 0) {
-        this.cursor.on('data', change =>
-          processNewChange({ changeStream: this, change, eventEmitter: true })
-        );
+        this.cursor.on('data', change => processNewChange(this, change));
       }
     });
 
@@ -128,7 +135,12 @@ class ChangeStream extends EventEmitter {
    * @returns {Promise|void} returns Promise if no callback passed
    */
   hasNext(callback) {
-    return maybePromise(this.parent, callback, cb => this.cursor.hasNext(cb));
+    return maybePromise(this.parent, callback, cb => {
+      getCursor(this, (err, cursor) => {
+        if (err) return cb(err); // failed to resume, raise an error
+        cursor.hasNext(cb);
+      });
+    });
   }
 
   /**
@@ -140,25 +152,28 @@ class ChangeStream extends EventEmitter {
    */
   next(callback) {
     return maybePromise(this.parent, callback, cb => {
-      if (this.isClosed()) {
-        return cb(new Error('Change Stream is not open.'));
-      }
-      this.cursor.next((error, change) => {
-        processNewChange({ changeStream: this, error, change, callback: cb });
+      getCursor(this, (err, cursor) => {
+        if (err) return cb(err); // failed to resume, raise an error
+        cursor.next((error, change) => {
+          if (error) {
+            this[kResumeQueue].push(() => this.next(cb));
+            processError(this, error, cb);
+            return;
+          }
+          processNewChange(this, change, cb);
+        });
       });
     });
   }
 
   /**
-   * Is the cursor closed
+   * Is the change stream closed
    * @method ChangeStream.prototype.isClosed
+   * @param  {boolean} [checkCursor=true] also check if the underlying cursor is closed
    * @return {boolean}
    */
   isClosed() {
-    if (this.cursor) {
-      return this.cursor.isClosed();
-    }
-    return true;
+    return this.closed || (this.cursor && this.cursor.isClosed());
   }
 
   /**
@@ -168,31 +183,22 @@ class ChangeStream extends EventEmitter {
    * @return {Promise} returns Promise if no callback passed
    */
   close(callback) {
-    if (!this.cursor) {
-      if (callback) return callback();
-      return this.promiseLibrary.resolve();
-    }
+    return maybePromise(this.parent, callback, cb => {
+      if (this.closed) return cb();
 
-    // Tidy up the existing cursor
-    const cursor = this.cursor;
+      // flag the change stream as explicitly closed
+      this.closed = true;
 
-    if (callback) {
-      return cursor.close(err => {
-        ['data', 'close', 'end', 'error'].forEach(event => cursor.removeAllListeners(event));
-        delete this.cursor;
+      if (!this.cursor) return cb();
 
-        return callback(err);
-      });
-    }
+      // Tidy up the existing cursor
+      const cursor = this.cursor;
 
-    const PromiseCtor = this.promiseLibrary || Promise;
-    return new PromiseCtor((resolve, reject) => {
-      cursor.close(err => {
+      return cursor.close(err => {
         ['data', 'close', 'end', 'error'].forEach(event => cursor.removeAllListeners(event));
-        delete this.cursor;
+        this.cursor = undefined;
 
-        if (err) return reject(err);
-        resolve();
+        return cb(err);
       });
     });
   }
@@ -287,7 +293,9 @@ class ChangeStreamCursor extends Cursor {
       ['resumeAfter', 'startAfter', 'startAtOperationTime'].forEach(key => delete result[key]);
 
       if (this.resumeToken) {
-        result.resumeAfter = this.resumeToken;
+        const resumeKey =
+          this.options.startAfter && !this.hasReceived ? 'startAfter' : 'resumeAfter';
+        result[resumeKey] = this.resumeToken;
       } else if (this.startAtOperationTime && maxWireVersion(this.server) >= 7) {
         result.startAtOperationTime = this.startAtOperationTime;
       }
@@ -296,10 +304,30 @@ class ChangeStreamCursor extends Cursor {
     return result;
   }
 
+  cacheResumeToken(resumeToken) {
+    if (this.bufferedCount() === 0 && this.cursorState.postBatchResumeToken) {
+      this.resumeToken = this.cursorState.postBatchResumeToken;
+    } else {
+      this.resumeToken = resumeToken;
+    }
+    this.hasReceived = true;
+  }
+
+  _processBatch(batchName, response) {
+    const cursor = response.cursor;
+    if (cursor.postBatchResumeToken) {
+      this.cursorState.postBatchResumeToken = cursor.postBatchResumeToken;
+
+      if (cursor[batchName].length === 0) {
+        this.resumeToken = cursor.postBatchResumeToken;
+      }
+    }
+  }
+
   _initializeCursor(callback) {
     super._initializeCursor((err, result) => {
-      if (err) {
-        callback(err, null);
+      if (err || result == null) {
+        callback(err, result);
         return;
       }
 
@@ -314,15 +342,9 @@ class ChangeStreamCursor extends Cursor {
         this.startAtOperationTime = response.operationTime;
       }
 
-      const cursor = response.cursor;
-      if (cursor.postBatchResumeToken) {
-        this.cursorState.postBatchResumeToken = cursor.postBatchResumeToken;
-
-        if (cursor.firstBatch.length === 0) {
-          this.resumeToken = cursor.postBatchResumeToken;
-        }
-      }
+      this._processBatch('firstBatch', response);
 
+      this.emit('init', result);
       this.emit('response');
       callback(err, result);
     });
@@ -331,19 +353,13 @@ class ChangeStreamCursor extends Cursor {
   _getMore(callback) {
     super._getMore((err, response) => {
       if (err) {
-        callback(err, null);
+        callback(err);
         return;
       }
 
-      const cursor = response.cursor;
-      if (cursor.postBatchResumeToken) {
-        this.cursorState.postBatchResumeToken = cursor.postBatchResumeToken;
-
-        if (cursor.nextBatch.length === 0) {
-          this.resumeToken = cursor.postBatchResumeToken;
-        }
-      }
+      this._processBatch('nextBatch', response);
 
+      this.emit('more', response);
       this.emit('response');
       callback(err, response);
     });
@@ -366,6 +382,7 @@ function createChangeStreamCursor(self, options) {
 
   const pipeline = [{ $changeStream: changeStreamStageOptions }].concat(self.pipeline);
   const cursorOptions = applyKnownOptions({}, options, CURSOR_OPTIONS);
+
   const changeStreamCursor = new ChangeStreamCursor(
     self.topology,
     new AggregateOperation(self.parent, pipeline, options),
@@ -384,7 +401,7 @@ function createChangeStreamCursor(self, options) {
    */
   if (self.listenerCount('change') > 0) {
     changeStreamCursor.on('data', function(change) {
-      processNewChange({ changeStream: self, change, eventEmitter: true });
+      processNewChange(self, change);
     });
   }
 
@@ -416,7 +433,7 @@ function createChangeStreamCursor(self, options) {
    * @type {Error}
    */
   changeStreamCursor.on('error', function(error) {
-    processNewChange({ changeStream: self, error, eventEmitter: true });
+    processError(self, error);
   });
 
   if (self.pipeDestinations) {
@@ -444,125 +461,157 @@ function applyKnownOptions(target, source, optionNames) {
 const SELECTION_TIMEOUT = 30000;
 function waitForTopologyConnected(topology, options, callback) {
   setTimeout(() => {
-    if (options && options.start == null) options.start = process.hrtime();
-    const start = options.start || process.hrtime();
+    if (options && options.start == null) {
+      options.start = now();
+    }
+
+    const start = options.start || now();
     const timeout = options.timeout || SELECTION_TIMEOUT;
     const readPreference = options.readPreference;
+    if (topology.isConnected({ readPreference })) {
+      return callback();
+    }
+
+    if (calculateDurationInMs(start) > timeout) {
+      return callback(new MongoError('Timed out waiting for connection'));
+    }
 
-    if (topology.isConnected({ readPreference })) return callback(null, null);
-    const hrElapsed = process.hrtime(start);
-    const elapsed = (hrElapsed[0] * 1e9 + hrElapsed[1]) / 1e6;
-    if (elapsed > timeout) return callback(new MongoError('Timed out waiting for connection'));
     waitForTopologyConnected(topology, options, callback);
-  }, 3000); // this is an arbitrary wait time to allow SDAM to transition
+  }, 500); // this is an arbitrary wait time to allow SDAM to transition
 }
 
-// Handle new change events. This method brings together the routes from the callback, event emitter, and promise ways of using ChangeStream.
-function processNewChange(args) {
-  const changeStream = args.changeStream;
-  const error = args.error;
-  const change = args.change;
-  const callback = args.callback;
-  const eventEmitter = args.eventEmitter || false;
+function processNewChange(changeStream, change, callback) {
+  const cursor = changeStream.cursor;
 
-  // If the changeStream is closed, then it should not process a change.
-  if (changeStream.isClosed()) {
-    // We do not error in the eventEmitter case.
-    if (eventEmitter) {
-      return;
-    }
+  // a null change means the cursor has been notified, implicitly closing the change stream
+  if (change == null) {
+    changeStream.closed = true;
+  }
 
-    const error = new MongoError('ChangeStream is closed');
-    return typeof callback === 'function'
-      ? callback(error, null)
-      : changeStream.promiseLibrary.reject(error);
+  if (changeStream.closed) {
+    if (callback) callback(new MongoError('ChangeStream is closed'));
+    return;
   }
 
-  const cursor = changeStream.cursor;
+  if (change && !change._id) {
+    const noResumeTokenError = new Error(
+      'A change stream document has been received that lacks a resume token (_id).'
+    );
+
+    if (!callback) return changeStream.emit('error', noResumeTokenError);
+    return callback(noResumeTokenError);
+  }
+
+  // cache the resume token
+  cursor.cacheResumeToken(change._id);
+
+  // wipe the startAtOperationTime if there was one so that there won't be a conflict
+  // between resumeToken and startAtOperationTime if we need to reconnect the cursor
+  changeStream.options.startAtOperationTime = undefined;
+
+  // Return the change
+  if (!callback) return changeStream.emit('change', change);
+  return callback(undefined, change);
+}
+
+function processError(changeStream, error, callback) {
   const topology = changeStream.topology;
-  const options = changeStream.cursor.options;
+  const cursor = changeStream.cursor;
 
-  if (error) {
-    if (isResumableError(error) && !changeStream.attemptingResume) {
-      changeStream.attemptingResume = true;
+  // If the change stream has been closed explictly, do not process error.
+  if (changeStream.closed) {
+    if (callback) callback(new MongoError('ChangeStream is closed'));
+    return;
+  }
 
-      // stop listening to all events from old cursor
-      ['data', 'close', 'end', 'error'].forEach(event =>
-        changeStream.cursor.removeAllListeners(event)
-      );
+  // if the resume succeeds, continue with the new cursor
+  function resumeWithCursor(newCursor) {
+    changeStream.cursor = newCursor;
+    processResumeQueue(changeStream);
+  }
 
-      // close internal cursor, ignore errors
-      changeStream.cursor.close();
+  // otherwise, raise an error and close the change stream
+  function unresumableError(err) {
+    if (!callback) {
+      changeStream.emit('error', err);
+      changeStream.emit('close');
+    }
+    processResumeQueue(changeStream, err);
+    changeStream.closed = true;
+  }
 
-      // attempt recreating the cursor
-      if (eventEmitter) {
-        waitForTopologyConnected(topology, { readPreference: options.readPreference }, err => {
-          if (err) {
-            changeStream.emit('error', err);
-            changeStream.emit('close');
-            return;
-          }
-          changeStream.cursor = createChangeStreamCursor(changeStream, cursor.resumeOptions);
-        });
+  if (cursor && isResumableError(error, maxWireVersion(cursor.server))) {
+    changeStream.cursor = undefined;
 
-        return;
-      }
+    // stop listening to all events from old cursor
+    ['data', 'close', 'end', 'error'].forEach(event => cursor.removeAllListeners(event));
 
-      if (callback) {
-        waitForTopologyConnected(topology, { readPreference: options.readPreference }, err => {
-          if (err) return callback(err, null);
+    // close internal cursor, ignore errors
+    cursor.close();
 
-          changeStream.cursor = createChangeStreamCursor(changeStream, cursor.resumeOptions);
-          changeStream.next(callback);
-        });
+    waitForTopologyConnected(topology, { readPreference: cursor.options.readPreference }, err => {
+      // if the topology can't reconnect, close the stream
+      if (err) return unresumableError(err);
 
-        return;
-      }
+      // create a new cursor, preserving the old cursor's options
+      const newCursor = createChangeStreamCursor(changeStream, cursor.resumeOptions);
 
-      return new Promise((resolve, reject) => {
-        waitForTopologyConnected(topology, { readPreference: options.readPreference }, err => {
-          if (err) return reject(err);
-          resolve();
-        });
-      })
-        .then(
-          () => (changeStream.cursor = createChangeStreamCursor(changeStream, cursor.resumeOptions))
-        )
-        .then(() => changeStream.next());
-    }
+      // attempt to continue in emitter mode
+      if (!callback) return resumeWithCursor(newCursor);
 
-    if (eventEmitter) return changeStream.emit('error', error);
-    if (typeof callback === 'function') return callback(error, null);
-    return changeStream.promiseLibrary.reject(error);
+      // attempt to continue in iterator mode
+      newCursor.hasNext(err => {
+        // if there's an error immediately after resuming, close the stream
+        if (err) return unresumableError(err);
+        resumeWithCursor(newCursor);
+      });
+    });
+    return;
   }
 
-  changeStream.attemptingResume = false;
-
-  if (change && !change._id) {
-    const noResumeTokenError = new Error(
-      'A change stream document has been received that lacks a resume token (_id).'
-    );
+  if (!callback) return changeStream.emit('error', error);
+  return callback(error);
+}
 
-    if (eventEmitter) return changeStream.emit('error', noResumeTokenError);
-    if (typeof callback === 'function') return callback(noResumeTokenError, null);
-    return changeStream.promiseLibrary.reject(noResumeTokenError);
+/**
+ * Safely provides a cursor across resume attempts
+ *
+ * @param {ChangeStream} changeStream the parent ChangeStream
+ * @param {function} callback gets the cursor or error
+ * @param {ChangeStreamCursor} [oldCursor] when resuming from an error, carry over options from previous cursor
+ */
+function getCursor(changeStream, callback) {
+  if (changeStream.isClosed()) {
+    callback(new MongoError('ChangeStream is closed.'));
+    return;
   }
 
-  // cache the resume token
-  if (cursor.bufferedCount() === 0 && cursor.cursorState.postBatchResumeToken) {
-    cursor.resumeToken = cursor.cursorState.postBatchResumeToken;
-  } else {
-    cursor.resumeToken = change._id;
+  // if a cursor exists and it is open, return it
+  if (changeStream.cursor) {
+    callback(undefined, changeStream.cursor);
+    return;
   }
 
-  // wipe the startAtOperationTime if there was one so that there won't be a conflict
-  // between resumeToken and startAtOperationTime if we need to reconnect the cursor
-  changeStream.options.startAtOperationTime = undefined;
+  // no cursor, queue callback until topology reconnects
+  changeStream[kResumeQueue].push(callback);
+}
 
-  // Return the change
-  if (eventEmitter) return changeStream.emit('change', change);
-  if (typeof callback === 'function') return callback(error, change);
-  return changeStream.promiseLibrary.resolve(change);
+/**
+ * Drain the resume queue when a new has become available
+ *
+ * @param {ChangeStream} changeStream the parent ChangeStream
+ * @param {ChangeStreamCursor?} changeStream.cursor the new cursor
+ * @param {Error} [err] error getting a new cursor
+ */
+function processResumeQueue(changeStream, err) {
+  while (changeStream[kResumeQueue].length) {
+    const request = changeStream[kResumeQueue].pop();
+    if (changeStream.isClosed() && !err) {
+      request(new MongoError('Change Stream is not open.'));
+      return;
+    }
+    request(err, changeStream.cursor);
+  }
 }
 
 /**

package/lib/cmap/connection.js

@@ -11,6 +11,8 @@ const wp = require('../core/wireprotocol');
 const apm = require('../core/connection/apm');
 const updateSessionFromResponse = require('../core/sessions').updateSessionFromResponse;
 const uuidV4 = require('../core/utils').uuidV4;
+const now = require('../utils').now;
+const calculateDurationInMs = require('../utils').calculateDurationInMs;
 
 const kStream = Symbol('stream');
 const kQueue = Symbol('queue');
@@ -30,6 +32,8 @@ class Connection extends EventEmitter {
     this.address = streamIdentifier(stream);
     this.bson = options.bson;
     this.socketTimeout = typeof options.socketTimeout === 'number' ? options.socketTimeout : 360000;
+    this.host = options.host || 'localhost';
+    this.port = options.port || 27017;
     this.monitorCommands =
       typeof options.monitorCommands === 'boolean' ? options.monitorCommands : false;
     this.closed = false;
@@ -37,7 +41,7 @@ class Connection extends EventEmitter {
 
     this[kDescription] = new StreamDescription(this.address, options);
     this[kGeneration] = options.generation;
-    this[kLastUseTime] = Date.now();
+    this[kLastUseTime] = now();
 
     // retain a reference to an `AutoEncrypter` if present
     if (options.autoEncrypter) {
@@ -108,7 +112,7 @@ class Connection extends EventEmitter {
   }
 
   get idleTime() {
-    return Date.now() - this[kLastUseTime];
+    return calculateDurationInMs(this[kLastUseTime]);
   }
 
   get clusterTime() {
@@ -120,7 +124,7 @@ class Connection extends EventEmitter {
   }
 
   markAvailable() {
-    this[kLastUseTime] = Date.now();
+    this[kLastUseTime] = now();
   }
 
   destroy(options, callback) {
@@ -326,7 +330,7 @@ function write(command, options, callback) {
   if (this.monitorCommands) {
     this.emit('commandStarted', new apm.CommandStartedEvent(this, command));
 
-    operationDescription.started = process.hrtime();
+    operationDescription.started = now();
     operationDescription.cb = (err, reply) => {
       if (err) {
         this.emit(