mongodb 6.2.0 → 6.3.0

package/src/cmap/connection.ts

@@ -1,3 +1,5 @@
+import { once } from 'events';
+import { on } from 'stream';
 import { clearTimeout, setTimeout } from 'timers';
 import { promisify } from 'util';
 
@@ -18,6 +20,7 @@ import {
   MongoMissingDependencyError,
   MongoNetworkError,
   MongoNetworkTimeoutError,
+  MongoParseError,
   MongoRuntimeError,
   MongoServerError,
   MongoWriteConcernError
@@ -27,6 +30,7 @@ import { type CancellationToken, TypedEventEmitter } from '../mongo_types';
 import type { ReadPreferenceLike } from '../read_preference';
 import { applySession, type ClientSession, updateSessionFromResponse } from '../sessions';
 import {
+  BufferPool,
   calculateDurationInMs,
   type Callback,
   HostAddress,
@@ -43,11 +47,19 @@ import {
   CommandStartedEvent,
   CommandSucceededEvent
 } from './command_monitoring_events';
-import { type BinMsg, Msg, Query, type Response, type WriteProtocolMessageType } from './commands';
+import {
+  OpCompressedRequest,
+  OpMsgRequest,
+  type OpMsgResponse,
+  OpQueryRequest,
+  type OpQueryResponse,
+  type WriteProtocolMessageType
+} from './commands';
 import type { Stream } from './connect';
 import type { ClientMetadata } from './handshake/client_metadata';
 import { MessageStream, type OperationDescription } from './message_stream';
 import { StreamDescription, type StreamDescriptionOptions } from './stream_description';
+import { decompressResponse } from './wire_protocol/compression';
 import { getReadPreference, isSharded } from './wire_protocol/shared';
 
 /** @internal */
@@ -324,7 +336,7 @@ export class Connection extends TypedEventEmitter<ConnectionEvents> {
     }, 1).unref(); // No need for this timer to hold the event loop open
   }
 
-  onMessage(message: BinMsg | Response) {
+  onMessage(message: OpMsgResponse | OpQueryResponse) {
     const delayedTimeoutId = this[kDelayedTimeoutId];
     if (delayedTimeoutId != null) {
       clearTimeout(delayedTimeoutId);
@@ -540,8 +552,8 @@ export class Connection extends TypedEventEmitter<ConnectionEvents> {
     );
 
     const message = shouldUseOpMsg
-      ? new Msg(ns.db, cmd, commandOptions)
-      : new Query(ns.db, cmd, commandOptions);
+      ? new OpMsgRequest(ns.db, cmd, commandOptions)
+      : new OpQueryRequest(ns.db, cmd, commandOptions);
 
     try {
       write(this, message, commandOptions, callback);
@@ -757,3 +769,530 @@ function write(
     operationDescription.cb();
   }
 }
+
+/** in-progress connection layer */
+
+/** @internal */
+export class ModernConnection extends TypedEventEmitter<ConnectionEvents> {
+  id: number | '<monitor>';
+  address: string;
+  socketTimeoutMS: number;
+  monitorCommands: boolean;
+  /** Indicates that the connection (including underlying TCP socket) has been closed. */
+  closed: boolean;
+  lastHelloMS?: number;
+  serverApi?: ServerApi;
+  helloOk?: boolean;
+  commandAsync: (
+    ns: MongoDBNamespace,
+    cmd: Document,
+    options: CommandOptions | undefined
+  ) => Promise<Document>;
+  /** @internal */
+  authContext?: AuthContext;
+
+  /**@internal */
+  [kDelayedTimeoutId]: NodeJS.Timeout | null;
+  /** @internal */
+  [kDescription]: StreamDescription;
+  /** @internal */
+  [kGeneration]: number;
+  /** @internal */
+  [kLastUseTime]: number;
+  /** @internal */
+  [kQueue]: Map<number, OperationDescription>;
+  /** @internal */
+  [kMessageStream]: MessageStream;
+  /** @internal */
+  socket: Stream;
+  /** @internal */
+  [kHello]: Document | null;
+  /** @internal */
+  [kClusterTime]: Document | null;
+
+  /** @event */
+  static readonly COMMAND_STARTED = COMMAND_STARTED;
+  /** @event */
+  static readonly COMMAND_SUCCEEDED = COMMAND_SUCCEEDED;
+  /** @event */
+  static readonly COMMAND_FAILED = COMMAND_FAILED;
+  /** @event */
+  static readonly CLUSTER_TIME_RECEIVED = CLUSTER_TIME_RECEIVED;
+  /** @event */
+  static readonly CLOSE = CLOSE;
+  /** @event */
+  static readonly MESSAGE = MESSAGE;
+  /** @event */
+  static readonly PINNED = PINNED;
+  /** @event */
+  static readonly UNPINNED = UNPINNED;
+
+  constructor(stream: Stream, options: ConnectionOptions) {
+    super();
+
+    this.commandAsync = promisify(
+      (
+        ns: MongoDBNamespace,
+        cmd: Document,
+        options: CommandOptions | undefined,
+        callback: Callback
+      ) => this.command(ns, cmd, options, callback as any)
+    );
+
+    this.id = options.id;
+    this.address = streamIdentifier(stream, options);
+    this.socketTimeoutMS = options.socketTimeoutMS ?? 0;
+    this.monitorCommands = options.monitorCommands;
+    this.serverApi = options.serverApi;
+    this.closed = false;
+    this[kHello] = null;
+    this[kClusterTime] = null;
+
+    this[kDescription] = new StreamDescription(this.address, options);
+    this[kGeneration] = options.generation;
+    this[kLastUseTime] = now();
+
+    // setup parser stream and message handling
+    this[kQueue] = new Map();
+    this[kMessageStream] = new MessageStream({
+      ...options,
+      maxBsonMessageSize: this.hello?.maxBsonMessageSize
+    });
+    this.socket = stream;
+
+    this[kDelayedTimeoutId] = null;
+
+    this[kMessageStream].on('message', message => this.onMessage(message));
+    this[kMessageStream].on('error', error => this.onError(error));
+    this.socket.on('close', () => this.onClose());
+    this.socket.on('timeout', () => this.onTimeout());
+    this.socket.on('error', () => {
+      /* ignore errors, listen to `close` instead */
+    });
+
+    // hook the message stream up to the passed in stream
+    this.socket.pipe(this[kMessageStream]);
+    this[kMessageStream].pipe(this.socket);
+  }
+
+  get description(): StreamDescription {
+    return this[kDescription];
+  }
+
+  get hello(): Document | null {
+    return this[kHello];
+  }
+
+  // the `connect` method stores the result of the handshake hello on the connection
+  set hello(response: Document | null) {
+    this[kDescription].receiveResponse(response);
+    this[kDescription] = Object.freeze(this[kDescription]);
+
+    // TODO: remove this, and only use the `StreamDescription` in the future
+    this[kHello] = response;
+  }
+
+  // Set the whether the message stream is for a monitoring connection.
+  set isMonitoringConnection(value: boolean) {
+    this[kMessageStream].isMonitoringConnection = value;
+  }
+
+  get isMonitoringConnection(): boolean {
+    return this[kMessageStream].isMonitoringConnection;
+  }
+
+  get serviceId(): ObjectId | undefined {
+    return this.hello?.serviceId;
+  }
+
+  get loadBalanced(): boolean {
+    return this.description.loadBalanced;
+  }
+
+  get generation(): number {
+    return this[kGeneration] || 0;
+  }
+
+  set generation(generation: number) {
+    this[kGeneration] = generation;
+  }
+
+  get idleTime(): number {
+    return calculateDurationInMs(this[kLastUseTime]);
+  }
+
+  get clusterTime(): Document | null {
+    return this[kClusterTime];
+  }
+
+  get stream(): Stream {
+    return this.socket;
+  }
+
+  get hasSessionSupport(): boolean {
+    return this.description.logicalSessionTimeoutMinutes != null;
+  }
+
+  get supportsOpMsg(): boolean {
+    return (
+      this.description != null &&
+      maxWireVersion(this as any as Connection) >= 6 &&
+      !this.description.__nodejs_mock_server__
+    );
+  }
+
+  markAvailable(): void {
+    this[kLastUseTime] = now();
+  }
+
+  onError(error: Error) {
+    this.cleanup(true, error);
+  }
+
+  onClose() {
+    const message = `connection ${this.id} to ${this.address} closed`;
+    this.cleanup(true, new MongoNetworkError(message));
+  }
+
+  onTimeout() {
+    this[kDelayedTimeoutId] = setTimeout(() => {
+      const message = `connection ${this.id} to ${this.address} timed out`;
+      const beforeHandshake = this.hello == null;
+      this.cleanup(true, new MongoNetworkTimeoutError(message, { beforeHandshake }));
+    }, 1).unref(); // No need for this timer to hold the event loop open
+  }
+
+  onMessage(message: OpMsgResponse | OpQueryResponse) {
+    const delayedTimeoutId = this[kDelayedTimeoutId];
+    if (delayedTimeoutId != null) {
+      clearTimeout(delayedTimeoutId);
+      this[kDelayedTimeoutId] = null;
+    }
+
+    const socketTimeoutMS = this.socket.timeout ?? 0;
+    this.socket.setTimeout(0);
+
+    // always emit the message, in case we are streaming
+    this.emit('message', message);
+    let operationDescription = this[kQueue].get(message.responseTo);
+
+    if (!operationDescription && this.isMonitoringConnection) {
+      // This is how we recover when the initial hello's requestId is not
+      // the responseTo when hello responses have been skipped:
+
+      // First check if the map is of invalid size
+      if (this[kQueue].size > 1) {
+        this.cleanup(true, new MongoRuntimeError(INVALID_QUEUE_SIZE));
+      } else {
+        // Get the first orphaned operation description.
+        const entry = this[kQueue].entries().next();
+        if (entry.value != null) {
+          const [requestId, orphaned]: [number, OperationDescription] = entry.value;
+          // If the orphaned operation description exists then set it.
+          operationDescription = orphaned;
+          // Remove the entry with the bad request id from the queue.
+          this[kQueue].delete(requestId);
+        }
+      }
+    }
+
+    if (!operationDescription) {
+      return;
+    }
+
+    const callback = operationDescription.cb;
+
+    // SERVER-45775: For exhaust responses we should be able to use the same requestId to
+    // track response, however the server currently synthetically produces remote requests
+    // making the `responseTo` change on each response
+    this[kQueue].delete(message.responseTo);
+    if ('moreToCome' in message && message.moreToCome) {
+      // If the operation description check above does find an orphaned
+      // description and sets the operationDescription then this line will put one
+      // back in the queue with the correct requestId and will resolve not being able
+      // to find the next one via the responseTo of the next streaming hello.
+      this[kQueue].set(message.requestId, operationDescription);
+      this.socket.setTimeout(socketTimeoutMS);
+    }
+
+    try {
+      // Pass in the entire description because it has BSON parsing options
+      message.parse(operationDescription);
+    } catch (err) {
+      // If this error is generated by our own code, it will already have the correct class applied
+      // if it is not, then it is coming from a catastrophic data parse failure or the BSON library
+      // in either case, it should not be wrapped
+      callback(err);
+      return;
+    }
+
+    if (message.documents[0]) {
+      const document: Document = message.documents[0];
+      const session = operationDescription.session;
+      if (session) {
+        updateSessionFromResponse(session, document);
+      }
+
+      if (document.$clusterTime) {
+        this[kClusterTime] = document.$clusterTime;
+        this.emit(Connection.CLUSTER_TIME_RECEIVED, document.$clusterTime);
+      }
+
+      if (document.writeConcernError) {
+        callback(new MongoWriteConcernError(document.writeConcernError, document), document);
+        return;
+      }
+
+      if (document.ok === 0 || document.$err || document.errmsg || document.code) {
+        callback(new MongoServerError(document));
+        return;
+      }
+    }
+
+    callback(undefined, message.documents[0]);
+  }
+
+  destroy(options: DestroyOptions, callback?: Callback): void {
+    if (this.closed) {
+      process.nextTick(() => callback?.());
+      return;
+    }
+    if (typeof callback === 'function') {
+      this.once('close', () => process.nextTick(() => callback()));
+    }
+
+    // load balanced mode requires that these listeners remain on the connection
+    // after cleanup on timeouts, errors or close so we remove them before calling
+    // cleanup.
+    this.removeAllListeners(Connection.PINNED);
+    this.removeAllListeners(Connection.UNPINNED);
+    const message = `connection ${this.id} to ${this.address} closed`;
+    this.cleanup(options.force, new MongoNetworkError(message));
+  }
+
+  /**
+   * A method that cleans up the connection.  When `force` is true, this method
+   * forcibly destroys the socket.
+   *
+   * If an error is provided, any in-flight operations will be closed with the error.
+   *
+   * This method does nothing if the connection is already closed.
+   */
+  private cleanup(force: boolean, error?: Error): void {
+    if (this.closed) {
+      return;
+    }
+
+    this.closed = true;
+
+    const completeCleanup = () => {
+      for (const op of this[kQueue].values()) {
+        op.cb(error);
+      }
+
+      this[kQueue].clear();
+
+      this.emit(Connection.CLOSE);
+    };
+
+    this.socket.removeAllListeners();
+    this[kMessageStream].removeAllListeners();
+
+    this[kMessageStream].destroy();
+
+    if (force) {
+      this.socket.destroy();
+      completeCleanup();
+      return;
+    }
+
+    if (!this.socket.writableEnded) {
+      this.socket.end(() => {
+        this.socket.destroy();
+        completeCleanup();
+      });
+    } else {
+      completeCleanup();
+    }
+  }
+
+  command(
+    ns: MongoDBNamespace,
+    command: Document,
+    options: CommandOptions | undefined,
+    callback: Callback
+  ): void {
+    let cmd = { ...command };
+
+    const readPreference = getReadPreference(options);
+    const session = options?.session;
+
+    let clusterTime = this.clusterTime;
+
+    if (this.serverApi) {
+      const { version, strict, deprecationErrors } = this.serverApi;
+      cmd.apiVersion = version;
+      if (strict != null) cmd.apiStrict = strict;
+      if (deprecationErrors != null) cmd.apiDeprecationErrors = deprecationErrors;
+    }
+
+    if (this.hasSessionSupport && session) {
+      if (
+        session.clusterTime &&
+        clusterTime &&
+        session.clusterTime.clusterTime.greaterThan(clusterTime.clusterTime)
+      ) {
+        clusterTime = session.clusterTime;
+      }
+
+      const err = applySession(session, cmd, options);
+      if (err) {
+        return callback(err);
+      }
+    } else if (session?.explicit) {
+      return callback(new MongoCompatibilityError('Current topology does not support sessions'));
+    }
+
+    // if we have a known cluster time, gossip it
+    if (clusterTime) {
+      cmd.$clusterTime = clusterTime;
+    }
+
+    if (
+      // @ts-expect-error ModernConnections cannot be passed as connections
+      isSharded(this) &&
+      !this.supportsOpMsg &&
+      readPreference &&
+      readPreference.mode !== 'primary'
+    ) {
+      cmd = {
+        $query: cmd,
+        $readPreference: readPreference.toJSON()
+      };
+    }
+
+    const commandOptions: Document = Object.assign(
+      {
+        numberToSkip: 0,
+        numberToReturn: -1,
+        checkKeys: false,
+        // This value is not overridable
+        secondaryOk: readPreference.secondaryOk()
+      },
+      options
+    );
+
+    const message = this.supportsOpMsg
+      ? new OpMsgRequest(ns.db, cmd, commandOptions)
+      : new OpQueryRequest(ns.db, cmd, commandOptions);
+
+    try {
+      write(this as any as Connection, message, commandOptions, callback);
+    } catch (err) {
+      callback(err);
+    }
+  }
+}
+
+const kDefaultMaxBsonMessageSize = 1024 * 1024 * 16 * 4;
+
+/**
+ * @internal
+ *
+ * This helper reads chucks of data out of a socket and buffers them until it has received a
+ * full wire protocol message.
+ *
+ * By itself, produces an infinite async generator of wire protocol messages and consumers must end
+ * the stream by calling `return` on the generator.
+ *
+ * Note that `for-await` loops call `return` automatically when the loop is exited.
+ */
+export async function* readWireProtocolMessages(
+  connection: ModernConnection
+): AsyncGenerator<Buffer> {
+  const bufferPool = new BufferPool();
+  const maxBsonMessageSize = connection.hello?.maxBsonMessageSize ?? kDefaultMaxBsonMessageSize;
+  for await (const [chunk] of on(connection.socket, 'data')) {
+    bufferPool.append(chunk);
+    const sizeOfMessage = bufferPool.getInt32();
+
+    if (sizeOfMessage == null) {
+      continue;
+    }
+
+    if (sizeOfMessage < 0) {
+      throw new MongoParseError(`Invalid message size: ${sizeOfMessage}`);
+    }
+
+    if (sizeOfMessage > maxBsonMessageSize) {
+      throw new MongoParseError(
+        `Invalid message size: ${sizeOfMessage}, max allowed: ${maxBsonMessageSize}`
+      );
+    }
+
+    if (sizeOfMessage > bufferPool.length) {
+      continue;
+    }
+
+    yield bufferPool.read(sizeOfMessage);
+  }
+}
+
+/**
+ * @internal
+ *
+ * Writes an OP_MSG or OP_QUERY request to the socket, optionally compressing the command. This method
+ * waits until the socket's buffer has emptied (the Nodejs socket `drain` event has fired).
+ */
+export async function writeCommand(
+  connection: ModernConnection,
+  command: WriteProtocolMessageType,
+  options: Partial<Pick<OperationDescription, 'agreedCompressor' | 'zlibCompressionLevel'>>
+): Promise<void> {
+  const drained = once(connection.socket, 'drain');
+  const finalCommand =
+    options.agreedCompressor === 'none' || !OpCompressedRequest.canCompress(command)
+      ? command
+      : new OpCompressedRequest(command, {
+          agreedCompressor: options.agreedCompressor ?? 'none',
+          zlibCompressionLevel: options.zlibCompressionLevel ?? 0
+        });
+  const buffer = Buffer.concat(await finalCommand.toBin());
+  connection.socket.push(buffer);
+  await drained;
+}
+
+/**
+ * @internal
+ *
+ * Returns an async generator that yields full wire protocol messages from the underlying socket.  This function
+ * yields messages until `moreToCome` is false or not present in a response, or the caller cancels the request
+ * by calling `return` on the generator.
+ *
+ * Note that `for-await` loops call `return` automatically when the loop is exited.
+ */
+export async function* readMany(
+  connection: ModernConnection
+): AsyncGenerator<OpMsgResponse | OpQueryResponse> {
+  for await (const message of readWireProtocolMessages(connection)) {
+    const response = await decompressResponse(message);
+    yield response;
+
+    if (!('moreToCome' in response) || !response.moreToCome) {
+      return;
+    }
+  }
+}
+
+/**
+ * @internal
+ *
+ * Reads a single wire protocol message out of a connection.
+ */
+export async function read(connection: ModernConnection): Promise<OpMsgResponse | OpQueryResponse> {
+  for await (const value of readMany(connection)) {
+    return value;
+  }
+
+  throw new MongoRuntimeError('unable to read message off of connection');
+}

package/src/cmap/errors.ts

@@ -56,7 +56,7 @@ export class PoolClearedError extends MongoNetworkError {
     super(errorMessage, pool.serverError ? { cause: pool.serverError } : undefined);
     this.address = pool.address;
 
-    this.addErrorLabel(MongoErrorLabel.RetryableWriteError);
+    this.addErrorLabel(MongoErrorLabel.PoolRequstedRetry);
   }
 
   override get name(): string {

package/src/cmap/message_stream.ts

@@ -5,19 +5,13 @@ import { MongoDecompressionError, MongoParseError } from '../error';
 import type { ClientSession } from '../sessions';
 import { BufferPool, type Callback } from '../utils';
 import {
-  BinMsg,
   type MessageHeader,
-  Msg,
-  Response,
+  OpCompressedRequest,
+  OpMsgResponse,
+  OpQueryResponse,
   type WriteProtocolMessageType
 } from './commands';
-import {
-  compress,
-  Compressor,
-  type CompressorName,
-  decompress,
-  uncompressibleCommands
-} from './wire_protocol/compression';
+import { compress, Compressor, type CompressorName, decompress } from './wire_protocol/compression';
 import { OP_COMPRESSED, OP_MSG } from './wire_protocol/constants';
 
 const MESSAGE_HEADER_SIZE = 16;
@@ -85,7 +79,7 @@ export class MessageStream extends Duplex {
     operationDescription: OperationDescription
   ): void {
     const agreedCompressor = operationDescription.agreedCompressor ?? 'none';
-    if (agreedCompressor === 'none' || !canCompress(command)) {
+    if (agreedCompressor === 'none' || !OpCompressedRequest.canCompress(command)) {
       const data = command.toBin();
       this.push(Array.isArray(data) ? Buffer.concat(data) : data);
       return;
@@ -128,14 +122,6 @@ export class MessageStream extends Duplex {
   }
 }
 
-// Return whether a command contains an uncompressible command term
-// Will return true if command contains no uncompressible command terms
-function canCompress(command: WriteProtocolMessageType) {
-  const commandDoc = command instanceof Msg ? command.command : command.query;
-  const commandName = Object.keys(commandDoc)[0];
-  return !uncompressibleCommands.has(commandName);
-}
-
 function processIncomingData(stream: MessageStream, callback: Callback<Buffer>): void {
   const buffer = stream[kBuffer];
   const sizeOfMessage = buffer.getInt32();
@@ -179,7 +165,7 @@ function processIncomingData(stream: MessageStream, callback: Callback<Buffer>):
     return false;
   };
 
-  let ResponseType = messageHeader.opCode === OP_MSG ? BinMsg : Response;
+  let ResponseType = messageHeader.opCode === OP_MSG ? OpMsgResponse : OpQueryResponse;
   if (messageHeader.opCode !== OP_COMPRESSED) {
     const messageBody = message.subarray(MESSAGE_HEADER_SIZE);
 
@@ -205,7 +191,7 @@ function processIncomingData(stream: MessageStream, callback: Callback<Buffer>):
   const compressedBuffer = message.slice(MESSAGE_HEADER_SIZE + 9);
 
   // recalculate based on wrapped opcode
-  ResponseType = messageHeader.opCode === OP_MSG ? BinMsg : Response;
+  ResponseType = messageHeader.opCode === OP_MSG ? OpMsgResponse : OpQueryResponse;
   decompress(compressorID, compressedBuffer).then(
     messageBody => {
       if (messageBody.length !== messageHeader.length) {