mongodb 6.7.0-dev.20240608.sha.0655c730 → 6.7.0-dev.20240614.sha.3ed6a2ad

package/src/cmap/wire_protocol/responses.ts

@@ -1,15 +1,17 @@
 import {
+  type BSONElement,
   type BSONSerializeOptions,
   BSONType,
   type Document,
   Long,
   parseToElementsToArray,
+  pluckBSONSerializeOptions,
   type Timestamp
 } from '../../bson';
 import { MongoUnexpectedServerResponseError } from '../../error';
 import { type ClusterTime } from '../../sdam/common';
-import { type MongoDBNamespace, ns } from '../../utils';
-import { OnDemandDocument } from './on_demand/document';
+import { decorateDecryptionResult, ns } from '../../utils';
+import { type JSTypeOf, OnDemandDocument } from './on_demand/document';
 
 // eslint-disable-next-line no-restricted-syntax
 const enum BSONElementOffset {
@@ -30,8 +32,7 @@ const enum BSONElementOffset {
  *
  * @param bytes - BSON document returned from the server
  */
-export function isErrorResponse(bson: Uint8Array): boolean {
-  const elements = parseToElementsToArray(bson, 0);
+export function isErrorResponse(bson: Uint8Array, elements: BSONElement[]): boolean {
   for (let eIdx = 0; eIdx < elements.length; eIdx++) {
     const element = elements[eIdx];
 
@@ -60,26 +61,49 @@ export function isErrorResponse(bson: Uint8Array): boolean {
 /** @internal */
 export type MongoDBResponseConstructor = {
   new (bson: Uint8Array, offset?: number, isArray?: boolean): MongoDBResponse;
+  make(bson: Uint8Array): MongoDBResponse;
 };
 
 /** @internal */
 export class MongoDBResponse extends OnDemandDocument {
+  // Wrap error thrown from BSON
+  public override get<const T extends keyof JSTypeOf>(
+    name: string | number,
+    as: T,
+    required?: false | undefined
+  ): JSTypeOf[T] | null;
+  public override get<const T extends keyof JSTypeOf>(
+    name: string | number,
+    as: T,
+    required: true
+  ): JSTypeOf[T];
+  public override get<const T extends keyof JSTypeOf>(
+    name: string | number,
+    as: T,
+    required?: boolean | undefined
+  ): JSTypeOf[T] | null {
+    try {
+      return super.get(name, as, required);
+    } catch (cause) {
+      throw new MongoUnexpectedServerResponseError(cause.message, { cause });
+    }
+  }
+
   static is(value: unknown): value is MongoDBResponse {
     return value instanceof MongoDBResponse;
   }
 
+  static make(bson: Uint8Array) {
+    const elements = parseToElementsToArray(bson, 0);
+    const isError = isErrorResponse(bson, elements);
+    return isError
+      ? new MongoDBResponse(bson, 0, false, elements)
+      : new this(bson, 0, false, elements);
+  }
+
   // {ok:1}
   static empty = new MongoDBResponse(new Uint8Array([13, 0, 0, 0, 16, 111, 107, 0, 1, 0, 0, 0, 0]));
 
-  /** Indicates this document is a server error */
-  public get isError() {
-    let isError = this.ok === 0;
-    isError ||= this.has('errmsg');
-    isError ||= this.has('code');
-    isError ||= this.has('$err'); // The '$err' field is used in OP_REPLY responses
-    return isError;
-  }
-
   /**
    * Drivers can safely assume that the `recoveryToken` field is always a BSON document but drivers MUST NOT modify the
    * contents of the document.
@@ -110,6 +134,7 @@ export class MongoDBResponse extends OnDemandDocument {
     return this.get('operationTime', BSONType.timestamp);
   }
 
+  /** Normalizes whatever BSON value is "ok" to a JS number 1 or 0. */
   public get ok(): 0 | 1 {
     return this.getNumber('ok') ? 1 : 0;
   }
@@ -144,13 +169,7 @@ export class MongoDBResponse extends OnDemandDocument {
 
   public override toObject(options?: BSONSerializeOptions): Record<string, any> {
     const exactBSONOptions = {
-      useBigInt64: options?.useBigInt64,
-      promoteLongs: options?.promoteLongs,
-      promoteValues: options?.promoteValues,
-      promoteBuffers: options?.promoteBuffers,
-      bsonRegExp: options?.bsonRegExp,
-      raw: options?.raw ?? false,
-      fieldsAsRaw: options?.fieldsAsRaw ?? {},
+      ...pluckBSONSerializeOptions(options ?? {}),
       validation: this.parseBsonSerializationOptions(options)
     };
     return super.toObject(exactBSONOptions);
@@ -169,69 +188,145 @@ export class MongoDBResponse extends OnDemandDocument {
 
 /** @internal */
 export class CursorResponse extends MongoDBResponse {
+  /**
+   * Devtools need to know which keys were encrypted before the driver automatically decrypted them.
+   * If decorating is enabled (`Symbol.for('@@mdb.decorateDecryptionResult')`), this field will be set,
+   * storing the original encrypted response from the server, so that we can build an object that has
+   * the list of BSON keys that were encrypted stored at a well known symbol: `Symbol.for('@@mdb.decryptedKeys')`.
+   */
+  encryptedResponse?: MongoDBResponse;
   /**
    * This supports a feature of the FindCursor.
    * It is an optimization to avoid an extra getMore when the limit has been reached
    */
-  static emptyGetMore = { id: new Long(0), length: 0, shift: () => null };
+  static emptyGetMore: CursorResponse = {
+    id: new Long(0),
+    length: 0,
+    shift: () => null
+  } as unknown as CursorResponse;
 
   static override is(value: unknown): value is CursorResponse {
     return value instanceof CursorResponse || value === CursorResponse.emptyGetMore;
   }
 
-  public id: Long;
-  public ns: MongoDBNamespace | null = null;
-  public batchSize = 0;
-
-  private batch: OnDemandDocument;
+  private _batch: OnDemandDocument | null = null;
   private iterated = 0;
 
-  constructor(bytes: Uint8Array, offset?: number, isArray?: boolean) {
-    super(bytes, offset, isArray);
+  get cursor() {
+    return this.get('cursor', BSONType.object, true);
+  }
 
-    const cursor = this.get('cursor', BSONType.object, true);
+  public get id(): Long {
+    try {
+      return Long.fromBigInt(this.cursor.get('id', BSONType.long, true));
+    } catch (cause) {
+      throw new MongoUnexpectedServerResponseError(cause.message, { cause });
+    }
+  }
 
-    const id = cursor.get('id', BSONType.long, true);
-    this.id = new Long(Number(id & 0xffff_ffffn), Number((id >> 32n) & 0xffff_ffffn));
+  public get ns() {
+    const namespace = this.cursor.get('ns', BSONType.string);
+    if (namespace != null) return ns(namespace);
+    return null;
+  }
+
+  public get length() {
+    return Math.max(this.batchSize - this.iterated, 0);
+  }
 
-    const namespace = cursor.get('ns', BSONType.string);
-    if (namespace != null) this.ns = ns(namespace);
+  private _encryptedBatch: OnDemandDocument | null = null;
+  get encryptedBatch() {
+    if (this.encryptedResponse == null) return null;
+    if (this._encryptedBatch != null) return this._encryptedBatch;
 
-    if (cursor.has('firstBatch')) this.batch = cursor.get('firstBatch', BSONType.array, true);
-    else if (cursor.has('nextBatch')) this.batch = cursor.get('nextBatch', BSONType.array, true);
+    const cursor = this.encryptedResponse?.get('cursor', BSONType.object);
+    if (cursor?.has('firstBatch'))
+      this._encryptedBatch = cursor.get('firstBatch', BSONType.array, true);
+    else if (cursor?.has('nextBatch'))
+      this._encryptedBatch = cursor.get('nextBatch', BSONType.array, true);
     else throw new MongoUnexpectedServerResponseError('Cursor document did not contain a batch');
 
-    this.batchSize = this.batch.size();
+    return this._encryptedBatch;
   }
 
-  get length() {
-    return Math.max(this.batchSize - this.iterated, 0);
+  private get batch() {
+    if (this._batch != null) return this._batch;
+    const cursor = this.cursor;
+    if (cursor.has('firstBatch')) this._batch = cursor.get('firstBatch', BSONType.array, true);
+    else if (cursor.has('nextBatch')) this._batch = cursor.get('nextBatch', BSONType.array, true);
+    else throw new MongoUnexpectedServerResponseError('Cursor document did not contain a batch');
+    return this._batch;
+  }
+
+  public get batchSize() {
+    return this.batch?.size();
   }
 
-  shift(options?: BSONSerializeOptions): any {
+  public get postBatchResumeToken() {
+    return (
+      this.cursor.get('postBatchResumeToken', BSONType.object)?.toObject({
+        promoteValues: false,
+        promoteLongs: false,
+        promoteBuffers: false
+      }) ?? null
+    );
+  }
+
+  public shift(options?: BSONSerializeOptions): any {
     if (this.iterated >= this.batchSize) {
       return null;
     }
 
     const result = this.batch.get(this.iterated, BSONType.object, true) ?? null;
+    const encryptedResult = this.encryptedBatch?.get(this.iterated, BSONType.object, true) ?? null;
+
     this.iterated += 1;
 
     if (options?.raw) {
       return result.toBytes();
     } else {
-      return result.toObject(options);
+      const object = result.toObject(options);
+      if (encryptedResult) {
+        decorateDecryptionResult(object, encryptedResult.toObject(options), true);
+      }
+      return object;
     }
   }
 
-  clear() {
+  public clear() {
     this.iterated = this.batchSize;
   }
+}
+
+/**
+ * Explain responses have nothing to do with cursor responses
+ * This class serves to temporarily avoid refactoring how cursors handle
+ * explain responses which is to detect that the response is not cursor-like and return the explain
+ * result as the "first and only" document in the "batch" and end the "cursor"
+ */
+export class ExplainedCursorResponse extends CursorResponse {
+  isExplain = true;
+
+  override get id(): Long {
+    return Long.fromBigInt(0n);
+  }
+
+  override get batchSize() {
+    return 0;
+  }
+
+  override get ns() {
+    return null;
+  }
 
-  pushMany() {
-    throw new Error('pushMany Unsupported method');
+  _length = 1;
+  override get length(): number {
+    return this._length;
   }
 
-  push() {
-    throw new Error('push Unsupported method');
+  override shift(options?: BSONSerializeOptions | undefined) {
+    if (this._length === 0) return null;
+    this._length -= 1;
+    return this.toObject(options);
   }
 }

package/src/constants.ts

@@ -165,3 +165,12 @@ export const LEGACY_HELLO_COMMAND = 'ismaster';
  * The legacy hello command that was deprecated in MongoDB 5.0.
  */
 export const LEGACY_HELLO_COMMAND_CAMEL_CASE = 'isMaster';
+
+// Typescript errors if we index objects with `Symbol.for(...)`, so
+// to avoid TS errors we pull them out into variables.  Then we can type
+// the objects (and class) that we expect to see them on and prevent TS
+// errors.
+/** @internal */
+export const kDecorateResult = Symbol.for('@@mdb.decorateDecryptionResult');
+/** @internal */
+export const kDecoratedKeys = Symbol.for('@@mdb.decryptedKeys');

package/src/cursor/abstract_cursor.ts

@@ -1,7 +1,7 @@
 import { Readable, Transform } from 'stream';
 
 import { type BSONSerializeOptions, type Document, Long, pluckBSONSerializeOptions } from '../bson';
-import { CursorResponse } from '../cmap/wire_protocol/responses';
+import { type CursorResponse } from '../cmap/wire_protocol/responses';
 import {
   MongoAPIError,
   MongoCursorExhaustedError,
@@ -11,15 +11,33 @@ import {
   MongoTailableCursorError
 } from '../error';
 import type { MongoClient } from '../mongo_client';
-import { type TODO_NODE_3286, TypedEventEmitter } from '../mongo_types';
-import { executeOperation, type ExecutionResult } from '../operations/execute_operation';
+import { TypedEventEmitter } from '../mongo_types';
+import { executeOperation } from '../operations/execute_operation';
 import { GetMoreOperation } from '../operations/get_more';
 import { KillCursorsOperation } from '../operations/kill_cursors';
 import { ReadConcern, type ReadConcernLike } from '../read_concern';
 import { ReadPreference, type ReadPreferenceLike } from '../read_preference';
 import type { Server } from '../sdam/server';
 import { ClientSession, maybeClearPinnedConnection } from '../sessions';
-import { List, type MongoDBNamespace, ns, squashError } from '../utils';
+import { type MongoDBNamespace, squashError } from '../utils';
+
+/**
+ * @internal
+ * TODO(NODE-2882): A cursor's getMore commands must be run on the same server it was started on
+ * and the same session must be used for the lifetime of the cursor. This object serves to get the
+ * server and session (along with the response) out of executeOperation back to the AbstractCursor.
+ *
+ * There may be a better design for communicating these values back to the cursor, currently an operation
+ * MUST store the selected server on itself so it can be read after executeOperation has returned.
+ */
+export interface InitialCursorResponse {
+  /** The server selected for the operation */
+  server: Server;
+  /** The session used for this operation, may be implicitly created */
+  session?: ClientSession;
+  /** The raw server response for the operation */
+  response: CursorResponse;
+}
 
 /** @public */
 export const CURSOR_FLAGS = [
@@ -118,13 +136,7 @@ export abstract class AbstractCursor<
   /** @internal */
   private cursorNamespace: MongoDBNamespace;
   /** @internal */
-  private documents: {
-    length: number;
-    shift(bsonOptions?: any): TSchema | null;
-    clear(): void;
-    pushMany(many: Iterable<TSchema>): void;
-    push(item: TSchema): void;
-  };
+  private documents: CursorResponse | null = null;
   /** @internal */
   private cursorClient: MongoClient;
   /** @internal */
@@ -155,7 +167,6 @@ export abstract class AbstractCursor<
     this.cursorClient = client;
     this.cursorNamespace = namespace;
     this.cursorId = null;
-    this.documents = new List();
     this.initialized = false;
     this.isClosed = false;
     this.isKilled = false;
@@ -252,16 +263,19 @@ export abstract class AbstractCursor<
 
   /** Returns current buffered documents length */
   bufferedCount(): number {
-    return this.documents.length;
+    return this.documents?.length ?? 0;
   }
 
   /** Returns current buffered documents */
   readBufferedDocuments(number?: number): TSchema[] {
     const bufferedDocs: TSchema[] = [];
-    const documentsToRead = Math.min(number ?? this.documents.length, this.documents.length);
+    const documentsToRead = Math.min(
+      number ?? this.documents?.length ?? 0,
+      this.documents?.length ?? 0
+    );
 
     for (let count = 0; count < documentsToRead; count++) {
-      const document = this.documents.shift(this.cursorOptions);
+      const document = this.documents?.shift(this.cursorOptions);
       if (document != null) {
         bufferedDocs.push(document);
       }
@@ -269,7 +283,6 @@ export abstract class AbstractCursor<
 
     return bufferedDocs;
   }
-
   async *[Symbol.asyncIterator](): AsyncGenerator<TSchema, void, void> {
     if (this.isClosed) {
       return;
@@ -281,11 +294,11 @@ export abstract class AbstractCursor<
           return;
         }
 
-        if (this.isClosed && this.documents.length === 0) {
+        if (this.closed && (this.documents?.length ?? 0) === 0) {
           return;
         }
 
-        if (this.cursorId != null && this.isDead && this.documents.length === 0) {
+        if (this.cursorId != null && this.isDead && (this.documents?.length ?? 0) === 0) {
           return;
         }
 
@@ -347,11 +360,11 @@ export abstract class AbstractCursor<
     }
 
     do {
-      if (this.documents.length !== 0) {
+      if ((this.documents?.length ?? 0) !== 0) {
         return true;
       }
       await this.fetchBatch();
-    } while (!this.isDead || this.documents.length !== 0);
+    } while (!this.isDead || (this.documents?.length ?? 0) !== 0);
 
     return false;
   }
@@ -363,13 +376,13 @@ export abstract class AbstractCursor<
     }
 
     do {
-      const doc = this.documents.shift();
+      const doc = this.documents?.shift(this.cursorOptions);
       if (doc != null) {
         if (this.transform != null) return await this.transformDocument(doc);
         return doc;
       }
       await this.fetchBatch();
-    } while (!this.isDead || this.documents.length !== 0);
+    } while (!this.isDead || (this.documents?.length ?? 0) !== 0);
 
     return null;
   }
@@ -382,7 +395,7 @@ export abstract class AbstractCursor<
       throw new MongoCursorExhaustedError();
     }
 
-    let doc = this.documents.shift();
+    let doc = this.documents?.shift(this.cursorOptions);
     if (doc != null) {
       if (this.transform != null) return await this.transformDocument(doc);
       return doc;
@@ -390,7 +403,7 @@ export abstract class AbstractCursor<
 
     await this.fetchBatch();
 
-    doc = this.documents.shift();
+    doc = this.documents?.shift(this.cursorOptions);
     if (doc != null) {
       if (this.transform != null) return await this.transformDocument(doc);
       return doc;
@@ -591,7 +604,7 @@ export abstract class AbstractCursor<
     }
 
     this.cursorId = null;
-    this.documents.clear();
+    this.documents?.clear();
     this.isClosed = false;
     this.isKilled = false;
     this.initialized = false;
@@ -615,10 +628,12 @@ export abstract class AbstractCursor<
   abstract clone(): AbstractCursor<TSchema>;
 
   /** @internal */
-  protected abstract _initialize(session: ClientSession | undefined): Promise<ExecutionResult>;
+  protected abstract _initialize(
+    session: ClientSession | undefined
+  ): Promise<InitialCursorResponse>;
 
   /** @internal */
-  async getMore(batchSize: number, useCursorResponse = false): Promise<Document | null> {
+  async getMore(batchSize: number): Promise<CursorResponse> {
     if (this.cursorId == null) {
       throw new MongoRuntimeError(
         'Unexpected null cursor id. A cursor creating command should have set this'
@@ -636,8 +651,7 @@ export abstract class AbstractCursor<
       {
         ...this.cursorOptions,
         session: this.cursorSession,
-        batchSize,
-        useCursorResponse
+        batchSize
       }
     );
 
@@ -656,27 +670,10 @@ export abstract class AbstractCursor<
       const state = await this._initialize(this.cursorSession);
       const response = state.response;
       this.selectedServer = state.server;
-      if (CursorResponse.is(response)) {
-        this.cursorId = response.id;
-        if (response.ns) this.cursorNamespace = response.ns;
-        this.documents = response;
-      } else if (response.cursor) {
-        // TODO(NODE-2674): Preserve int64 sent from MongoDB
-        this.cursorId = getCursorId(response);
-        if (response.cursor.ns) this.cursorNamespace = ns(response.cursor.ns);
-        this.documents.pushMany(response.cursor.firstBatch);
-      }
-
-      if (this.cursorId == null) {
-        // When server responses return without a cursor document, we close this cursor
-        // and return the raw server response. This is the case for explain commands
-        this.cursorId = Long.ZERO;
-        // TODO(NODE-3286): ExecutionResult needs to accept a generic parameter
-        this.documents.push(state.response as TODO_NODE_3286);
-      }
-
-      // the cursor is now initialized, even if it is dead
-      this.initialized = true;
+      this.cursorId = response.id;
+      this.cursorNamespace = response.ns ?? this.namespace;
+      this.documents = response;
+      this.initialized = true; // the cursor is now initialized, even if it is dead
     } catch (error) {
       // the cursor is now initialized, even if an error occurred
       this.initialized = true;
@@ -708,7 +705,7 @@ export abstract class AbstractCursor<
     if (this.cursorId == null) {
       await this.cursorInit();
       // If the cursor died or returned documents, return
-      if (this.documents.length !== 0 || this.isDead) return;
+      if ((this.documents?.length ?? 0) !== 0 || this.isDead) return;
       // Otherwise, run a getMore
     }
 
@@ -717,16 +714,8 @@ export abstract class AbstractCursor<
 
     try {
       const response = await this.getMore(batchSize);
-      // CursorResponse is disabled in this PR
-      // however the special `emptyGetMore` can be returned from find cursors
-      if (CursorResponse.is(response)) {
-        this.cursorId = response.id;
-        this.documents = response;
-      } else if (response?.cursor) {
-        const cursorId = getCursorId(response);
-        this.documents.pushMany(response.cursor.nextBatch);
-        this.cursorId = cursorId;
-      }
+      this.cursorId = response.id;
+      this.documents = response;
     } catch (error) {
       try {
         await this.cleanup(error);
@@ -789,7 +778,7 @@ export abstract class AbstractCursor<
   /** @internal */
   private emitClose() {
     try {
-      if (!this.hasEmittedClose && (this.documents.length === 0 || this.isClosed)) {
+      if (!this.hasEmittedClose && ((this.documents?.length ?? 0) === 0 || this.isClosed)) {
         // @ts-expect-error: CursorEvents is generic so Parameters<CursorEvents["close"]> may not be assignable to `[]`. Not sure how to require extenders do not add parameters.
         this.emit('close');
       }
@@ -827,15 +816,6 @@ export abstract class AbstractCursor<
   }
 }
 
-/** A temporary helper to box up the many possible type issue of cursor ids */
-function getCursorId(response: Document) {
-  return typeof response.cursor.id === 'number'
-    ? Long.fromNumber(response.cursor.id)
-    : typeof response.cursor.id === 'bigint'
-    ? Long.fromBigInt(response.cursor.id)
-    : response.cursor.id;
-}
-
 class ReadableCursorStream extends Readable {
   private _cursor: AbstractCursor;
   private _readInProgress = false;

package/src/cursor/aggregation_cursor.ts

@@ -2,12 +2,12 @@ import type { Document } from '../bson';
 import type { ExplainVerbosityLike } from '../explain';
 import type { MongoClient } from '../mongo_client';
 import { AggregateOperation, type AggregateOptions } from '../operations/aggregate';
-import { executeOperation, type ExecutionResult } from '../operations/execute_operation';
+import { executeOperation } from '../operations/execute_operation';
 import type { ClientSession } from '../sessions';
 import type { Sort } from '../sort';
 import type { MongoDBNamespace } from '../utils';
 import { mergeOptions } from '../utils';
-import type { AbstractCursorOptions } from './abstract_cursor';
+import type { AbstractCursorOptions, InitialCursorResponse } from './abstract_cursor';
 import { AbstractCursor } from './abstract_cursor';
 
 /** @public */
@@ -51,7 +51,7 @@ export class AggregationCursor<TSchema = any> extends AbstractCursor<TSchema> {
   }
 
   /** @internal */
-  async _initialize(session: ClientSession): Promise<ExecutionResult> {
+  async _initialize(session: ClientSession): Promise<InitialCursorResponse> {
     const aggregateOperation = new AggregateOperation(this.namespace, this.pipeline, {
       ...this.aggregateOptions,
       ...this.cursorOptions,
@@ -60,20 +60,21 @@ export class AggregationCursor<TSchema = any> extends AbstractCursor<TSchema> {
 
     const response = await executeOperation(this.client, aggregateOperation);
 
-    // TODO: NODE-2882
     return { server: aggregateOperation.server, session, response };
   }
 
   /** Execute the explain for the cursor */
   async explain(verbosity?: ExplainVerbosityLike): Promise<Document> {
-    return await executeOperation(
-      this.client,
-      new AggregateOperation(this.namespace, this.pipeline, {
-        ...this.aggregateOptions, // NOTE: order matters here, we may need to refine this
-        ...this.cursorOptions,
-        explain: verbosity ?? true
-      })
-    );
+    return (
+      await executeOperation(
+        this.client,
+        new AggregateOperation(this.namespace, this.pipeline, {
+          ...this.aggregateOptions, // NOTE: order matters here, we may need to refine this
+          ...this.cursorOptions,
+          explain: verbosity ?? true
+        })
+      )
+    ).shift(this.aggregateOptions);
   }
 
   /** Add a stage to the aggregation pipeline