mongodb 6.9.0-dev.20241010.sha.6ecf198f → 6.9.0-dev.20241011.sha.8def42de

package/src/operations/client_bulk_write/client_bulk_write.ts

@@ -27,6 +27,14 @@ export class ClientBulkWriteOperation extends CommandOperation<ClientBulkWriteCu
     this.ns = new MongoDBNamespace('admin', '$cmd');
   }
 
+  override resetBatch(): boolean {
+    return this.commandBuilder.resetBatch();
+  }
+
+  override get canRetryWrite(): boolean {
+    return this.commandBuilder.isBatchRetryable;
+  }
+
   /**
    * Execute the command. Superclass will handle write concern, etc.
    * @param server - The server.
@@ -41,14 +49,20 @@ export class ClientBulkWriteOperation extends CommandOperation<ClientBulkWriteCu
 
     if (server.description.type === ServerType.LoadBalancer) {
       if (session) {
-        // Checkout a connection to build the command.
-        const connection = await server.pool.checkOut();
-        // Pin the connection to the session so it get used to execute the command and we do not
-        // perform a double check-in/check-out.
-        session.pin(connection);
+        let connection;
+        if (!session.pinnedConnection) {
+          // Checkout a connection to build the command.
+          connection = await server.pool.checkOut();
+          // Pin the connection to the session so it get used to execute the command and we do not
+          // perform a double check-in/check-out.
+          session.pin(connection);
+        } else {
+          connection = session.pinnedConnection;
+        }
         command = this.commandBuilder.buildBatch(
           connection.hello?.maxMessageSizeBytes,
-          connection.hello?.maxWriteBatchSize
+          connection.hello?.maxWriteBatchSize,
+          connection.hello?.maxBsonObjectSize
         );
       } else {
         throw new MongoClientBulkWriteExecutionError(
@@ -59,16 +73,26 @@ export class ClientBulkWriteOperation extends CommandOperation<ClientBulkWriteCu
       // At this point we have a server and the auto connect code has already
       // run in executeOperation, so the server description will be populated.
       // We can use that to build the command.
-      if (!server.description.maxWriteBatchSize || !server.description.maxMessageSizeBytes) {
+      if (
+        !server.description.maxWriteBatchSize ||
+        !server.description.maxMessageSizeBytes ||
+        !server.description.maxBsonObjectSize
+      ) {
         throw new MongoClientBulkWriteExecutionError(
-          'In order to execute a client bulk write, both maxWriteBatchSize and maxMessageSizeBytes must be provided by the servers hello response.'
+          'In order to execute a client bulk write, both maxWriteBatchSize, maxMessageSizeBytes and maxBsonObjectSize must be provided by the servers hello response.'
         );
       }
       command = this.commandBuilder.buildBatch(
         server.description.maxMessageSizeBytes,
-        server.description.maxWriteBatchSize
+        server.description.maxWriteBatchSize,
+        server.description.maxBsonObjectSize
       );
     }
+
+    // Check after the batch is built if we cannot retry it and override the option.
+    if (!this.canRetryWrite) {
+      this.options.willRetryWrite = false;
+    }
     return await super.executeCommand(server, session, command, ClientBulkWriteCursorResponse);
   }
 }
@@ -77,5 +101,7 @@ export class ClientBulkWriteOperation extends CommandOperation<ClientBulkWriteCu
 defineAspects(ClientBulkWriteOperation, [
   Aspect.WRITE_OPERATION,
   Aspect.SKIP_COLLATION,
-  Aspect.CURSOR_CREATING
+  Aspect.CURSOR_CREATING,
+  Aspect.RETRYABLE,
+  Aspect.COMMAND_BATCHING
 ]);

package/src/operations/client_bulk_write/command_builder.ts

@@ -1,8 +1,9 @@
 import { BSON, type Document } from '../../bson';
 import { DocumentSequence } from '../../cmap/commands';
+import { MongoAPIError, MongoInvalidArgumentError } from '../../error';
 import { type PkFactory } from '../../mongo_client';
 import type { Filter, OptionalId, UpdateFilter, WithoutId } from '../../mongo_types';
-import { DEFAULT_PK_FACTORY } from '../../utils';
+import { DEFAULT_PK_FACTORY, hasAtomicOperators } from '../../utils';
 import { type CollationOptions } from '../command';
 import { type Hint } from '../operation';
 import type {
@@ -38,8 +39,14 @@ export class ClientBulkWriteCommandBuilder {
   models: AnyClientBulkWriteModel[];
   options: ClientBulkWriteOptions;
   pkFactory: PkFactory;
+  /** The current index in the models array that is being processed. */
   currentModelIndex: number;
+  /** The model index that the builder was on when it finished the previous batch. Used for resets when retrying. */
+  previousModelIndex: number;
+  /** The last array of operations that were created. Used by the results merger for indexing results. */
   lastOperations: Document[];
+  /** Returns true if the current batch being created has no multi-updates. */
+  isBatchRetryable: boolean;
 
   /**
    * Create the command builder.
@@ -54,7 +61,9 @@ export class ClientBulkWriteCommandBuilder {
     this.options = options;
     this.pkFactory = pkFactory ?? DEFAULT_PK_FACTORY;
     this.currentModelIndex = 0;
+    this.previousModelIndex = 0;
     this.lastOperations = [];
+    this.isBatchRetryable = true;
   }
 
   /**
@@ -76,27 +85,57 @@ export class ClientBulkWriteCommandBuilder {
     return this.currentModelIndex < this.models.length;
   }
 
+  /**
+   * When we need to retry a command we need to set the current
+   * model index back to its previous value.
+   */
+  resetBatch(): boolean {
+    this.currentModelIndex = this.previousModelIndex;
+    return true;
+  }
+
   /**
    * Build a single batch of a client bulk write command.
    * @param maxMessageSizeBytes - The max message size in bytes.
    * @param maxWriteBatchSize - The max write batch size.
    * @returns The client bulk write command.
    */
-  buildBatch(maxMessageSizeBytes: number, maxWriteBatchSize: number): ClientBulkWriteCommand {
+  buildBatch(
+    maxMessageSizeBytes: number,
+    maxWriteBatchSize: number,
+    maxBsonObjectSize: number
+  ): ClientBulkWriteCommand {
+    // We start by assuming the batch has no multi-updates, so it is retryable
+    // until we find them.
+    this.isBatchRetryable = true;
     let commandLength = 0;
     let currentNamespaceIndex = 0;
     const command: ClientBulkWriteCommand = this.baseCommand();
     const namespaces = new Map<string, number>();
+    // In the case of retries we need to mark where we started this batch.
+    this.previousModelIndex = this.currentModelIndex;
 
     while (this.currentModelIndex < this.models.length) {
       const model = this.models[this.currentModelIndex];
       const ns = model.namespace;
       const nsIndex = namespaces.get(ns);
 
+      // Multi updates are not retryable.
+      if (model.name === 'deleteMany' || model.name === 'updateMany') {
+        this.isBatchRetryable = false;
+      }
+
       if (nsIndex != null) {
         // Build the operation and serialize it to get the bytes buffer.
         const operation = buildOperation(model, nsIndex, this.pkFactory);
-        const operationBuffer = BSON.serialize(operation);
+        let operationBuffer;
+        try {
+          operationBuffer = BSON.serialize(operation);
+        } catch (cause) {
+          throw new MongoInvalidArgumentError(`Could not serialize operation to BSON`, { cause });
+        }
+
+        validateBufferSize('ops', operationBuffer, maxBsonObjectSize);
 
         // Check if the operation buffer can fit in the command. If it can,
         // then add the operation to the document sequence and increment the
@@ -119,9 +158,18 @@ export class ClientBulkWriteCommandBuilder {
         // construct our nsInfo and ops documents and buffers.
         namespaces.set(ns, currentNamespaceIndex);
         const nsInfo = { ns: ns };
-        const nsInfoBuffer = BSON.serialize(nsInfo);
         const operation = buildOperation(model, currentNamespaceIndex, this.pkFactory);
-        const operationBuffer = BSON.serialize(operation);
+        let nsInfoBuffer;
+        let operationBuffer;
+        try {
+          nsInfoBuffer = BSON.serialize(nsInfo);
+          operationBuffer = BSON.serialize(operation);
+        } catch (cause) {
+          throw new MongoInvalidArgumentError(`Could not serialize ns info to BSON`, { cause });
+        }
+
+        validateBufferSize('nsInfo', nsInfoBuffer, maxBsonObjectSize);
+        validateBufferSize('ops', operationBuffer, maxBsonObjectSize);
 
         // Check if the operation and nsInfo buffers can fit in the command. If they
         // can, then add the operation and nsInfo to their respective document
@@ -179,6 +227,14 @@ export class ClientBulkWriteCommandBuilder {
   }
 }
 
+function validateBufferSize(name: string, buffer: Uint8Array, maxBsonObjectSize: number) {
+  if (buffer.length > maxBsonObjectSize) {
+    throw new MongoInvalidArgumentError(
+      `Client bulk write operation ${name} of length ${buffer.length} exceeds the max bson object size of ${maxBsonObjectSize}`
+    );
+  }
+}
+
 /** @internal */
 interface ClientInsertOperation {
   insert: number;
@@ -293,6 +349,18 @@ export const buildUpdateManyOperation = (
   return createUpdateOperation(model, index, true);
 };
 
+/**
+ * Validate the update document.
+ * @param update - The update document.
+ */
+function validateUpdate(update: Document) {
+  if (!hasAtomicOperators(update)) {
+    throw new MongoAPIError(
+      'Client bulk write update models must only contain atomic modifiers (start with $) and must not be empty.'
+    );
+  }
+}
+
 /**
  * Creates a delete operation based on the parameters.
  */
@@ -301,6 +369,11 @@ function createUpdateOperation(
   index: number,
   multi: boolean
 ): ClientUpdateOperation {
+  // Update documents provided in UpdateOne and UpdateMany write models are
+  // required only to contain atomic modifiers (i.e. keys that start with "$").
+  // Drivers MUST throw an error if an update document is empty or if the
+  // document's first key does not start with "$".
+  validateUpdate(model.update);
   const document: ClientUpdateOperation = {
     update: index,
     multi: multi,
@@ -343,6 +416,12 @@ export const buildReplaceOneOperation = (
   model: ClientReplaceOneModel,
   index: number
 ): ClientReplaceOneOperation => {
+  if (hasAtomicOperators(model.replacement)) {
+    throw new MongoAPIError(
+      'Client bulk write replace models must not contain atomic modifiers (start with $) and must not be empty.'
+    );
+  }
+
   const document: ClientReplaceOneOperation = {
     update: index,
     multi: false,

package/src/operations/client_bulk_write/common.ts

@@ -181,6 +181,12 @@ export interface ClientBulkWriteResult {
   deleteResults?: Map<number, ClientDeleteResult>;
 }
 
+/** @public */
+export interface ClientBulkWriteError {
+  code: number;
+  message: string;
+}
+
 /** @public */
 export interface ClientInsertOneResult {
   /**

package/src/operations/client_bulk_write/executor.ts

@@ -1,4 +1,9 @@
 import { ClientBulkWriteCursor } from '../../cursor/client_bulk_write_cursor';
+import {
+  MongoClientBulkWriteError,
+  MongoClientBulkWriteExecutionError,
+  MongoServerError
+} from '../../error';
 import { type MongoClient } from '../../mongo_client';
 import { WriteConcern } from '../../write_concern';
 import { executeOperation } from '../execute_operation';
@@ -31,9 +36,18 @@ export class ClientBulkWriteExecutor {
     operations: AnyClientBulkWriteModel[],
     options?: ClientBulkWriteOptions
   ) {
+    if (operations.length === 0) {
+      throw new MongoClientBulkWriteExecutionError('No client bulk write models were provided.');
+    }
+
     this.client = client;
     this.operations = operations;
-    this.options = { ...options };
+    this.options = {
+      ordered: true,
+      bypassDocumentValidation: false,
+      verboseResults: false,
+      ...options
+    };
 
     // If no write concern was provided, we inherit one from the client.
     if (!this.options.writeConcern) {
@@ -65,15 +79,42 @@ export class ClientBulkWriteExecutor {
     } else {
       const resultsMerger = new ClientBulkWriteResultsMerger(this.options);
       // For each command will will create and exhaust a cursor for the results.
-      let currentBatchOffset = 0;
       while (commandBuilder.hasNextBatch()) {
         const cursor = new ClientBulkWriteCursor(this.client, commandBuilder, this.options);
-        const docs = await cursor.toArray();
-        const operations = cursor.operations;
-        resultsMerger.merge(currentBatchOffset, operations, cursor.response, docs);
-        // Set the new batch index so we can back back to the index in the original models.
-        currentBatchOffset += operations.length;
+        try {
+          await resultsMerger.merge(cursor);
+        } catch (error) {
+          // Write concern errors are recorded in the writeConcernErrors field on MongoClientBulkWriteError.
+          // When a write concern error is encountered, it should not terminate execution of the bulk write
+          // for either ordered or unordered bulk writes. However, drivers MUST throw an exception at the end
+          // of execution if any write concern errors were observed.
+          if (error instanceof MongoServerError && !(error instanceof MongoClientBulkWriteError)) {
+            // Server side errors need to be wrapped inside a MongoClientBulkWriteError, where the root
+            // cause is the error property and a partial result is to be included.
+            const bulkWriteError = new MongoClientBulkWriteError({
+              message: 'Mongo client bulk write encountered an error during execution'
+            });
+            bulkWriteError.cause = error;
+            bulkWriteError.partialResult = resultsMerger.result;
+            throw bulkWriteError;
+          } else {
+            // Client side errors are just thrown.
+            throw error;
+          }
+        }
       }
+
+      // If we have write concern errors or unordered write errors at the end we throw.
+      if (resultsMerger.writeConcernErrors.length > 0 || resultsMerger.writeErrors.size > 0) {
+        const error = new MongoClientBulkWriteError({
+          message: 'Mongo client bulk write encountered errors during execution.'
+        });
+        error.writeConcernErrors = resultsMerger.writeConcernErrors;
+        error.writeErrors = resultsMerger.writeErrors;
+        error.partialResult = resultsMerger.result;
+        throw error;
+      }
+
       return resultsMerger.result;
     }
   }

package/src/operations/client_bulk_write/results_merger.ts

@@ -1,6 +1,9 @@
+import { MongoWriteConcernError } from '../..';
 import { type Document } from '../../bson';
-import { type ClientBulkWriteCursorResponse } from '../../cmap/wire_protocol/responses';
+import { type ClientBulkWriteCursor } from '../../cursor/client_bulk_write_cursor';
+import { MongoClientBulkWriteError } from '../../error';
 import {
+  type ClientBulkWriteError,
   type ClientBulkWriteOptions,
   type ClientBulkWriteResult,
   type ClientDeleteResult,
@@ -15,6 +18,9 @@ import {
 export class ClientBulkWriteResultsMerger {
   result: ClientBulkWriteResult;
   options: ClientBulkWriteOptions;
+  currentBatchOffset: number;
+  writeConcernErrors: Document[];
+  writeErrors: Map<number, ClientBulkWriteError>;
 
   /**
    * Instantiate the merger.
@@ -22,6 +28,9 @@ export class ClientBulkWriteResultsMerger {
    */
   constructor(options: ClientBulkWriteOptions) {
     this.options = options;
+    this.currentBatchOffset = 0;
+    this.writeConcernErrors = [];
+    this.writeErrors = new Map();
     this.result = {
       insertedCount: 0,
       upsertedCount: 0,
@@ -47,55 +56,126 @@ export class ClientBulkWriteResultsMerger {
    * @param documents - The documents in the cursor.
    * @returns The current result.
    */
-  merge(
-    currentBatchOffset: number,
-    operations: Document[],
-    response: ClientBulkWriteCursorResponse,
-    documents: Document[]
-  ): ClientBulkWriteResult {
-    // Update the counts from the cursor response.
-    this.result.insertedCount += response.insertedCount;
-    this.result.upsertedCount += response.upsertedCount;
-    this.result.matchedCount += response.matchedCount;
-    this.result.modifiedCount += response.modifiedCount;
-    this.result.deletedCount += response.deletedCount;
-
-    if (this.options.verboseResults) {
-      // Iterate all the documents in the cursor and update the result.
-      for (const document of documents) {
+  async merge(cursor: ClientBulkWriteCursor): Promise<ClientBulkWriteResult> {
+    let writeConcernErrorResult;
+    try {
+      for await (const document of cursor) {
         // Only add to maps if ok: 1
         if (document.ok === 1) {
-          // Get the corresponding operation from the command.
-          const operation = operations[document.idx];
-          // Handle insert results.
-          if ('insert' in operation) {
-            this.result.insertResults?.set(document.idx + currentBatchOffset, {
-              insertedId: operation.document._id
+          if (this.options.verboseResults) {
+            this.processDocument(cursor, document);
+          }
+        } else {
+          // If an individual write error is encountered during an ordered bulk write, drivers MUST
+          // record the error in writeErrors and immediately throw the exception. Otherwise, drivers
+          // MUST continue to iterate the results cursor and execute any further bulkWrite batches.
+          if (this.options.ordered) {
+            const error = new MongoClientBulkWriteError({
+              message: 'Mongo client ordered bulk write encountered a write error.'
+            });
+            error.writeErrors.set(document.idx + this.currentBatchOffset, {
+              code: document.code,
+              message: document.errmsg
+            });
+            error.partialResult = this.result;
+            throw error;
+          } else {
+            this.writeErrors.set(document.idx + this.currentBatchOffset, {
+              code: document.code,
+              message: document.errmsg
             });
           }
-          // Handle update results.
-          if ('update' in operation) {
-            const result: ClientUpdateResult = {
-              matchedCount: document.n,
-              modifiedCount: document.nModified ?? 0,
-              // Check if the bulk did actually upsert.
-              didUpsert: document.upserted != null
-            };
-            if (document.upserted) {
-              result.upsertedId = document.upserted._id;
+        }
+      }
+    } catch (error) {
+      if (error instanceof MongoWriteConcernError) {
+        const result = error.result;
+        writeConcernErrorResult = {
+          insertedCount: result.nInserted,
+          upsertedCount: result.nUpserted,
+          matchedCount: result.nMatched,
+          modifiedCount: result.nModified,
+          deletedCount: result.nDeleted,
+          writeConcernError: result.writeConcernError
+        };
+        if (this.options.verboseResults && result.cursor.firstBatch) {
+          for (const document of result.cursor.firstBatch) {
+            if (document.ok === 1) {
+              this.processDocument(cursor, document);
             }
-            this.result.updateResults?.set(document.idx + currentBatchOffset, result);
-          }
-          // Handle delete results.
-          if ('delete' in operation) {
-            this.result.deleteResults?.set(document.idx + currentBatchOffset, {
-              deletedCount: document.n
-            });
           }
         }
+      } else {
+        throw error;
+      }
+    } finally {
+      // Update the counts from the cursor response.
+      if (cursor.response) {
+        const response = cursor.response;
+        this.incrementCounts(response);
       }
+
+      // Increment the batch offset.
+      this.currentBatchOffset += cursor.operations.length;
+    }
+
+    // If we have write concern errors ensure they are added.
+    if (writeConcernErrorResult) {
+      const writeConcernError = writeConcernErrorResult.writeConcernError as Document;
+      this.incrementCounts(writeConcernErrorResult);
+      this.writeConcernErrors.push({
+        code: writeConcernError.code,
+        message: writeConcernError.errmsg
+      });
     }
 
     return this.result;
   }
+
+  /**
+   * Process an individual document in the results.
+   * @param cursor - The cursor.
+   * @param document - The document to process.
+   */
+  private processDocument(cursor: ClientBulkWriteCursor, document: Document) {
+    // Get the corresponding operation from the command.
+    const operation = cursor.operations[document.idx];
+    // Handle insert results.
+    if ('insert' in operation) {
+      this.result.insertResults?.set(document.idx + this.currentBatchOffset, {
+        insertedId: operation.document._id
+      });
+    }
+    // Handle update results.
+    if ('update' in operation) {
+      const result: ClientUpdateResult = {
+        matchedCount: document.n,
+        modifiedCount: document.nModified ?? 0,
+        // Check if the bulk did actually upsert.
+        didUpsert: document.upserted != null
+      };
+      if (document.upserted) {
+        result.upsertedId = document.upserted._id;
+      }
+      this.result.updateResults?.set(document.idx + this.currentBatchOffset, result);
+    }
+    // Handle delete results.
+    if ('delete' in operation) {
+      this.result.deleteResults?.set(document.idx + this.currentBatchOffset, {
+        deletedCount: document.n
+      });
+    }
+  }
+
+  /**
+   * Increment the result counts.
+   * @param document - The document with the results.
+   */
+  private incrementCounts(document: Document) {
+    this.result.insertedCount += document.insertedCount;
+    this.result.upsertedCount += document.upsertedCount;
+    this.result.matchedCount += document.matchedCount;
+    this.result.modifiedCount += document.modifiedCount;
+    this.result.deletedCount += document.deletedCount;
+  }
 }

package/src/operations/execute_operation.ts

@@ -230,6 +230,10 @@ async function tryOperation<
         });
       }
 
+      if (operation.hasAspect(Aspect.COMMAND_BATCHING) && !operation.canRetryWrite) {
+        throw previousOperationError;
+      }
+
       if (hasWriteAspect && !isRetryableWriteError(previousOperationError))
         throw previousOperationError;
 
@@ -260,6 +264,10 @@ async function tryOperation<
     }
 
     try {
+      // If tries > 0 and we are command batching we need to reset the batch.
+      if (tries > 0 && operation.hasAspect(Aspect.COMMAND_BATCHING)) {
+        operation.resetBatch();
+      }
       return await operation.execute(server, session);
     } catch (operationError) {
       if (!(operationError instanceof MongoError)) throw operationError;

package/src/operations/operation.ts

@@ -11,7 +11,8 @@ export const Aspect = {
   EXPLAINABLE: Symbol('EXPLAINABLE'),
   SKIP_COLLATION: Symbol('SKIP_COLLATION'),
   CURSOR_CREATING: Symbol('CURSOR_CREATING'),
-  MUST_SELECT_SAME_SERVER: Symbol('MUST_SELECT_SAME_SERVER')
+  MUST_SELECT_SAME_SERVER: Symbol('MUST_SELECT_SAME_SERVER'),
+  COMMAND_BATCHING: Symbol('COMMAND_BATCHING')
 } as const;
 
 /** @public */
@@ -98,6 +99,10 @@ export abstract class AbstractOperation<TResult = any> {
     this[kSession] = undefined;
   }
 
+  resetBatch(): boolean {
+    return true;
+  }
+
   get canRetryRead(): boolean {
     return this.hasAspect(Aspect.RETRYABLE) && this.hasAspect(Aspect.READ_OPERATION);
   }