@fluidframework/container-runtime 2.1.0-276326 → 2.1.0-281041

package/src/opLifecycle/opGroupingManager.ts

@@ -59,27 +59,28 @@ export class OpGroupingManager {
 	public groupBatch(batch: IBatch): IBatch<[BatchMessage]> {
 		assert(this.shouldGroup(batch), 0x946 /* cannot group the provided batch */);
 
-		if (batch.content.length >= 1000) {
+		if (batch.messages.length >= 1000) {
 			this.logger.sendTelemetryEvent({
 				eventName: "GroupLargeBatch",
-				length: batch.content.length,
+				length: batch.messages.length,
 				threshold: this.config.opCountThreshold,
 				reentrant: batch.hasReentrantOps,
-				referenceSequenceNumber: batch.content[0].referenceSequenceNumber,
+				// Non null asserting here because of the length check above
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				referenceSequenceNumber: batch.messages[0]!.referenceSequenceNumber,
 			});
 		}
 
-		for (const message of batch.content) {
+		for (const message of batch.messages) {
 			if (message.metadata) {
-				const keys = Object.keys(message.metadata);
-				assert(keys.length < 2, 0x5dd /* cannot group ops with metadata */);
-				assert(keys.length === 0 || keys[0] === "batch", 0x5de /* unexpected op metadata */);
+				const { batch: _batch, batchId, ...rest } = message.metadata;
+				assert(Object.keys(rest).length === 0, 0x5dd /* cannot group ops with metadata */);
 			}
 		}
 
 		const serializedContent = JSON.stringify({
 			type: OpGroupingManager.groupedBatchOp,
-			contents: batch.content.map<IGroupedMessage>((message) => ({
+			contents: batch.messages.map<IGroupedMessage>((message) => ({
 				contents: message.contents === undefined ? undefined : JSON.parse(message.contents),
 				metadata: message.metadata,
 				compression: message.compression,
@@ -88,10 +89,12 @@ export class OpGroupingManager {
 
 		const groupedBatch: IBatch<[BatchMessage]> = {
 			...batch,
-			content: [
+			messages: [
 				{
 					metadata: undefined,
-					referenceSequenceNumber: batch.content[0].referenceSequenceNumber,
+					// TODO why are we non null asserting here?
+					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+					referenceSequenceNumber: batch.messages[0]!.referenceSequenceNumber,
 					contents: serializedContent,
 				},
 			],
@@ -118,7 +121,7 @@ export class OpGroupingManager {
 			// Grouped batching must be enabled
 			this.config.groupedBatchingEnabled &&
 			// The number of ops in the batch must surpass the configured threshold
-			batch.content.length >= this.config.opCountThreshold &&
+			batch.messages.length >= this.config.opCountThreshold &&
 			// Support for reentrant batches must be explicitly enabled
 			(this.config.reentrantBatchGroupingEnabled || batch.hasReentrantOps !== true)
 		);

package/src/opLifecycle/opSplitter.ts

@@ -121,7 +121,7 @@ export class OpSplitter {
 	public splitFirstBatchMessage(batch: IBatch): IBatch {
 		assert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);
 		assert(
-			batch.contentSizeInBytes > 0 && batch.content.length > 0,
+			batch.contentSizeInBytes > 0 && batch.messages.length > 0,
 			0x514 /* Batch needs to be non-empty */,
 		);
 		assert(
@@ -134,13 +134,15 @@ export class OpSplitter {
 			0x516 /* Chunk size needs to be smaller than the max batch size */,
 		);
 
-		const firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split
+		// Non null asserting here because of the length check above
+		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+		const firstMessage = batch.messages[0]!; // we expect this to be the large compressed op, which needs to be split
 		assert(
 			(firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,
 			0x518 /* First message in the batch needs to be chunkable */,
 		);
 
-		const restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers
+		const restOfMessages = batch.messages.slice(1); // we expect these to be empty ops, created to reserve sequence numbers
 		const socketSize = estimateSocketSize(batch);
 		const chunks = splitOp(
 			firstMessage,
@@ -163,7 +165,9 @@ export class OpSplitter {
 		// The last chunk will be part of the new batch and needs to
 		// preserve the batch metadata of the original batch
 		const lastChunk = chunkToBatchMessage(
-			chunks[chunks.length - 1],
+			// Non null asserting here because of the length assert above
+			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+			chunks[chunks.length - 1]!,
 			batch.referenceSequenceNumber,
 			{ batch: firstMessage.metadata?.batch },
 		);
@@ -171,7 +175,7 @@ export class OpSplitter {
 		this.logger.sendPerformanceEvent({
 			// Used to be "Chunked compressed batch"
 			eventName: "CompressedChunkedBatch",
-			length: batch.content.length,
+			length: batch.messages.length,
 			sizeInBytes: batch.contentSizeInBytes,
 			chunks: chunks.length,
 			chunkSizeInBytes: this.chunkSizeInBytes,
@@ -179,7 +183,7 @@ export class OpSplitter {
 		});
 
 		return {
-			content: [lastChunk, ...restOfMessages],
+			messages: [lastChunk, ...restOfMessages],
 			contentSizeInBytes: lastChunk.contents?.length ?? 0,
 			referenceSequenceNumber: batch.referenceSequenceNumber,
 		};

package/src/opLifecycle/outbox.ts

@@ -9,19 +9,20 @@ import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
 import { assert } from "@fluidframework/core-utils/internal";
 import {
 	GenericError,
-	MonitoringContext,
 	UsageError,
-	createChildMonitoringContext,
+	createChildLogger,
+	type ITelemetryLoggerExt,
 } from "@fluidframework/telemetry-utils/internal";
 
 import { ICompressionRuntimeOptions } from "../containerRuntime.js";
-import { IPendingBatchMessage, PendingStateManager } from "../pendingStateManager.js";
+import { PendingMessageResubmitData, PendingStateManager } from "../pendingStateManager.js";
 
 import {
 	BatchManager,
 	BatchSequenceNumbers,
 	estimateSocketSize,
 	sequenceNumbersMatch,
+	type BatchId,
 } from "./batchManager.js";
 import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions.js";
 import { OpCompressor } from "./opCompressor.js";
@@ -48,7 +49,7 @@ export interface IOutboxParameters {
 	readonly logger: ITelemetryBaseLogger;
 	readonly groupingManager: OpGroupingManager;
 	readonly getCurrentSequenceNumbers: () => BatchSequenceNumbers;
-	readonly reSubmit: (message: IPendingBatchMessage) => void;
+	readonly reSubmit: (message: PendingMessageResubmitData) => void;
 	readonly opReentrancy: () => boolean;
 	readonly closeContainer: (error?: ICriticalContainerError) => void;
 }
@@ -87,8 +88,15 @@ export function getLongStack<T>(action: () => T, length: number = 50): T {
 	}
 }
 
+/**
+ * The Outbox collects messages submitted by the ContainerRuntime into a batch,
+ * and then flushes the batch when requested.
+ *
+ * @remarks There are actually multiple independent batches (some are for a specific message type),
+ * to support slight variation in semantics for each batch (e.g. support for rebasing or grouping).
+ */
 export class Outbox {
-	private readonly mc: MonitoringContext;
+	private readonly logger: ITelemetryLoggerExt;
 	private readonly mainBatch: BatchManager;
 	private readonly blobAttachBatch: BatchManager;
 	private readonly idAllocationBatch: BatchManager;
@@ -105,7 +113,8 @@ export class Outbox {
 	private mismatchedOpsReported = 0;
 
 	constructor(private readonly params: IOutboxParameters) {
-		this.mc = createChildMonitoringContext({ logger: params.logger, namespace: "Outbox" });
+		this.logger = createChildLogger({ logger: params.logger, namespace: "Outbox" });
+
 		const isCompressionEnabled =
 			this.params.config.compressionOptions.minimumBatchSizeInBytes !==
 			Number.POSITIVE_INFINITY;
@@ -158,7 +167,7 @@ export class Outbox {
 		}
 
 		if (++this.mismatchedOpsReported <= this.maxMismatchedOpsToReport) {
-			this.mc.logger.sendTelemetryEvent(
+			this.logger.sendTelemetryEvent(
 				{
 					category: this.params.config.disablePartialFlush ? "error" : "generic",
 					eventName: "ReferenceSequenceNumberMismatch",
@@ -225,28 +234,48 @@ export class Outbox {
 		}
 	}
 
-	public flush() {
+	/**
+	 * Flush all the batches to the ordering service.
+	 * This method is expected to be called at the end of a batch.
+	 * @param resubmittingBatchId - If defined, indicates this is a resubmission of a batch
+	 * with the given Batch ID, which must be preserved
+	 */
+	public flush(resubmittingBatchId?: BatchId) {
 		if (this.isContextReentrant()) {
 			const error = new UsageError("Flushing is not supported inside DDS event handlers");
 			this.params.closeContainer(error);
 			throw error;
 		}
 
-		this.flushAll();
+		this.flushAll(resubmittingBatchId);
 	}
 
-	private flushAll() {
+	private flushAll(resubmittingBatchId?: BatchId) {
+		// Don't use resubmittingBatchId for idAllocationBatch.
+		// ID Allocation messages are not directly resubmitted so we don't want to reuse the batch ID.
 		this.flushInternal(this.idAllocationBatch);
-		this.flushInternal(this.blobAttachBatch, true /* disableGroupedBatching */);
-		this.flushInternal(this.mainBatch);
+		this.flushInternal(
+			this.blobAttachBatch,
+			true /* disableGroupedBatching */,
+			resubmittingBatchId,
+		);
+		this.flushInternal(
+			this.mainBatch,
+			false /* disableGroupedBatching */,
+			resubmittingBatchId,
+		);
 	}
 
-	private flushInternal(batchManager: BatchManager, disableGroupedBatching: boolean = false) {
+	private flushInternal(
+		batchManager: BatchManager,
+		disableGroupedBatching: boolean = false,
+		resubmittingBatchId?: BatchId,
+	) {
 		if (batchManager.empty) {
 			return;
 		}
 
-		const rawBatch = batchManager.popBatch();
+		const rawBatch = batchManager.popBatch(resubmittingBatchId);
 		const shouldGroup =
 			!disableGroupedBatching && this.params.groupingManager.shouldGroup(rawBatch);
 		if (batchManager.options.canRebase && rawBatch.hasReentrantOps === true && shouldGroup) {
@@ -267,9 +296,13 @@ export class Outbox {
 				shouldGroup ? this.params.groupingManager.groupBatch(rawBatch) : rawBatch,
 			);
 			clientSequenceNumber = this.sendBatch(processedBatch);
+			assert(
+				clientSequenceNumber === undefined || clientSequenceNumber >= 0,
+				"unexpected negative clientSequenceNumber (empty batch should yield undefined)",
+			);
 		}
 
-		this.params.pendingStateManager.onFlushBatch(rawBatch.content, clientSequenceNumber);
+		this.params.pendingStateManager.onFlushBatch(rawBatch.messages, clientSequenceNumber);
 	}
 
 	/**
@@ -283,7 +316,7 @@ export class Outbox {
 		assert(batchManager.options.canRebase, 0x9a7 /* BatchManager does not support rebase */);
 
 		this.rebasing = true;
-		for (const message of rawBatch.content) {
+		for (const message of rawBatch.messages) {
 			this.params.reSubmit({
 				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 				content: message.contents!,
@@ -293,10 +326,10 @@ export class Outbox {
 		}
 
 		if (this.batchRebasesToReport > 0) {
-			this.mc.logger.sendTelemetryEvent(
+			this.logger.sendTelemetryEvent(
 				{
 					eventName: "BatchRebase",
-					length: rawBatch.content.length,
+					length: rawBatch.messages.length,
 					referenceSequenceNumber: rawBatch.referenceSequenceNumber,
 				},
 				new UsageError("BatchRebase"),
@@ -323,7 +356,7 @@ export class Outbox {
 	 */
 	private compressBatch(batch: IBatch): IBatch {
 		if (
-			batch.content.length === 0 ||
+			batch.messages.length === 0 ||
 			this.params.config.compressionOptions === undefined ||
 			this.params.config.compressionOptions.minimumBatchSizeInBytes >
 				batch.contentSizeInBytes ||
@@ -345,7 +378,7 @@ export class Outbox {
 			throw new GenericError("BatchTooLarge", /* error */ undefined, {
 				batchSize: batch.contentSizeInBytes,
 				compressedBatchSize: compressedBatch.contentSizeInBytes,
-				count: compressedBatch.content.length,
+				count: compressedBatch.messages.length,
 				limit: this.params.config.maxBatchSizeInBytes,
 				chunkingEnabled: this.params.splitter.isBatchChunkingEnabled,
 				compressionOptions: JSON.stringify(this.params.config.compressionOptions),
@@ -363,16 +396,16 @@ export class Outbox {
 	 * @returns the clientSequenceNumber of the start of the batch, or undefined if nothing was sent
 	 */
 	private sendBatch(batch: IBatch) {
-		const length = batch.content.length;
+		const length = batch.messages.length;
 		if (length === 0) {
 			return undefined; // Nothing submitted
 		}
 
 		const socketSize = estimateSocketSize(batch);
 		if (socketSize >= this.params.config.maxBatchSizeInBytes) {
-			this.mc.logger.sendPerformanceEvent({
+			this.logger.sendPerformanceEvent({
 				eventName: "LargeBatch",
-				length: batch.content.length,
+				length: batch.messages.length,
 				sizeInBytes: batch.contentSizeInBytes,
 				socketSize,
 			});
@@ -383,7 +416,9 @@ export class Outbox {
 			// Legacy path - supporting old loader versions. Can be removed only when LTS moves above
 			// version that has support for batches (submitBatchFn)
 			assert(
-				batch.content[0].compression === undefined,
+				// Non null asserting here because of the length check above
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				batch.messages[0]!.compression === undefined,
 				0x5a6 /* Compression should not have happened if the loader does not support it */,
 			);
 
@@ -391,7 +426,7 @@ export class Outbox {
 		} else {
 			assert(batch.referenceSequenceNumber !== undefined, 0x58e /* Batch must not be empty */);
 			clientSequenceNumber = this.params.submitBatchFn(
-				batch.content.map((message) => ({
+				batch.messages.map<IBatchMessage>((message) => ({
 					contents: message.contents,
 					metadata: message.metadata,
 					compression: message.compression,
@@ -407,7 +442,10 @@ export class Outbox {
 		return clientSequenceNumber;
 	}
 
-	public checkpoint() {
+	/**
+	 * @returns A checkpoint object per batch that facilitates iterating over the batch messages when rolling back.
+	 */
+	public getBatchCheckpoints() {
 		// This variable is declared with a specific type so that we have a standard import of the IBatchCheckpoint type.
 		// When the type is inferred, the generated .d.ts uses a dynamic import which doesn't resolve.
 		const mainBatch: IBatchCheckpoint = this.mainBatch.checkpoint();

package/src/opLifecycle/remoteMessageProcessor.ts

@@ -3,6 +3,7 @@
  * Licensed under the MIT License.
  */
 
+import { assert } from "@fluidframework/core-utils/internal";
 import {
 	MessageType,
 	ISequencedDocumentMessage,
@@ -12,9 +13,9 @@ import {
 	ContainerMessageType,
 	type InboundContainerRuntimeMessage,
 	type InboundSequencedContainerRuntimeMessage,
-	type InboundSequencedContainerRuntimeMessageOrSystemMessage,
 	type InboundSequencedRecentlyAddedContainerRuntimeMessage,
 } from "../messageTypes.js";
+import { asBatchMetadata } from "../metadata.js";
 
 import { OpDecompressor } from "./opDecompressor.js";
 import { OpGroupingManager, isGroupedBatch } from "./opGroupingManager.js";
@@ -27,6 +28,15 @@ import { OpSplitter, isChunkedMessage } from "./opSplitter.js";
  * @internal
  */
 export class RemoteMessageProcessor {
+	/**
+	 * Client Sequence Number of the first message in the current batch being processed.
+	 * If undefined, we are expecting the next message to start a new batch.
+	 *
+	 * @remarks For chunked batches, this is the CSN of the "representative" chunk (the final chunk)
+	 */
+	private batchStartCsn: number | undefined;
+	private readonly processorBatch: InboundSequencedContainerRuntimeMessage[] = [];
+
 	constructor(
 		private readonly opSplitter: OpSplitter,
 		private readonly opDecompressor: OpDecompressor,
@@ -42,7 +52,7 @@ export class RemoteMessageProcessor {
 	}
 
 	/**
-	 * Ungroups and Unchunks the runtime ops encapsulated by the single remoteMessage received over the wire
+	 * Ungroups and Unchunks the runtime ops of a batch received over the wire
 	 * @param remoteMessageCopy - A shallow copy of a message from another client, possibly virtualized
 	 * (grouped, compressed, and/or chunked).
 	 * Being a shallow copy, it's considered mutable, meaning no other Container or other parallel procedure
@@ -57,21 +67,24 @@ export class RemoteMessageProcessor {
 	 * 3. If grouped, ungroup the message
 	 * For more details, see https://github.com/microsoft/FluidFramework/blob/main/packages/runtime/container-runtime/src/opLifecycle/README.md#inbound
 	 *
-	 * @returns the unchunked, decompressed, ungrouped, unpacked SequencedContainerRuntimeMessages encapsulated in the remote message.
-	 * For ops that weren't virtualized (e.g. System ops that the ContainerRuntime will ultimately ignore),
-	 * a singleton array [remoteMessageCopy] is returned
+	 * @returns all the unchunked, decompressed, ungrouped, unpacked InboundSequencedContainerRuntimeMessage from a single batch
+	 * or undefined if the batch is not yet complete.
 	 */
-	public process(
-		remoteMessageCopy: ISequencedDocumentMessage,
-	): InboundSequencedContainerRuntimeMessageOrSystemMessage[] {
+	public process(remoteMessageCopy: ISequencedDocumentMessage):
+		| {
+				messages: InboundSequencedContainerRuntimeMessage[];
+				batchStartCsn: number;
+		  }
+		| undefined {
 		let message = remoteMessageCopy;
+
 		ensureContentsDeserialized(message);
 
 		if (isChunkedMessage(message)) {
 			const chunkProcessingResult = this.opSplitter.processChunk(message);
 			// Only continue further if current chunk is the final chunk
 			if (!chunkProcessingResult.isFinalChunk) {
-				return [];
+				return;
 			}
 			// This message will always be compressed
 			message = chunkProcessingResult.message;
@@ -90,12 +103,72 @@ export class RemoteMessageProcessor {
 		}
 
 		if (isGroupedBatch(message)) {
-			return this.opGroupingManager.ungroupOp(message).map(unpack);
+			// We should be awaiting a new batch (batchStartCsn undefined)
+			assert(this.batchStartCsn === undefined, "Grouped batch interrupting another batch");
+			assert(
+				this.processorBatch.length === 0,
+				"Processor batch should be empty on grouped batch",
+			);
+			return {
+				messages: this.opGroupingManager.ungroupOp(message).map(unpack),
+				batchStartCsn: message.clientSequenceNumber,
+			};
 		}
 
+		const batchStartCsn = this.getAndUpdateBatchStartCsn(message);
+
 		// Do a final unpack of runtime messages in case the message was not grouped, compressed, or chunked
 		unpackRuntimeMessage(message);
-		return [message as InboundSequencedContainerRuntimeMessageOrSystemMessage];
+		this.processorBatch.push(message as InboundSequencedContainerRuntimeMessage);
+
+		// this.batchStartCsn is undefined only if we have processed all messages in the batch.
+		// If it's still defined, we're still in the middle of a batch, so we return nothing, letting
+		// containerRuntime know that we're waiting for more messages to complete the batch.
+		if (this.batchStartCsn !== undefined) {
+			// batch not yet complete
+			return undefined;
+		}
+
+		const messages = [...this.processorBatch];
+		this.processorBatch.length = 0;
+		return {
+			messages,
+			batchStartCsn,
+		};
+	}
+
+	/**
+	 * Based on pre-existing batch tracking info and the current message's batch metadata,
+	 * this will return the starting CSN for this message's batch, and will also update
+	 * the batch tracking info (this.batchStartCsn) based on whether we're still mid-batch.
+	 */
+	private getAndUpdateBatchStartCsn(message: ISequencedDocumentMessage): number {
+		const batchMetadataFlag = asBatchMetadata(message.metadata)?.batch;
+		if (this.batchStartCsn === undefined) {
+			// We are waiting for a new batch
+			assert(batchMetadataFlag !== false, "Unexpected batch end marker");
+
+			// Start of a new multi-message batch
+			if (batchMetadataFlag === true) {
+				this.batchStartCsn = message.clientSequenceNumber;
+				return this.batchStartCsn;
+			}
+
+			// Single-message batch (Since metadata flag is undefined)
+			// IMPORTANT: Leave this.batchStartCsn undefined, we're ready for the next batch now.
+			return message.clientSequenceNumber;
+		}
+
+		// We are in the middle or end of an existing multi-message batch. Return the current batchStartCsn
+		const batchStartCsn = this.batchStartCsn;
+
+		assert(batchMetadataFlag !== true, "Unexpected batch start marker");
+		if (batchMetadataFlag === false) {
+			// Batch end? Then get ready for the next batch to start
+			this.batchStartCsn = undefined;
+		}
+
+		return batchStartCsn;
 	}
 }
 

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.1.0-276326";
+export const pkgVersion = "2.1.0-281041";