@fluidframework/container-runtime 2.32.0 → 2.33.0

package/src/opLifecycle/index.ts

@@ -7,7 +7,6 @@ export {
 	BatchId,
 	BatchManager,
 	BatchSequenceNumbers,
-	estimateSocketSize,
 	getEffectiveBatchId,
 	generateBatchId,
 	IBatchManagerOptions,
@@ -15,6 +14,7 @@ export {
 export {
 	LocalBatch,
 	LocalBatchMessage,
+	LocalEmptyBatchPlaceholder,
 	OutboundBatch,
 	OutboundBatchMessage,
 	OutboundSingletonBatch,
@@ -26,7 +26,12 @@ export {
 	serializeOp,
 	ensureContentsDeserialized,
 } from "./opSerialization.js";
-export { Outbox, getLongStack } from "./outbox.js";
+export {
+	estimateSocketSize,
+	localBatchToOutboundBatch,
+	Outbox,
+	getLongStack,
+} from "./outbox.js";
 export { OpCompressor } from "./opCompressor.js";
 export { OpDecompressor } from "./opDecompressor.js";
 export { OpSplitter, splitOp, isChunkedMessage } from "./opSplitter.js";

package/src/opLifecycle/opCompressor.ts

@@ -13,10 +13,10 @@ import {
 } from "@fluidframework/telemetry-utils/internal";
 import { compress } from "lz4js";
 
-import { CompressionAlgorithms } from "../containerRuntime.js";
+import { CompressionAlgorithms } from "../compressionDefinitions.js";
 
-import { estimateSocketSize } from "./batchManager.js";
 import { type OutboundBatchMessage, type OutboundSingletonBatch } from "./definitions.js";
+import { estimateSocketSize } from "./outbox.js";
 
 /**
  * Compresses batches of ops.

package/src/opLifecycle/opDecompressor.ts

@@ -13,7 +13,7 @@ import {
 } from "@fluidframework/telemetry-utils/internal";
 import { decompress } from "lz4js";
 
-import { CompressionAlgorithms } from "../containerRuntime.js";
+import { CompressionAlgorithms } from "../compressionDefinitions.js";
 import { IBatchMetadata } from "../metadata.js";
 
 /**

package/src/opLifecycle/opGroupingManager.ts

@@ -12,8 +12,8 @@ import {
 } from "@fluidframework/telemetry-utils/internal";
 
 import {
+	type LocalEmptyBatchPlaceholder,
 	type OutboundBatch,
-	type OutboundBatchMessage,
 	type OutboundSingletonBatch,
 } from "./definitions.js";
 
@@ -67,7 +67,10 @@ export class OpGroupingManager {
 	public createEmptyGroupedBatch(
 		resubmittingBatchId: string,
 		referenceSequenceNumber: number,
-	): { outboundBatch: OutboundSingletonBatch; placeholderMessage: OutboundBatchMessage } {
+	): {
+		outboundBatch: OutboundSingletonBatch;
+		placeholderMessage: LocalEmptyBatchPlaceholder;
+	} {
 		assert(
 			this.config.groupedBatchingEnabled,
 			0xa00 /* cannot create empty grouped batch when grouped batching is disabled */,
@@ -77,15 +80,14 @@ export class OpGroupingManager {
 			contents: [],
 		});
 
-		const placeholderMessage: OutboundBatchMessage = {
+		const placeholderMessage: LocalEmptyBatchPlaceholder = {
 			metadata: { batchId: resubmittingBatchId },
 			localOpMetadata: { emptyBatch: true },
 			referenceSequenceNumber,
-			contents: serializedOp,
 		};
 		const outboundBatch: OutboundSingletonBatch = {
 			contentSizeInBytes: 0,
-			messages: [placeholderMessage],
+			messages: [{ ...placeholderMessage, contents: serializedOp }],
 			referenceSequenceNumber,
 		};
 		return { outboundBatch, placeholderMessage };

package/src/opLifecycle/opSerialization.ts

@@ -30,10 +30,14 @@ export function ensureContentsDeserialized(mutableMessage: ISequencedDocumentMes
 /**
  * Before submitting an op to the Outbox, its contents must be serialized using this function.
  * @remarks - The deserialization on process happens via the function {@link ensureContentsDeserialized}.
+ *
+ * @param toSerialize - op message to serialize. Also supports an array of ops.
  */
-export function serializeOp(op: LocalContainerRuntimeMessage): string {
+export function serializeOp(
+	toSerialize: LocalContainerRuntimeMessage | LocalContainerRuntimeMessage[],
+): string {
 	return JSON.stringify(
-		op,
+		toSerialize,
 		// replacer:
 		(key, value: unknown) => {
 			// If 'value' is an IFluidHandle return its encoded form.

package/src/opLifecycle/opSplitter.ts

@@ -16,12 +16,12 @@ import {
 
 import { ContainerMessageType, ContainerRuntimeChunkedOpMessage } from "../messageTypes.js";
 
-import { estimateSocketSize } from "./batchManager.js";
 import {
 	IChunkedOp,
 	type OutboundBatchMessage,
 	type OutboundSingletonBatch,
 } from "./definitions.js";
+import { estimateSocketSize } from "./outbox.js";
 
 export function isChunkedMessage(message: ISequencedDocumentMessage): boolean {
 	return isChunkedContents(message.contents);

package/src/opLifecycle/outbox.ts

@@ -4,23 +4,25 @@
  */
 
 import { IBatchMessage } from "@fluidframework/container-definitions/internal";
-import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import {
+	ITelemetryBaseLogger,
+	type ITelemetryBaseProperties,
+} from "@fluidframework/core-interfaces";
 import { assert, Lazy } from "@fluidframework/core-utils/internal";
 import {
 	DataProcessingError,
-	GenericError,
 	UsageError,
 	createChildLogger,
+	type IFluidErrorBase,
 	type ITelemetryLoggerExt,
 } from "@fluidframework/telemetry-utils/internal";
 
-import { ICompressionRuntimeOptions } from "../containerRuntime.js";
+import { ICompressionRuntimeOptions } from "../compressionDefinitions.js";
 import { PendingMessageResubmitData, PendingStateManager } from "../pendingStateManager.js";
 
 import {
 	BatchManager,
 	BatchSequenceNumbers,
-	estimateSocketSize,
 	sequenceNumbersMatch,
 	type BatchId,
 } from "./batchManager.js";
@@ -34,6 +36,7 @@ import {
 } from "./definitions.js";
 import { OpCompressor } from "./opCompressor.js";
 import { OpGroupingManager } from "./opGroupingManager.js";
+import { serializeOp } from "./opSerialization.js";
 import { OpSplitter } from "./opSplitter.js";
 
 export interface IOutboxConfig {
@@ -106,6 +109,61 @@ export function getLongStack<T>(action: () => T, length: number = 50): T {
 	}
 }
 
+/**
+ * Convert from local batch to outbound batch, including computing contentSizeInBytes.
+ */
+export function localBatchToOutboundBatch({
+	staged: _staged, // Peel this off the incoming batch, it's irrelevant (see Note below)
+	...localBatch
+}: LocalBatch): OutboundBatch {
+	// Note: the staged property might be misleading here, in case a pre-staging batch is resubmitted during staging mode.
+	// It will be set to true, but the batch was not actually staged.
+
+	// Shallow copy each message as we switch types
+	const outboundMessages = localBatch.messages.map<OutboundBatchMessage>(
+		({ runtimeOp, ...message }) => ({
+			contents: serializeOp(runtimeOp),
+			...message,
+		}),
+	);
+	const contentSizeInBytes = outboundMessages.reduce(
+		(acc, message) => acc + (message.contents?.length ?? 0),
+		0,
+	);
+
+	// Shallow copy the local batch, updating the messages to be outbound messages and adding contentSizeInBytes
+	const outboundBatch: OutboundBatch = {
+		...localBatch,
+		messages: outboundMessages,
+		contentSizeInBytes,
+	};
+
+	return outboundBatch;
+}
+
+/**
+ * Estimated size of the stringification overhead for an op accumulated
+ * from runtime to loader to the service.
+ */
+const opOverhead = 200;
+
+/**
+ * Estimates the real size in bytes on the socket for a given batch. It assumes that
+ * the envelope size (and the size of an empty op) is 200 bytes, taking into account
+ * extra overhead from stringification.
+ *
+ * @remarks
+ * Also content will be stringified, and that adds a lot of overhead due to a lot of escape characters.
+ * Not taking it into account, as compression work should help there - compressed payload will be
+ * initially stored as base64, and that requires only 2 extra escape characters.
+ *
+ * @param batch - the batch to inspect
+ * @returns An estimate of the payload size in bytes which will be produced when the batch is sent over the wire
+ */
+export const estimateSocketSize = (batch: OutboundBatch): number => {
+	return batch.contentSizeInBytes + opOverhead * batch.messages.length;
+};
+
 /**
  * The Outbox collects messages submitted by the ContainerRuntime into a batch,
  * and then flushes the batch when requested.
@@ -133,18 +191,9 @@ export class Outbox {
 	constructor(private readonly params: IOutboxParameters) {
 		this.logger = createChildLogger({ logger: params.logger, namespace: "Outbox" });
 
-		const isCompressionEnabled =
-			this.params.config.compressionOptions.minimumBatchSizeInBytes !==
-			Number.POSITIVE_INFINITY;
-		// We need to allow infinite size batches if we enable compression
-		const hardLimit = isCompressionEnabled
-			? Number.POSITIVE_INFINITY
-			: this.params.config.maxBatchSizeInBytes;
-
-		this.mainBatch = new BatchManager({ hardLimit, canRebase: true });
-		this.blobAttachBatch = new BatchManager({ hardLimit, canRebase: true });
+		this.mainBatch = new BatchManager({ canRebase: true });
+		this.blobAttachBatch = new BatchManager({ canRebase: true });
 		this.idAllocationBatch = new BatchManager({
-			hardLimit,
 			canRebase: false,
 			ignoreBatchId: true,
 		});
@@ -224,7 +273,7 @@ export class Outbox {
 							? "generic"
 							: "error",
 					eventName: "ReferenceSequenceNumberMismatch",
-					Data_details: {
+					details: {
 						expectedDueToReentrancy,
 						mainReferenceSequenceNumber: mainBatchSeqNums.referenceSequenceNumber,
 						mainClientSequenceNumber: mainBatchSeqNums.clientSequenceNumber,
@@ -274,20 +323,11 @@ export class Outbox {
 		batchManager: BatchManager,
 		message: LocalBatchMessage,
 	): void {
-		if (
-			!batchManager.push(
-				message,
-				this.isContextReentrant(),
-				this.params.getCurrentSequenceNumbers().clientSequenceNumber,
-			)
-		) {
-			throw new GenericError("BatchTooLarge", /* error */ undefined, {
-				opSize: message.serializedOp?.length ?? 0,
-				batchSize: batchManager.contentSizeInBytes,
-				count: batchManager.length,
-				limit: batchManager.options.hardLimit,
-			});
-		}
+		batchManager.push(
+			message,
+			this.isContextReentrant(),
+			this.params.getCurrentSequenceNumbers().clientSequenceNumber,
+		);
 	}
 
 	/**
@@ -297,17 +337,19 @@ export class Outbox {
 	 * @throws If called from a reentrant context, or if the batch being flushed is too large.
 	 * @param resubmittingBatchId - If defined, indicates this is a resubmission of a batch
 	 * with the given Batch ID, which must be preserved
+	 * @param resubmittingStagedBatch - If defined, indicates this is a resubmission of a batch that is staged,
+	 * meaning it should not be sent to the ordering service yet.
 	 */
-	public flush(resubmittingBatchId?: BatchId): void {
+	public flush(resubmittingBatchId?: BatchId, resubmittingStagedBatch?: boolean): void {
 		assert(
 			!this.isContextReentrant(),
 			0xb7b /* Flushing must not happen while incoming changes are being processed */,
 		);
 
-		this.flushAll(resubmittingBatchId);
+		this.flushAll(resubmittingBatchId, resubmittingStagedBatch);
 	}
 
-	private flushAll(resubmittingBatchId?: BatchId): void {
+	private flushAll(resubmittingBatchId?: BatchId, resubmittingStagedBatch?: boolean): void {
 		const allBatchesEmpty =
 			this.idAllocationBatch.empty && this.blobAttachBatch.empty && this.mainBatch.empty;
 		if (allBatchesEmpty) {
@@ -317,27 +359,35 @@ export class Outbox {
 			// by the rest of the system, including remote clients.
 			// In some cases we *must* resubmit the empty batch (to match up with a non-empty version tracked locally by a container fork), so we do it always.
 			if (resubmittingBatchId) {
-				this.flushEmptyBatch(resubmittingBatchId);
+				this.flushEmptyBatch(resubmittingBatchId, resubmittingStagedBatch === true);
 			}
 			return;
 		}
 
 		// Don't use resubmittingBatchId for idAllocationBatch.
 		// ID Allocation messages are not directly resubmitted so we don't want to reuse the batch ID.
-		this.flushInternal(this.idAllocationBatch);
-		this.flushInternal(
-			this.blobAttachBatch,
-			true /* disableGroupedBatching */,
+		this.flushInternal({
+			batchManager: this.idAllocationBatch,
+			// Note: For now, we will never stage ID Allocation messages.
+			// They won't contain personal info and no harm in extra allocations in case of discarding the staged changes
+		});
+		this.flushInternal({
+			batchManager: this.blobAttachBatch,
+			disableGroupedBatching: true,
 			resubmittingBatchId,
-		);
-		this.flushInternal(
-			this.mainBatch,
-			false /* disableGroupedBatching */,
+			resubmittingStagedBatch,
+		});
+		this.flushInternal({
+			batchManager: this.mainBatch,
 			resubmittingBatchId,
-		);
+			resubmittingStagedBatch,
+		});
 	}
 
-	private flushEmptyBatch(resubmittingBatchId: BatchId): void {
+	private flushEmptyBatch(
+		resubmittingBatchId: BatchId,
+		resubmittingStagedBatch: boolean,
+	): void {
 		const referenceSequenceNumber =
 			this.params.getCurrentSequenceNumbers().referenceSequenceNumber;
 		assert(
@@ -350,28 +400,42 @@ export class Outbox {
 				referenceSequenceNumber,
 			);
 		let clientSequenceNumber: number | undefined;
-		if (this.params.shouldSend()) {
+		if (this.params.shouldSend() && !resubmittingStagedBatch) {
 			clientSequenceNumber = this.sendBatch(outboundBatch);
 		}
 
 		// Push the empty batch placeholder to the PendingStateManager
-		this.params.pendingStateManager.onFlushBatch(
-			[{ ...placeholderMessage, serializedOp: "", contents: undefined }], // placeholder message - serializedOp will never be used
+		this.params.pendingStateManager.onFlushEmptyBatch(
+			placeholderMessage,
 			clientSequenceNumber,
+			resubmittingStagedBatch,
 		);
 		return;
 	}
 
-	private flushInternal(
-		batchManager: BatchManager,
-		disableGroupedBatching: boolean = false,
-		resubmittingBatchId?: BatchId,
-	): void {
+	private flushInternal(params: {
+		batchManager: BatchManager;
+		disableGroupedBatching?: boolean;
+		resubmittingBatchId?: BatchId; // undefined if not resubmitting
+		resubmittingStagedBatch?: boolean; // undefined if not resubmitting
+	}): void {
+		const {
+			batchManager,
+			disableGroupedBatching = false,
+			resubmittingBatchId,
+			resubmittingStagedBatch,
+		} = params;
 		if (batchManager.empty) {
 			return;
 		}
 
 		const rawBatch = batchManager.popBatch(resubmittingBatchId);
+
+		// When resubmitting, we respect the staged state of the original batch.
+		// In this case rawBatch.staged will match the state of inStagingMode when
+		// the resubmit occurred, which is not relevant.
+		const staged = resubmittingStagedBatch ?? rawBatch.staged === true;
+
 		const groupingEnabled =
 			!disableGroupedBatching && this.params.groupingManager.groupedBatchingEnabled();
 		if (
@@ -386,6 +450,9 @@ export class Outbox {
 			// If a batch contains reentrant ops (ops created as a result from processing another op)
 			// it needs to be rebased so that we can ensure consistent reference sequence numbers
 			// and eventual consistency at the DDS level.
+			// Note: Since this is happening in the same turn the ops were originally created with,
+			// and they haven't gone to PendingStateManager yet, we can just let them respect
+			// ContainerRuntime.inStagingMode
 			this.rebase(rawBatch, batchManager);
 			return;
 		}
@@ -394,7 +461,7 @@ export class Outbox {
 		// Did we disconnect? (i.e. is shouldSend false?)
 		// If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
 		// Because flush() is a task that executes async (on clean stack), we can get here in disconnected state.
-		if (this.params.shouldSend()) {
+		if (this.params.shouldSend() && !staged) {
 			const virtualizedBatch = this.virtualizeBatch(rawBatch, groupingEnabled);
 
 			clientSequenceNumber = this.sendBatch(virtualizedBatch);
@@ -407,6 +474,7 @@ export class Outbox {
 		this.params.pendingStateManager.onFlushBatch(
 			rawBatch.messages,
 			clientSequenceNumber,
+			staged,
 			batchManager.options.ignoreBatchId,
 		);
 	}
@@ -424,7 +492,7 @@ export class Outbox {
 		this.rebasing = true;
 		for (const message of rawBatch.messages) {
 			this.params.reSubmit({
-				content: message.serializedOp,
+				runtimeOp: message.runtimeOp,
 				localOpMetadata: message.localOpMetadata,
 				opMetadata: message.metadata,
 			});
@@ -442,7 +510,7 @@ export class Outbox {
 			this.batchRebasesToReport--;
 		}
 
-		this.flushInternal(batchManager);
+		this.flushInternal({ batchManager });
 		this.rebasing = false;
 	}
 
@@ -465,15 +533,7 @@ export class Outbox {
 	 */
 	private virtualizeBatch(localBatch: LocalBatch, groupingEnabled: boolean): OutboundBatch {
 		// Shallow copy the local batch, updating the messages to be outbound messages
-		const originalBatch: OutboundBatch = {
-			...localBatch,
-			messages: localBatch.messages.map<OutboundBatchMessage>(
-				({ serializedOp, ...message }) => ({
-					contents: serializedOp,
-					...message,
-				}),
-			),
-		};
+		const originalBatch = localBatchToOutboundBatch(localBatch);
 
 		const originalOrGroupedBatch = groupingEnabled
 			? this.params.groupingManager.groupBatch(originalBatch)
@@ -489,7 +549,6 @@ export class Outbox {
 		const singletonBatch = originalOrGroupedBatch as OutboundSingletonBatch;
 
 		if (
-			this.params.config.compressionOptions === undefined ||
 			this.params.config.compressionOptions.minimumBatchSizeInBytes >
 				singletonBatch.contentSizeInBytes ||
 			this.params.submitBatchFn === undefined
@@ -506,21 +565,11 @@ export class Outbox {
 				: this.params.splitter.splitSingletonBatchMessage(compressedBatch);
 		}
 
+		// We want to distinguish this "BatchTooLarge" case from the generic "BatchTooLarge" case in sendBatch
 		if (compressedBatch.contentSizeInBytes >= this.params.config.maxBatchSizeInBytes) {
-			throw DataProcessingError.create(
-				"BatchTooLarge",
-				"compressionInsufficient",
-				/* sequencedMessage */ undefined,
-				{
-					batchSize: singletonBatch.contentSizeInBytes,
-					compressedBatchSize: compressedBatch.contentSizeInBytes,
-					count: compressedBatch.messages.length,
-					limit: this.params.config.maxBatchSizeInBytes,
-					chunkingEnabled: this.params.splitter.isBatchChunkingEnabled,
-					compressionOptions: JSON.stringify(this.params.config.compressionOptions),
-					socketSize: estimateSocketSize(singletonBatch),
-				},
-			);
+			throw this.makeBatchTooLargeError(compressedBatch, "CompressionInsufficient", {
+				uncompressedSizeInBytes: singletonBatch.contentSizeInBytes,
+			});
 		}
 
 		return compressedBatch;
@@ -540,12 +589,7 @@ export class Outbox {
 
 		const socketSize = estimateSocketSize(batch);
 		if (socketSize >= this.params.config.maxBatchSizeInBytes) {
-			this.logger.sendPerformanceEvent({
-				eventName: "LargeBatch",
-				length: batch.messages.length,
-				sizeInBytes: batch.contentSizeInBytes,
-				socketSize,
-			});
+			throw this.makeBatchTooLargeError(batch, "CannotSend");
 		}
 
 		let clientSequenceNumber: number;
@@ -577,6 +621,31 @@ export class Outbox {
 		return clientSequenceNumber;
 	}
 
+	private makeBatchTooLargeError(
+		batch: OutboundBatch,
+		codepath: string,
+		moreDetails?: ITelemetryBaseProperties,
+	): IFluidErrorBase {
+		return DataProcessingError.create(
+			"BatchTooLarge",
+			codepath,
+			/* sequencedMessage */ undefined,
+			{
+				errorDetails: {
+					opCount: batch.messages.length,
+					contentSizeInBytes: batch.contentSizeInBytes,
+					socketSize: estimateSocketSize(batch),
+					maxBatchSizeInBytes: this.params.config.maxBatchSizeInBytes,
+					groupedBatchingEnabled: this.params.groupingManager.groupedBatchingEnabled(),
+					compressionOptions: JSON.stringify(this.params.config.compressionOptions),
+					chunkingEnabled: this.params.splitter.isBatchChunkingEnabled,
+					chunkSizeInBytes: this.params.splitter.chunkSizeInBytes,
+					...moreDetails,
+				},
+			},
+		);
+	}
+
 	/**
 	 * Gets a checkpoint object per batch that facilitates iterating over the batch messages when rolling back.
 	 */

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.32.0";
+export const pkgVersion = "2.33.0";