@fluidframework/container-runtime 2.0.0-dev.3.1.0.125672 → 2.0.0-dev.4.2.0.153917

package/src/opLifecycle/opSplitter.ts

@@ -13,6 +13,7 @@ import {
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { ChildLogger } from "@fluidframework/telemetry-utils";
 import { ContainerMessageType, ContainerRuntimeMessage } from "../containerRuntime";
+import { estimateSocketSize } from "./batchManager";
 import { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from "./definitions";
 
 /**
@@ -25,8 +26,10 @@ export class OpSplitter {
 
 	constructor(
 		chunks: [string, string[]][],
-		private readonly submitBatchFn: ((batch: IBatchMessage[]) => number) | undefined,
-		private readonly chunkSizeInBytes: number,
+		private readonly submitBatchFn:
+			| ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number)
+			| undefined,
+		public readonly chunkSizeInBytes: number,
 		private readonly maxBatchSizeInBytes: number,
 		logger: ITelemetryLogger,
 	) {
@@ -132,12 +135,16 @@ export class OpSplitter {
 	 * @param batch - the compressed batch which needs to be processed
 	 * @returns A new adjusted batch which can be sent over the wire
 	 */
-	public splitCompressedBatch(batch: IBatch): IBatch {
+	public splitFirstBatchMessage(batch: IBatch): IBatch {
 		assert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);
 		assert(
 			batch.contentSizeInBytes > 0 && batch.content.length > 0,
 			0x514 /* Batch needs to be non-empty */,
 		);
+		assert(
+			batch.referenceSequenceNumber !== undefined,
+			0x58a /* Batch must have a reference sequence number if non-empty */,
+		);
 		assert(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);
 		assert(
 			this.chunkSizeInBytes < this.maxBatchSizeInBytes,
@@ -145,43 +152,53 @@ export class OpSplitter {
 		);
 
 		const firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split
-		assert(
-			firstMessage.metadata?.compressed === true || firstMessage.compression !== undefined,
-			0x517 /* Batch needs to be compressed */,
-		);
 		assert(
 			(firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,
 			0x518 /* First message in the batch needs to be chunkable */,
 		);
 
 		const restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers
-		const chunks = splitOp(firstMessage, this.chunkSizeInBytes);
+		const socketSize = estimateSocketSize(batch);
+		const chunks = splitOp(
+			firstMessage,
+			this.chunkSizeInBytes,
+			// If we estimate that the socket batch size will exceed the batch limit
+			// we will inject an empty op to minimize the risk of the payload failing due to
+			// the overhead from the trailing empty ops in the batch.
+			socketSize >= this.maxBatchSizeInBytes,
+		);
 
 		assert(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);
 		// Send the first N-1 chunks immediately
 		for (const chunk of chunks.slice(0, -1)) {
-			this.submitBatchFn([chunkToBatchMessage(chunk, firstMessage.referenceSequenceNumber)]);
+			this.submitBatchFn(
+				[chunkToBatchMessage(chunk, batch.referenceSequenceNumber)],
+				batch.referenceSequenceNumber,
+			);
 		}
 
 		// The last chunk will be part of the new batch and needs to
 		// preserve the batch metadata of the original batch
 		const lastChunk = chunkToBatchMessage(
 			chunks[chunks.length - 1],
-			firstMessage.referenceSequenceNumber,
+			batch.referenceSequenceNumber,
 			{ batch: firstMessage.metadata?.batch },
 		);
 
 		this.logger.sendPerformanceEvent({
-			eventName: "Chunked compressed batch",
+			// Used to be "Chunked compressed batch"
+			eventName: "CompressedChunkedBatch",
 			length: batch.content.length,
 			sizeInBytes: batch.contentSizeInBytes,
 			chunks: chunks.length,
 			chunkSizeInBytes: this.chunkSizeInBytes,
+			socketSize,
 		});
 
 		return {
 			content: [lastChunk, ...restOfMessages],
 			contentSizeInBytes: lastChunk.contents?.length ?? 0,
+			referenceSequenceNumber: batch.referenceSequenceNumber,
 		};
 	}
 }
@@ -204,7 +221,23 @@ const chunkToBatchMessage = (
 	};
 };
 
-export const splitOp = (op: BatchMessage, chunkSizeInBytes: number): IChunkedOp[] => {
+/**
+ * Splits an op into smaller ops (chunks), based on the size of the op and the `chunkSizeInBytes` parameter.
+ *
+ * The last op of the result will be bundled with empty ops in the same batch. There is a risk of the batch payload
+ * exceeding the 1MB limit due to the overhead from the empty ops. If the last op is large, the risk is even higher.
+ * To minimize the odds, an extra empty op can be added to the result using the `extraOp` parameter.
+ *
+ * @param op - the op to be split
+ * @param chunkSizeInBytes - how large should the chunks be
+ * @param extraOp - should an extra empty op be added to the result
+ * @returns an array of chunked ops
+ */
+export const splitOp = (
+	op: BatchMessage,
+	chunkSizeInBytes: number,
+	extraOp: boolean = false,
+): IChunkedOp[] => {
 	const chunks: IChunkedOp[] = [];
 	assert(
 		op.contents !== undefined && op.contents !== null,
@@ -212,17 +245,17 @@ export const splitOp = (op: BatchMessage, chunkSizeInBytes: number): IChunkedOp[
 	);
 
 	const contentLength = op.contents.length;
-	const chunkN = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1;
+	const chunkCount = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1 + (extraOp ? 1 : 0);
 	let offset = 0;
-	for (let i = 1; i <= chunkN; i++) {
+	for (let chunkId = 1; chunkId <= chunkCount; chunkId++) {
 		const chunk: IChunkedOp = {
-			chunkId: i,
+			chunkId,
 			contents: op.contents.substr(offset, chunkSizeInBytes),
 			originalType: op.deserializedContent.type,
-			totalChunks: chunkN,
+			totalChunks: chunkCount,
 		};
 
-		if (i === chunkN) {
+		if (chunkId === chunkCount) {
 			// We don't need to port these to all the chunks,
 			// as we rebuild the original op when we process the
 			// last chunk, therefore it is the only one that needs it.
@@ -232,7 +265,13 @@ export const splitOp = (op: BatchMessage, chunkSizeInBytes: number): IChunkedOp[
 
 		chunks.push(chunk);
 		offset += chunkSizeInBytes;
+		assert(
+			chunkId >= chunkCount - 1 || offset <= contentLength,
+			0x58b /* Content offset within bounds */,
+		);
 	}
 
+	assert(offset >= contentLength, 0x58c /* Content offset equal or larger than content length */);
+	assert(chunks.length === chunkCount, 0x5a5 /* Expected number of chunks */);
 	return chunks;
 };

package/src/opLifecycle/outbox.ts

@@ -6,20 +6,26 @@
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { assert } from "@fluidframework/common-utils";
 import { IContainerContext } from "@fluidframework/container-definitions";
-import { GenericError } from "@fluidframework/container-utils";
+import { GenericError, UsageError } from "@fluidframework/container-utils";
 import { MessageType } from "@fluidframework/protocol-definitions";
+import {
+	ChildLogger,
+	loggerToMonitoringContext,
+	MonitoringContext,
+} from "@fluidframework/telemetry-utils";
 import { ICompressionRuntimeOptions } from "../containerRuntime";
 import { PendingStateManager } from "../pendingStateManager";
-import { BatchManager } from "./batchManager";
+import { BatchManager, estimateSocketSize } from "./batchManager";
 import { BatchMessage, IBatch } from "./definitions";
 import { OpCompressor } from "./opCompressor";
+import { OpGroupingManager } from "./opGroupingManager";
 import { OpSplitter } from "./opSplitter";
 
 export interface IOutboxConfig {
 	readonly compressionOptions: ICompressionRuntimeOptions;
 	// The maximum size of a batch that we can send over the wire.
 	readonly maxBatchSizeInBytes: number;
-	readonly enableOpReentryCheck?: boolean;
+	readonly disablePartialFlush: boolean;
 }
 
 export interface IOutboxParameters {
@@ -30,14 +36,26 @@ export interface IOutboxParameters {
 	readonly compressor: OpCompressor;
 	readonly splitter: OpSplitter;
 	readonly logger: ITelemetryLogger;
+	readonly groupingManager: OpGroupingManager;
 }
 
 export class Outbox {
+	private readonly mc: MonitoringContext;
 	private readonly attachFlowBatch: BatchManager;
 	private readonly mainBatch: BatchManager;
-	private readonly defaultAttachFlowSoftLimitInBytes = 64 * 1024;
+	private readonly defaultAttachFlowSoftLimitInBytes = 320 * 1024;
+
+	/**
+	 * Track the number of ops which were detected to have a mismatched
+	 * reference sequence number, in order to self-throttle the telemetry events.
+	 *
+	 * This should be removed as part of ADO:2322
+	 */
+	private readonly maxMismatchedOpsToReport = 3;
+	private mismatchedOpsReported = 0;
 
 	constructor(private readonly params: IOutboxParameters) {
+		this.mc = loggerToMonitoringContext(ChildLogger.create(params.logger, "Outbox"));
 		const isCompressionEnabled =
 			this.params.config.compressionOptions.minimumBatchSizeInBytes !==
 			Number.POSITIVE_INFINITY;
@@ -45,28 +63,63 @@ export class Outbox {
 		const hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;
 		const softLimit = isCompressionEnabled ? Infinity : this.defaultAttachFlowSoftLimitInBytes;
 
-		this.attachFlowBatch = new BatchManager(
-			{
-				hardLimit,
-				softLimit,
-				enableOpReentryCheck: params.config.enableOpReentryCheck,
-			},
-			params.logger,
-		);
-		this.mainBatch = new BatchManager(
-			{
-				hardLimit,
-				enableOpReentryCheck: params.config.enableOpReentryCheck,
-			},
-			params.logger,
-		);
+		this.attachFlowBatch = new BatchManager({ hardLimit, softLimit });
+		this.mainBatch = new BatchManager({ hardLimit });
 	}
 
 	public get isEmpty(): boolean {
 		return this.attachFlowBatch.length === 0 && this.mainBatch.length === 0;
 	}
 
+	/**
+	 * If we detect that the reference sequence number of the incoming message does not match
+	 * what was already in the batch managers, this means that batching has been interrupted so
+	 * we will flush the accumulated messages to account for that and create a new batch with the new
+	 * message as the first message.
+	 *
+	 * @param message - the incoming message
+	 */
+	private maybeFlushPartialBatch(message: BatchMessage) {
+		const mainBatchReference = this.mainBatch.referenceSequenceNumber;
+		const attachFlowBatchReference = this.attachFlowBatch.referenceSequenceNumber;
+		assert(
+			this.params.config.disablePartialFlush ||
+				mainBatchReference === undefined ||
+				attachFlowBatchReference === undefined ||
+				mainBatchReference === attachFlowBatchReference,
+			0x58d /* Reference sequence numbers from both batches must be in sync */,
+		);
+
+		if (
+			(mainBatchReference === undefined ||
+				mainBatchReference === message.referenceSequenceNumber) &&
+			(attachFlowBatchReference === undefined ||
+				attachFlowBatchReference === message.referenceSequenceNumber)
+		) {
+			// The reference sequence numbers are stable, there is nothing to do
+			return;
+		}
+
+		if (++this.mismatchedOpsReported <= this.maxMismatchedOpsToReport) {
+			this.mc.logger.sendErrorEvent(
+				{
+					eventName: "ReferenceSequenceNumberMismatch",
+					mainReferenceSequenceNumber: mainBatchReference,
+					attachReferenceSequenceNumber: attachFlowBatchReference,
+					messageReferenceSequenceNumber: message.referenceSequenceNumber,
+				},
+				new UsageError("Submission of an out of order message"),
+			);
+		}
+
+		if (!this.params.config.disablePartialFlush) {
+			this.flush();
+		}
+	}
+
 	public submit(message: BatchMessage) {
+		this.maybeFlushPartialBatch(message);
+
 		if (!this.mainBatch.push(message)) {
 			throw new GenericError("BatchTooLarge", /* error */ undefined, {
 				opSize: message.contents?.length ?? 0,
@@ -78,6 +131,8 @@ export class Outbox {
 	}
 
 	public submitAttach(message: BatchMessage) {
+		this.maybeFlushPartialBatch(message);
+
 		if (!this.attachFlowBatch.push(message)) {
 			// BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
 			// when queue is not empty.
@@ -113,69 +168,82 @@ export class Outbox {
 
 	private flushInternal(rawBatch: IBatch) {
 		const processedBatch = this.compressBatch(rawBatch);
-		const clientSequenceNumber = this.sendBatch(processedBatch);
+		this.sendBatch(processedBatch);
 
-		this.persistBatch(clientSequenceNumber, rawBatch.content);
+		this.persistBatch(rawBatch.content);
 	}
 
 	private compressBatch(batch: IBatch): IBatch {
 		if (
 			batch.content.length === 0 ||
 			this.params.config.compressionOptions === undefined ||
-			this.params.config.compressionOptions.minimumBatchSizeInBytes > batch.contentSizeInBytes
+			this.params.config.compressionOptions.minimumBatchSizeInBytes >
+				batch.contentSizeInBytes ||
+			this.params.containerContext.submitBatchFn === undefined
 		) {
-			// Nothing to do if the batch is empty or if compression is disabled or if we don't need to compress
-			return batch;
+			// Nothing to do if the batch is empty or if compression is disabled or not supported, or if we don't need to compress
+			return this.params.groupingManager.groupBatch(batch);
 		}
 
-		const compressedBatch = this.params.compressor.compressBatch(batch);
-		if (compressedBatch.contentSizeInBytes <= this.params.config.maxBatchSizeInBytes) {
-			// If we don't reach the maximum supported size of a batch, it can safely be sent as is
-			return compressedBatch;
-		}
+		const compressedBatch = this.params.groupingManager.groupBatch(
+			this.params.compressor.compressBatch(batch),
+		);
 
 		if (this.params.splitter.isBatchChunkingEnabled) {
-			return this.params.splitter.splitCompressedBatch(compressedBatch);
+			return compressedBatch.contentSizeInBytes <= this.params.splitter.chunkSizeInBytes
+				? compressedBatch
+				: this.params.splitter.splitFirstBatchMessage(compressedBatch);
+		}
+
+		if (compressedBatch.contentSizeInBytes >= this.params.config.maxBatchSizeInBytes) {
+			throw new GenericError("BatchTooLarge", /* error */ undefined, {
+				batchSize: batch.contentSizeInBytes,
+				compressedBatchSize: compressedBatch.contentSizeInBytes,
+				count: compressedBatch.content.length,
+				limit: this.params.config.maxBatchSizeInBytes,
+				chunkingEnabled: this.params.splitter.isBatchChunkingEnabled,
+				compressionOptions: JSON.stringify(this.params.config.compressionOptions),
+				socketSize: estimateSocketSize(batch),
+			});
 		}
 
-		// If we've reached this point, the runtime would attempt to send a batch larger than the allowed size
-		throw new GenericError("BatchTooLarge", /* error */ undefined, {
-			batchSize: batch.contentSizeInBytes,
-			compressedBatchSize: compressedBatch.contentSizeInBytes,
-			count: compressedBatch.content.length,
-			limit: this.params.config.maxBatchSizeInBytes,
-			chunkingEnabled: this.params.splitter.isBatchChunkingEnabled,
-			compressionOptions: JSON.stringify(this.params.config.compressionOptions),
-		});
+		return compressedBatch;
 	}
 
 	/**
 	 * Sends the batch object to the container context to be sent over the wire.
 	 *
 	 * @param batch - batch to be sent
-	 * @returns the client sequence number of the last batched op which was sent and
-	 * -1 if there are no ops or the container cannot send ops.
 	 */
-	private sendBatch(batch: IBatch): number {
-		let clientSequenceNumber: number = -1;
+	private sendBatch(batch: IBatch) {
 		const length = batch.content.length;
 
 		// Did we disconnect in the middle of turn-based batch?
 		// If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
 		if (length === 0 || !this.params.shouldSend()) {
-			return clientSequenceNumber;
+			return;
+		}
+
+		const socketSize = estimateSocketSize(batch);
+		if (socketSize >= this.params.config.maxBatchSizeInBytes) {
+			this.mc.logger.sendPerformanceEvent({
+				eventName: "LargeBatch",
+				length: batch.content.length,
+				sizeInBytes: batch.contentSizeInBytes,
+				socketSize,
+			});
 		}
 
 		if (this.params.containerContext.submitBatchFn === undefined) {
 			// Legacy path - supporting old loader versions. Can be removed only when LTS moves above
 			// version that has support for batches (submitBatchFn)
-			for (const message of batch.content) {
-				// Legacy path doesn't support compressed payloads and will submit uncompressed payload anyways
-				if (message.metadata?.compressed) {
-					delete message.metadata.compressed;
-				}
+			assert(
+				batch.content[0].compression === undefined,
+				0x5a6 /* Compression should not have happened if the loader does not support it */,
+			);
 
-				clientSequenceNumber = this.params.containerContext.submitFn(
+			for (const message of batch.content) {
+				this.params.containerContext.submitFn(
 					MessageType.Operation,
 					message.deserializedContent,
 					true, // batch
@@ -185,37 +253,33 @@ export class Outbox {
 
 			this.params.containerContext.deltaManager.flush();
 		} else {
-			// returns clientSequenceNumber of last message in a batch
-			clientSequenceNumber = this.params.containerContext.submitBatchFn(
+			assert(
+				batch.referenceSequenceNumber !== undefined,
+				0x58e /* Batch must not be empty */,
+			);
+			this.params.containerContext.submitBatchFn(
 				batch.content.map((message) => ({
 					contents: message.contents,
 					metadata: message.metadata,
 					compression: message.compression,
+					referenceSequenceNumber: message.referenceSequenceNumber,
 				})),
+				batch.referenceSequenceNumber,
 			);
 		}
-
-		// Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.
-		clientSequenceNumber -= length - 1;
-		assert(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);
-		return clientSequenceNumber;
 	}
 
-	private persistBatch(initialClientSequenceNumber: number, batch: BatchMessage[]) {
-		let clientSequenceNumber = initialClientSequenceNumber;
+	private persistBatch(batch: BatchMessage[]) {
 		// Let the PendingStateManager know that a message was submitted.
 		// In future, need to shift toward keeping batch as a whole!
 		for (const message of batch) {
 			this.params.pendingStateManager.onSubmitMessage(
 				message.deserializedContent.type,
-				clientSequenceNumber,
 				message.referenceSequenceNumber,
 				message.deserializedContent.contents,
 				message.localOpMetadata,
 				message.metadata,
 			);
-
-			clientSequenceNumber++;
 		}
 	}
 

package/src/opLifecycle/remoteMessageProcessor.ts

@@ -6,12 +6,14 @@
 import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
 import { ContainerMessageType, ContainerRuntimeMessage } from "../containerRuntime";
 import { OpDecompressor } from "./opDecompressor";
+import { OpGroupingManager } from "./opGroupingManager";
 import { OpSplitter } from "./opSplitter";
 
 export class RemoteMessageProcessor {
 	constructor(
 		private readonly opSplitter: OpSplitter,
 		private readonly opDecompressor: OpDecompressor,
+		private readonly opGroupingManager: OpGroupingManager,
 	) {}
 
 	public get partialMessages(): ReadonlyMap<string, string[]> {
@@ -22,30 +24,44 @@ export class RemoteMessageProcessor {
 		this.opSplitter.clearPartialChunks(clientId);
 	}
 
-	public process(remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage {
-		let message = copy(remoteMessage);
-		message = this.opDecompressor.processMessage(message).message;
-		unpackRuntimeMessage(message);
-
-		const chunkProcessingResult = this.opSplitter.processRemoteMessage(message);
-		message = chunkProcessingResult.message;
-		if (chunkProcessingResult.state !== "Processed") {
-			// If the message is not chunked or if the splitter is still rebuilding the original message,
-			// there is no need to continue processing
-			return message;
-		}
+	public process(remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage[] {
+		const result: ISequencedDocumentMessage[] = [];
 
-		const decompressionAfterChunking = this.opDecompressor.processMessage(message);
-		message = decompressionAfterChunking.message;
-		if (decompressionAfterChunking.state === "Skipped") {
-			// After chunking, if the original message was not compressed,
-			// there is no need to continue processing
-			return message;
-		}
+		// Ungroup before processing chunks
+		for (let ungroupedMessage of this.opGroupingManager.ungroupOp(copy(remoteMessage))) {
+			ungroupedMessage = this.opDecompressor.processMessage(ungroupedMessage).message;
+			unpackRuntimeMessage(ungroupedMessage);
 
-		// The message needs to be unpacked after chunking + decompression
-		unpack(message);
-		return message;
+			const chunkProcessingResult = this.opSplitter.processRemoteMessage(ungroupedMessage);
+			ungroupedMessage = chunkProcessingResult.message;
+			if (chunkProcessingResult.state !== "Processed") {
+				// If the message is not chunked or if the splitter is still rebuilding the original message,
+				// there is no need to continue processing
+				result.push(ungroupedMessage);
+				continue;
+			}
+
+			// Ungroup the chunked message before decompressing
+			for (let ungroupedMessageAfterChunking of this.opGroupingManager.ungroupOp(
+				ungroupedMessage,
+			)) {
+				const decompressionAfterChunking = this.opDecompressor.processMessage(
+					ungroupedMessageAfterChunking,
+				);
+				ungroupedMessageAfterChunking = decompressionAfterChunking.message;
+				if (decompressionAfterChunking.state === "Skipped") {
+					// After chunking, if the original message was not compressed,
+					// there is no need to continue processing
+					result.push(ungroupedMessageAfterChunking);
+					continue;
+				}
+
+				// The message needs to be unpacked after chunking + decompression
+				unpack(ungroupedMessageAfterChunking);
+				result.push(ungroupedMessageAfterChunking);
+			}
+		}
+		return result;
 	}
 }
 

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-dev.3.1.0.125672";
+export const pkgVersion = "2.0.0-dev.4.2.0.153917";

package/src/pendingStateManager.ts

@@ -29,12 +29,13 @@ export interface IPendingMessage {
 /**
  * This represents an explicit flush call and is added to the pending queue when flush is called on the ContainerRuntime
  * to flush pending messages.
- * @deprecated Use batch metadata on IPendingMessage instead. To be removed in 2.0.0-internal.4.0.0 (AB#2496)
+ * ! TODO: Remove in "2.0.0-internal.5.0.0" AB#2496
  */
 export interface IPendingFlush {
 	type: "flush";
 }
 
+/** ! TODO: Remove in "2.0.0-internal.5.0.0" AB#2496 */
 export type IPendingState = IPendingMessage | IPendingFlush;
 
 export interface IPendingLocalState {
@@ -107,17 +108,11 @@ export class PendingStateManager implements IDisposable {
 		);
 		if (!this.pendingMessages.isEmpty()) {
 			return {
-				pendingStates: this.pendingMessages.toArray().reduce((arr, message) => {
-					// delete localOpMetadata since it may not be serializable
-					// and will be regenerated by applyStashedOp()
-					arr.push({ ...message, localOpMetadata: undefined });
-
-					// TODO: Remove in 2.0.0-internal.4.0.0 (AB#2496)
-					if (message.opMetadata?.batch === false) {
-						arr.push({ type: "flush" });
-					}
-					return arr;
-				}, new Array<IPendingState>()),
+				// delete localOpMetadata since it may not be serializable
+				// and will be regenerated by applyStashedOp()
+				pendingStates: this.pendingMessages
+					.toArray()
+					.map((message) => ({ ...message, localOpMetadata: undefined })),
 			};
 		}
 	}
@@ -130,7 +125,7 @@ export class PendingStateManager implements IDisposable {
 		 * Convert old local state format to the new format
 		 * The old format contained "flush" messages as the indicator of batch ends
 		 * The new format instead uses batch metadata on the last message to indicate batch ends
-		 * ! TODO: Remove this conversion in "2.0.0-internal.4.0.0" as rollback from future version will be new format
+		 * ! TODO: Remove this conversion in "2.0.0-internal.5.0.0" as version from "2.0.0-internal.4.0.0" will be new format
 		 * AB#2496 tracks removal
 		 */
 		if (initialLocalState?.pendingStates) {
@@ -168,13 +163,11 @@ export class PendingStateManager implements IDisposable {
 	 * Called when a message is submitted locally. Adds the message and the associated details to the pending state
 	 * queue.
 	 * @param type - The container message type.
-	 * @param clientSequenceNumber - The clientSequenceNumber associated with the message.
 	 * @param content - The message content.
 	 * @param localOpMetadata - The local metadata associated with the message.
 	 */
 	public onSubmitMessage(
 		type: ContainerMessageType,
-		clientSequenceNumber: number,
 		referenceSequenceNumber: number,
 		content: any,
 		localOpMetadata: unknown,
@@ -183,7 +176,7 @@ export class PendingStateManager implements IDisposable {
 		const pendingMessage: IPendingMessage = {
 			type: "message",
 			messageType: type,
-			clientSequenceNumber,
+			clientSequenceNumber: -1, // dummy value (not to be used anywhere)
 			referenceSequenceNumber,
 			content,
 			localOpMetadata,
@@ -241,18 +234,34 @@ export class PendingStateManager implements IDisposable {
 		);
 		this.pendingMessages.shift();
 
-		// Processing part - Verify that there has been no data corruption.
-		// The clientSequenceNumber of the incoming message must match that of the pending message.
-		if (pendingMessage.clientSequenceNumber !== message.clientSequenceNumber) {
+		if (pendingMessage.messageType !== message.type) {
 			// Close the container because this could indicate data corruption.
-			const error = DataProcessingError.create(
-				"pending local message clientSequenceNumber mismatch",
-				"unexpectedAckReceived",
-				message,
-				{ expectedClientSequenceNumber: pendingMessage.clientSequenceNumber },
+			this.stateHandler.close(
+				DataProcessingError.create(
+					"pending local message type mismatch",
+					"unexpectedAckReceived",
+					message,
+					{
+						expectedMessageType: pendingMessage.messageType,
+					},
+				),
 			);
+			return;
+		}
 
-			this.stateHandler.close(error);
+		const pendingMessageContent = JSON.stringify(pendingMessage.content);
+		const messageContent = JSON.stringify(message.contents);
+
+		// Stringified content does not match
+		if (pendingMessageContent !== messageContent) {
+			// Close the container because this could indicate data corruption.
+			this.stateHandler.close(
+				DataProcessingError.create(
+					"pending local message content mismatch",
+					"unexpectedAckReceived",
+					message,
+				),
+			);
 			return;
 		}
 
@@ -322,8 +331,6 @@ export class PendingStateManager implements IDisposable {
 								hasBatchStart: batchBeginMetadata === true,
 								hasBatchEnd: batchEndMetadata === false,
 								messageType: message.type,
-								batchStartSequenceNumber:
-									this.pendingBatchBeginMessage.clientSequenceNumber,
 								pendingMessagesCount: this.pendingMessagesCount,
 							},
 						),