@fluidframework/container-runtime 2.0.0-rc.2.0.2 → 2.0.0-rc.3.0.0

package/src/opLifecycle/opDecompressor.ts

@@ -3,15 +3,15 @@
  * Licensed under the MIT License.
  */
 
-import { decompress } from "lz4js";
-import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { assert } from "@fluidframework/core-utils";
 import { IsoBuffer, Uint8ArrayToString } from "@fluid-internal/client-utils";
-import { createChildLogger } from "@fluidframework/telemetry-utils";
 import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { assert } from "@fluidframework/core-utils/internal";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { createChildLogger } from "@fluidframework/telemetry-utils/internal";
+import { decompress } from "lz4js";
+
 import { CompressionAlgorithms } from "../containerRuntime.js";
 import { IBatchMetadata } from "../metadata.js";
-import { IMessageProcessingResult } from "./definitions.js";
 
 /**
  * Compression makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.
@@ -38,100 +38,7 @@ export class OpDecompressor {
 		this.logger = createChildLogger({ logger, namespace: "OpDecompressor" });
 	}
 
-	public processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {
-		assert(
-			message.compression === undefined || message.compression === CompressionAlgorithms.lz4,
-			0x511 /* Only lz4 compression is supported */,
-		);
-
-		if (
-			(message.metadata as IBatchMetadata | undefined)?.batch === true &&
-			this.isCompressed(message)
-		) {
-			// Beginning of a compressed batch
-			assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
-			this.activeBatch = true;
-
-			const contents = IsoBuffer.from(
-				(message.contents as IPackedContentsContents).packedContents,
-				"base64",
-			);
-			const decompressedMessage = decompress(contents);
-			const intoString = Uint8ArrayToString(decompressedMessage);
-			const asObj = JSON.parse(intoString);
-			this.rootMessageContents = asObj;
-
-			return {
-				message: newMessage(message, this.rootMessageContents[this.processedCount++]),
-				state: "Accepted",
-			};
-		}
-
-		if (
-			this.rootMessageContents !== undefined &&
-			(message.metadata as IBatchMetadata | undefined)?.batch === undefined &&
-			this.activeBatch
-		) {
-			assert(message.contents === undefined, 0x512 /* Expecting empty message */);
-
-			// Continuation of compressed batch
-			return {
-				message: newMessage(message, this.rootMessageContents[this.processedCount++]),
-				state: "Accepted",
-			};
-		}
-
-		if (
-			this.rootMessageContents !== undefined &&
-			(message.metadata as IBatchMetadata | undefined)?.batch === false
-		) {
-			// End of compressed batch
-			const returnMessage = newMessage(
-				message,
-				this.rootMessageContents[this.processedCount++],
-			);
-
-			this.activeBatch = false;
-			this.rootMessageContents = undefined;
-			this.processedCount = 0;
-
-			return {
-				message: returnMessage,
-				state: "Processed",
-			};
-		}
-
-		if (
-			(message.metadata as IBatchMetadata | undefined)?.batch === undefined &&
-			this.isCompressed(message)
-		) {
-			// Single compressed message
-			assert(
-				this.activeBatch === false,
-				0x4ba /* shouldn't receive compressed message in middle of a batch */,
-			);
-
-			const contents = IsoBuffer.from(
-				(message.contents as IPackedContentsContents).packedContents,
-				"base64",
-			);
-			const decompressedMessage = decompress(contents);
-			const intoString = new TextDecoder().decode(decompressedMessage);
-			const asObj = JSON.parse(intoString);
-
-			return {
-				message: newMessage(message, asObj[0]),
-				state: "Processed",
-			};
-		}
-
-		return {
-			message,
-			state: "Skipped",
-		};
-	}
-
-	private isCompressed(message: ISequencedDocumentMessage) {
+	public isCompressedMessage(message: ISequencedDocumentMessage): boolean {
 		if (message.compression === CompressionAlgorithms.lz4) {
 			return true;
 		}
@@ -174,6 +81,86 @@ export class OpDecompressor {
 
 		return false;
 	}
+
+	public get currentlyUnrolling() {
+		return this.activeBatch;
+	}
+
+	/** Is the decompressed and stored batch only comprised of a single message */
+	private isSingleMessageBatch = false;
+
+	/**
+	 * Decompress the given compressed message and store it to be subsequently unrolled.
+	 * The stored message will be of type `any[]` where each element represents a message's `contents`
+	 */
+	public decompressAndStore(message: ISequencedDocumentMessage): void {
+		assert(
+			message.compression === undefined || message.compression === CompressionAlgorithms.lz4,
+			0x511 /* Only lz4 compression is supported */,
+		);
+		assert(
+			this.isCompressedMessage(message),
+			0x940 /* provided message should be compressed */,
+		);
+
+		assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
+		this.activeBatch = true;
+
+		const batchMetadata = (message.metadata as IBatchMetadata | undefined)?.batch;
+		if (batchMetadata === undefined) {
+			this.isSingleMessageBatch = true;
+		} else {
+			assert(batchMetadata === true, 0x941 /* invalid batch metadata */);
+		}
+
+		const contents = IsoBuffer.from(
+			(message.contents as IPackedContentsContents).packedContents,
+			"base64",
+		);
+		const decompressedMessage = decompress(contents);
+		const intoString = Uint8ArrayToString(decompressedMessage);
+		const asObj = JSON.parse(intoString);
+		this.rootMessageContents = asObj;
+	}
+
+	/**
+	 * Unroll the next message from the decompressed content provided to {@link decompressAndStore}
+	 * @returns the unrolled `ISequencedDocumentMessage`
+	 */
+	public unroll(message: ISequencedDocumentMessage): ISequencedDocumentMessage {
+		assert(this.currentlyUnrolling, 0x942 /* not currently unrolling */);
+		assert(this.rootMessageContents !== undefined, 0x943 /* missing rootMessageContents */);
+		assert(
+			this.rootMessageContents.length > this.processedCount,
+			0x944 /* no more content to unroll */,
+		);
+
+		const batchMetadata = (message.metadata as IBatchMetadata | undefined)?.batch;
+
+		if (batchMetadata === false || this.isSingleMessageBatch) {
+			// End of compressed batch
+			const returnMessage = newMessage(
+				message,
+				this.rootMessageContents[this.processedCount],
+			);
+
+			this.activeBatch = false;
+			this.isSingleMessageBatch = false;
+			this.rootMessageContents = undefined;
+			this.processedCount = 0;
+
+			return returnMessage;
+		} else if (batchMetadata === true) {
+			// Start of compressed batch
+			return newMessage(message, this.rootMessageContents[this.processedCount++]);
+		}
+
+		assert(batchMetadata === undefined, 0x945 /* invalid batch metadata */);
+		assert(message.contents === undefined, 0x512 /* Expecting empty message */);
+
+		// Continuation of compressed batch
+		return newMessage(message, this.rootMessageContents[this.processedCount++]);
+	}
 }
 
 // We should not be mutating the input message nor its metadata
@@ -186,5 +173,8 @@ const newMessage = (
 	compression: undefined,
 	// TODO: It should already be the case that we're not modifying any metadata, not clear if/why this shallow clone should be required.
 	// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
-	metadata: { ...(originalMessage.metadata as any) },
+	metadata:
+		originalMessage.metadata === undefined
+			? undefined
+			: { ...(originalMessage.metadata as any) },
 });

package/src/opLifecycle/opGroupingManager.ts

@@ -3,11 +3,11 @@
  * Licensed under the MIT License.
  */
 
-import { assert } from "@fluidframework/core-utils";
-import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { createChildLogger } from "@fluidframework/telemetry-utils";
 import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
-import { ContainerMessageType } from "../messageTypes.js";
+import { assert } from "@fluidframework/core-utils/internal";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { createChildLogger } from "@fluidframework/telemetry-utils/internal";
+
 import { IBatch } from "./definitions.js";
 
 /**
@@ -28,6 +28,10 @@ function isGroupContents(opContents: any): opContents is IGroupedBatchMessageCon
 	return opContents?.type === OpGroupingManager.groupedBatchOp;
 }
 
+export function isGroupedBatch(op: ISequencedDocumentMessage): boolean {
+	return isGroupContents(op.contents);
+}
+
 export interface OpGroupingManagerConfig {
 	readonly groupedBatchingEnabled: boolean;
 	readonly opCountThreshold: number;
@@ -46,9 +50,7 @@ export class OpGroupingManager {
 	}
 
 	public groupBatch(batch: IBatch): IBatch {
-		if (!this.shouldGroup(batch)) {
-			return batch;
-		}
+		assert(this.shouldGroup(batch), 0x946 /* cannot group the provided batch */);
 
 		if (batch.content.length >= 1000) {
 			this.logger.sendTelemetryEvent({
@@ -84,11 +86,9 @@ export class OpGroupingManager {
 			...batch,
 			content: [
 				{
-					localOpMetadata: undefined,
 					metadata: undefined,
 					referenceSequenceNumber: batch.content[0].referenceSequenceNumber,
 					contents: serializedContent,
-					type: OpGroupingManager.groupedBatchOp as ContainerMessageType,
 				},
 			],
 		};
@@ -96,13 +96,11 @@ export class OpGroupingManager {
 	}
 
 	public ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[] {
-		if (!isGroupContents(op.contents)) {
-			return [op];
-		}
+		assert(isGroupContents(op.contents), 0x947 /* can only ungroup a grouped batch */);
+		const contents: IGroupedBatchMessageContents = op.contents;
 
-		const messages = op.contents.contents;
 		let fakeCsn = 1;
-		return messages.map((subMessage) => ({
+		return contents.contents.map((subMessage) => ({
 			...op,
 			clientSequenceNumber: fakeCsn++,
 			contents: subMessage.contents,

package/src/opLifecycle/opSplitter.ts

@@ -3,18 +3,33 @@
  * Licensed under the MIT License.
  */
 
+import { IBatchMessage } from "@fluidframework/container-definitions/internal";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { assert } from "@fluidframework/core-utils/internal";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import {
-	createChildLogger,
 	DataCorruptionError,
+	createChildLogger,
 	extractSafePropertiesFromMessage,
-} from "@fluidframework/telemetry-utils";
-import { assert } from "@fluidframework/core-utils";
-import { IBatchMessage } from "@fluidframework/container-definitions";
-import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+} from "@fluidframework/telemetry-utils/internal";
+
 import { ContainerMessageType, ContainerRuntimeChunkedOpMessage } from "../messageTypes.js";
+
 import { estimateSocketSize } from "./batchManager.js";
-import { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from "./definitions.js";
+import { BatchMessage, IBatch, IChunkedOp } from "./definitions.js";
+
+export function isChunkedMessage(message: ISequencedDocumentMessage): boolean {
+	return isChunkedContents(message.contents);
+}
+
+interface IChunkedContents {
+	type: typeof ContainerMessageType.ChunkedOp;
+	contents: IChunkedOp;
+}
+
+function isChunkedContents(contents: any): contents is IChunkedContents {
+	return contents?.type === ContainerMessageType.ChunkedOp;
+}
 
 /**
  * Responsible for creating and reconstructing chunked messages.
@@ -45,44 +60,6 @@ export class OpSplitter {
 		return this.chunkMap;
 	}
 
-	public processRemoteMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {
-		if (message.type !== ContainerMessageType.ChunkedOp) {
-			return {
-				message,
-				state: "Skipped",
-			};
-		}
-
-		// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
-		// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
-		const clientId = message.clientId as string;
-		const chunkedContent = message.contents as IChunkedOp;
-		this.addChunk(clientId, chunkedContent, message);
-
-		if (chunkedContent.chunkId < chunkedContent.totalChunks) {
-			// We are processing the op in chunks but haven't reached
-			// the last chunk yet in order to reconstruct the original op
-			return {
-				message,
-				state: "Accepted",
-			};
-		}
-
-		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		const serializedContent = this.chunkMap.get(clientId)!.join("");
-		this.clearPartialChunks(clientId);
-
-		const newMessage = { ...message };
-		newMessage.contents = serializedContent === "" ? undefined : JSON.parse(serializedContent);
-		newMessage.type = chunkedContent.originalType;
-		newMessage.metadata = chunkedContent.originalMetadata;
-		newMessage.compression = chunkedContent.originalCompression;
-		return {
-			message: newMessage,
-			state: "Processed",
-		};
-	}
-
 	public clearPartialChunks(clientId: string) {
 		if (this.chunkMap.has(clientId)) {
 			this.chunkMap.delete(clientId);
@@ -203,8 +180,53 @@ export class OpSplitter {
 			referenceSequenceNumber: batch.referenceSequenceNumber,
 		};
 	}
+
+	public processChunk(message: ISequencedDocumentMessage): ProcessChunkResult {
+		assert(isChunkedContents(message.contents), 0x948 /* message not of type ChunkedOp */);
+		const contents: IChunkedContents = message.contents;
+
+		// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
+		// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+		const clientId = message.clientId as string;
+		const chunkedContent = contents.contents;
+		this.addChunk(clientId, chunkedContent, message);
+
+		if (chunkedContent.chunkId < chunkedContent.totalChunks) {
+			// We are processing the op in chunks but haven't reached
+			// the last chunk yet in order to reconstruct the original op
+			return {
+				isFinalChunk: false,
+			};
+		}
+
+		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+		const serializedContent = this.chunkMap.get(clientId)!.join("");
+		this.clearPartialChunks(clientId);
+
+		const newMessage = { ...message };
+		newMessage.contents = serializedContent === "" ? undefined : JSON.parse(serializedContent);
+		// back-compat with 1.x builds
+		// This is only required / present for non-compressed, chunked ops
+		// For compressed ops, we have op grouping enabled, and type of each op is preserved within compressed content.
+		newMessage.type = (chunkedContent as any).originalType;
+		newMessage.metadata = chunkedContent.originalMetadata;
+		newMessage.compression = chunkedContent.originalCompression;
+		return {
+			message: newMessage,
+			isFinalChunk: true,
+		};
+	}
 }
 
+type ProcessChunkResult =
+	| {
+			readonly isFinalChunk: false;
+	  }
+	| {
+			readonly isFinalChunk: true;
+			readonly message: ISequencedDocumentMessage;
+	  };
+
 const chunkToBatchMessage = (
 	chunk: IChunkedOp,
 	referenceSequenceNumber: number,
@@ -216,9 +238,7 @@ const chunkToBatchMessage = (
 	};
 	return {
 		contents: JSON.stringify(payload),
-		type: payload.type,
 		metadata,
-		localOpMetadata: undefined,
 		referenceSequenceNumber,
 	};
 };
@@ -253,7 +273,6 @@ export const splitOp = (
 		const chunk: IChunkedOp = {
 			chunkId,
 			contents: op.contents.substr(offset, chunkSizeInBytes),
-			originalType: op.type,
 			totalChunks: chunkCount,
 		};
 
@@ -263,6 +282,15 @@ export const splitOp = (
 			// last chunk, therefore it is the only one that needs it.
 			chunk.originalMetadata = op.metadata;
 			chunk.originalCompression = op.compression;
+
+			// back-compat with 1.x builds
+			// 2.x builds only do chunking for compressed ops.
+			// originalType is no longer used in such cases, as each op preserves its type within compressed payload.
+			// But, if 1.x builds see this op, and there is no type on the message, then it will ignore this message silently.
+			// This is really bad, as we will crash on later ops and it's very hard to debug these cases.
+			// If we put some known type here, then we will crash on it (as 1.x does not understand compression, and thus will not
+			// find info on the op like address of the channel to deliver the op)
+			(chunk as any).originalType = "component";
 		}
 
 		chunks.push(chunk);

package/src/opLifecycle/outbox.ts

@@ -3,18 +3,20 @@
  * Licensed under the MIT License.
  */
 
+import { ICriticalContainerError } from "@fluidframework/container-definitions";
+import { IBatchMessage } from "@fluidframework/container-definitions/internal";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { assert } from "@fluidframework/core-utils/internal";
 import {
-	createChildMonitoringContext,
 	GenericError,
 	MonitoringContext,
 	UsageError,
-} from "@fluidframework/telemetry-utils";
-import { assert } from "@fluidframework/core-utils";
-import { IBatchMessage, ICriticalContainerError } from "@fluidframework/container-definitions";
-import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+	createChildMonitoringContext,
+} from "@fluidframework/telemetry-utils/internal";
+
 import { ICompressionRuntimeOptions } from "../containerRuntime.js";
 import { IPendingBatchMessage, PendingStateManager } from "../pendingStateManager.js";
-import { ContainerMessageType } from "../messageTypes.js";
+
 import {
 	BatchManager,
 	BatchSequenceNumbers,
@@ -120,7 +122,12 @@ export class Outbox {
 	}
 
 	public get messageCount(): number {
-		return this.attachFlowBatch.length + this.mainBatch.length + this.blobAttachBatch.length;
+		return (
+			this.attachFlowBatch.length +
+			this.mainBatch.length +
+			this.blobAttachBatch.length +
+			this.idAllocationBatch.length
+		);
 	}
 
 	public get isEmpty(): boolean {
@@ -182,20 +189,12 @@ export class Outbox {
 	}
 
 	public submit(message: BatchMessage) {
-		assert(
-			message.type !== ContainerMessageType.IdAllocation,
-			0x8f8 /* Allocation message submitted to mainBatch. */,
-		);
 		this.maybeFlushPartialBatch();
 
 		this.addMessageToBatchManager(this.mainBatch, message);
 	}
 
 	public submitAttach(message: BatchMessage) {
-		assert(
-			message.type === ContainerMessageType.Attach,
-			0x8f9 /* Non attach message submitted to attachFlowBatch. */,
-		);
 		this.maybeFlushPartialBatch();
 
 		if (
@@ -227,10 +226,6 @@ export class Outbox {
 	}
 
 	public submitBlobAttach(message: BatchMessage) {
-		assert(
-			message.type === ContainerMessageType.BlobAttach,
-			0x8fa /* Non blobAttach message submitted to blobAttachBatch. */,
-		);
 		this.maybeFlushPartialBatch();
 
 		this.addMessageToBatchManager(this.blobAttachBatch, message);
@@ -249,10 +244,6 @@ export class Outbox {
 	}
 
 	public submitIdAllocation(message: BatchMessage) {
-		assert(
-			message.type === ContainerMessageType.IdAllocation,
-			0x8fb /* Non allocation message submitted to idAllocationBatch. */,
-		);
 		this.maybeFlushPartialBatch();
 
 		if (
@@ -323,10 +314,9 @@ export class Outbox {
 		}
 
 		const rawBatch = batchManager.popBatch();
-		if (
-			rawBatch.hasReentrantOps === true &&
-			this.params.groupingManager.shouldGroup(rawBatch)
-		) {
+		const shouldGroup =
+			!disableGroupedBatching && this.params.groupingManager.shouldGroup(rawBatch);
+		if (rawBatch.hasReentrantOps === true && shouldGroup) {
 			assert(!this.rebasing, 0x6fa /* A rebased batch should never have reentrant ops */);
 			// If a batch contains reentrant ops (ops created as a result from processing another op)
 			// it needs to be rebased so that we can ensure consistent reference sequence numbers
@@ -335,8 +325,15 @@ export class Outbox {
 			return;
 		}
 
-		const processedBatch = this.compressBatch(rawBatch, disableGroupedBatching);
-		this.sendBatch(processedBatch);
+		// Did we disconnect?
+		// If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
+		// Because flush() is a task that executes async (on clean stack), we can get here in disconnected state.
+		if (this.params.shouldSend()) {
+			const processedBatch = this.compressBatch(
+				shouldGroup ? this.params.groupingManager.groupBatch(rawBatch) : rawBatch,
+			);
+			this.sendBatch(processedBatch);
+		}
 
 		this.persistBatch(rawBatch.content);
 	}
@@ -380,7 +377,7 @@ export class Outbox {
 		return this.params.opReentrancy() && !this.rebasing;
 	}
 
-	private compressBatch(batch: IBatch, disableGroupedBatching: boolean): IBatch {
+	private compressBatch(batch: IBatch): IBatch {
 		if (
 			batch.content.length === 0 ||
 			this.params.config.compressionOptions === undefined ||
@@ -389,12 +386,10 @@ export class Outbox {
 			this.params.submitBatchFn === undefined
 		) {
 			// Nothing to do if the batch is empty or if compression is disabled or not supported, or if we don't need to compress
-			return disableGroupedBatching ? batch : this.params.groupingManager.groupBatch(batch);
+			return batch;
 		}
 
-		const compressedBatch = this.params.compressor.compressBatch(
-			disableGroupedBatching ? batch : this.params.groupingManager.groupBatch(batch),
-		);
+		const compressedBatch = this.params.compressor.compressBatch(batch);
 
 		if (this.params.splitter.isBatchChunkingEnabled) {
 			return compressedBatch.contentSizeInBytes <= this.params.splitter.chunkSizeInBytes
@@ -424,10 +419,7 @@ export class Outbox {
 	 */
 	private sendBatch(batch: IBatch) {
 		const length = batch.content.length;
-
-		// Did we disconnect in the middle of turn-based batch?
-		// If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
-		if (length === 0 || !this.params.shouldSend()) {
+		if (length === 0) {
 			return;
 		}