@fluidframework/container-runtime 2.0.0-dev.2.2.0.111723 → 2.0.0-dev.3.1.0.125672

package/src/opLifecycle/definitions.ts

@@ -0,0 +1,70 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { IBatchMessage } from "@fluidframework/container-definitions";
+import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
+import { CompressionAlgorithms, ContainerMessageType, ContainerRuntimeMessage } from "..";
+
+/**
+ * Batch message type used internally by the runtime
+ */
+export type BatchMessage = IBatchMessage & {
+	localOpMetadata: unknown;
+	deserializedContent: ContainerRuntimeMessage;
+	referenceSequenceNumber: number;
+	compression?: CompressionAlgorithms;
+};
+
+/**
+ * Batch interface used internally by the runtime.
+ */
+export interface IBatch {
+	/**
+	 * Sum of the in-memory content sizes of all messages in the batch.
+	 * If the batch is compressed, this number reflects the post-compression size.
+	 */
+	readonly contentSizeInBytes: number;
+	/**
+	 * All the messages in the batch
+	 */
+	readonly content: BatchMessage[];
+}
+
+export interface IBatchCheckpoint {
+	rollback: (action: (message: BatchMessage) => void) => void;
+}
+
+export interface IChunkedOp {
+	chunkId: number;
+	totalChunks: number;
+	contents: string;
+	originalType: MessageType | ContainerMessageType;
+	originalMetadata?: Record<string, unknown>;
+	originalCompression?: string;
+}
+
+/**
+ * The state of remote message processing:
+ * `Processed` - the message can be considered processed
+ * `Skipped` - the message was ignored by the processor
+ * `Accepted` - the message was processed partially. Eventually, a message
+ * will make the processor return `Processed`.
+ */
+export type ProcessingState = "Processed" | "Skipped" | "Accepted";
+
+/**
+ * Return type for functions which process remote messages
+ */
+export interface IMessageProcessingResult {
+	/**
+	 * A shallow copy of the input message if processing happened, or
+	 * the original message otherwise
+	 */
+	readonly message: ISequencedDocumentMessage;
+	/**
+	 * Processing result of the input message.
+	 */
+	readonly state: ProcessingState;
+}

package/src/opLifecycle/index.ts

@@ -0,0 +1,18 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export { BatchManager } from "./batchManager";
+export {
+	BatchMessage,
+	IBatch,
+	IBatchCheckpoint,
+	IChunkedOp,
+	IMessageProcessingResult,
+} from "./definitions";
+export { Outbox } from "./outbox";
+export { OpCompressor } from "./opCompressor";
+export { OpDecompressor } from "./opDecompressor";
+export { OpSplitter, splitOp } from "./opSplitter";
+export { RemoteMessageProcessor, unpackRuntimeMessage } from "./remoteMessageProcessor";

package/src/opLifecycle/opCompressor.ts

@@ -0,0 +1,82 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { IsoBuffer } from "@fluidframework/common-utils";
+import { UsageError } from "@fluidframework/container-utils";
+import { ChildLogger } from "@fluidframework/telemetry-utils";
+import { compress } from "lz4js";
+import { CompressionAlgorithms } from "../containerRuntime";
+import { IBatch, BatchMessage } from "./definitions";
+
+/**
+ * Compresses batches of ops. It generates a single compressed op that contains
+ * the contents of each op in the batch. It then submits empty ops for each original
+ * op to reserve sequence numbers.
+ */
+export class OpCompressor {
+	private readonly logger;
+
+	constructor(logger: ITelemetryLogger) {
+		this.logger = ChildLogger.create(logger, "OpCompressor");
+	}
+
+	public compressBatch(batch: IBatch): IBatch {
+		const compressionStart = Date.now();
+		const contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));
+		const compressedContents = compress(contentsAsBuffer);
+		const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
+		const duration = Date.now() - compressionStart;
+
+		if (batch.contentSizeInBytes > 200000) {
+			this.logger.sendPerformanceEvent({
+				eventName: "CompressedBatch",
+				duration,
+				sizeBeforeCompression: batch.contentSizeInBytes,
+				sizeAfterCompression: compressedContent.length,
+			});
+		}
+
+		const messages: BatchMessage[] = [];
+		messages.push({
+			...batch.content[0],
+			contents: JSON.stringify({ packedContents: compressedContent }),
+			metadata: batch.content[0].metadata,
+			compression: CompressionAlgorithms.lz4,
+		});
+
+		for (const message of batch.content.slice(1)) {
+			messages.push({ ...message, contents: undefined });
+		}
+
+		return {
+			contentSizeInBytes: compressedContent.length,
+			content: messages,
+		};
+	}
+
+	private serializeBatch(batch: IBatch): string {
+		try {
+			return JSON.stringify(batch.content.map((message) => message.deserializedContent));
+		} catch (e: any) {
+			if (e.message === "Invalid string length") {
+				// This is how JSON.stringify signals that
+				// the content size exceeds its capacity
+				const error = new UsageError("Payload too large");
+				this.logger.sendErrorEvent(
+					{
+						eventName: "BatchTooLarge",
+						size: batch.contentSizeInBytes,
+						length: batch.content.length,
+					},
+					error,
+				);
+				throw error;
+			}
+
+			throw e;
+		}
+	}
+}

package/src/opLifecycle/opDecompressor.ts

@@ -0,0 +1,124 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { decompress } from "lz4js";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { assert, IsoBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
+import { CompressionAlgorithms } from "../containerRuntime";
+import { IMessageProcessingResult } from "./definitions";
+
+/**
+ * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
+ * This class relies on some implicit contracts defined below:
+ * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true
+ * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
+ * 3. The final message of a batch will have batch metadata set to false
+ * 4. An individually compressed op will have undefined batch metadata and compression set to true
+ */
+export class OpDecompressor {
+	private activeBatch = false;
+	private rootMessageContents: any | undefined;
+	private processedCount = 0;
+
+	public processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {
+		assert(
+			message.compression === undefined || message.compression === CompressionAlgorithms.lz4,
+			0x511 /* Only lz4 compression is supported */,
+		);
+
+		if (message.metadata?.batch === true && message.compression === CompressionAlgorithms.lz4) {
+			// Beginning of a compressed batch
+			assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
+			if (message.compression) {
+				// lz4 is the only supported compression algorithm for now
+				assert(
+					message.compression === CompressionAlgorithms.lz4,
+					0x4b9 /* lz4 is currently the only supported compression algorithm */,
+				);
+			}
+
+			this.activeBatch = true;
+
+			const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+			const decompressedMessage = decompress(contents);
+			const intoString = Uint8ArrayToString(decompressedMessage);
+			const asObj = JSON.parse(intoString);
+			this.rootMessageContents = asObj;
+
+			return {
+				message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+				state: "Accepted",
+			};
+		}
+
+		if (
+			this.rootMessageContents !== undefined &&
+			message.metadata?.batch === undefined &&
+			this.activeBatch
+		) {
+			assert(message.contents === undefined, 0x512 /* Expecting empty message */);
+
+			// Continuation of compressed batch
+			return {
+				message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+				state: "Accepted",
+			};
+		}
+
+		if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {
+			// End of compressed batch
+			const returnMessage = newMessage(
+				message,
+				this.rootMessageContents[this.processedCount++],
+			);
+
+			this.activeBatch = false;
+			this.rootMessageContents = undefined;
+			this.processedCount = 0;
+
+			return {
+				message: returnMessage,
+				state: "Processed",
+			};
+		}
+
+		if (
+			message.metadata?.batch === undefined &&
+			message.compression === CompressionAlgorithms.lz4
+		) {
+			// Single compressed message
+			assert(
+				this.activeBatch === false,
+				0x4ba /* shouldn't receive compressed message in middle of a batch */,
+			);
+
+			const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+			const decompressedMessage = decompress(contents);
+			const intoString = new TextDecoder().decode(decompressedMessage);
+			const asObj = JSON.parse(intoString);
+
+			return {
+				message: newMessage(message, asObj[0]),
+				state: "Processed",
+			};
+		}
+
+		return {
+			message,
+			state: "Skipped",
+		};
+	}
+}
+
+// We should not be mutating the input message nor its metadata
+const newMessage = (
+	originalMessage: ISequencedDocumentMessage,
+	contents: any,
+): ISequencedDocumentMessage => ({
+	...originalMessage,
+	contents,
+	compression: undefined,
+	metadata: { ...originalMessage.metadata },
+});

package/src/opLifecycle/opSplitter.ts

@@ -0,0 +1,238 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { assert } from "@fluidframework/common-utils";
+import { IBatchMessage } from "@fluidframework/container-definitions";
+import {
+	DataCorruptionError,
+	extractSafePropertiesFromMessage,
+} from "@fluidframework/container-utils";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { ChildLogger } from "@fluidframework/telemetry-utils";
+import { ContainerMessageType, ContainerRuntimeMessage } from "../containerRuntime";
+import { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from "./definitions";
+
+/**
+ * Responsible for creating and reconstructing chunked messages.
+ */
+export class OpSplitter {
+	// Local copy of incomplete received chunks.
+	private readonly chunkMap: Map<string, string[]>;
+	private readonly logger;
+
+	constructor(
+		chunks: [string, string[]][],
+		private readonly submitBatchFn: ((batch: IBatchMessage[]) => number) | undefined,
+		private readonly chunkSizeInBytes: number,
+		private readonly maxBatchSizeInBytes: number,
+		logger: ITelemetryLogger,
+	) {
+		this.chunkMap = new Map<string, string[]>(chunks);
+		this.logger = ChildLogger.create(logger, "OpSplitter");
+	}
+
+	public get isBatchChunkingEnabled(): boolean {
+		return this.chunkSizeInBytes < Number.POSITIVE_INFINITY && this.submitBatchFn !== undefined;
+	}
+
+	public get chunks(): ReadonlyMap<string, string[]> {
+		return this.chunkMap;
+	}
+
+	public processRemoteMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {
+		if (message.type !== ContainerMessageType.ChunkedOp) {
+			return {
+				message,
+				state: "Skipped",
+			};
+		}
+
+		const clientId = message.clientId;
+		const chunkedContent = message.contents as IChunkedOp;
+		this.addChunk(clientId, chunkedContent, message);
+
+		if (chunkedContent.chunkId < chunkedContent.totalChunks) {
+			// We are processing the op in chunks but haven't reached
+			// the last chunk yet in order to reconstruct the original op
+			return {
+				message,
+				state: "Accepted",
+			};
+		}
+
+		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+		const serializedContent = this.chunkMap.get(clientId)!.join("");
+		this.clearPartialChunks(clientId);
+
+		const newMessage = { ...message };
+		newMessage.contents = serializedContent === "" ? undefined : JSON.parse(serializedContent);
+		newMessage.type = chunkedContent.originalType;
+		newMessage.metadata = chunkedContent.originalMetadata;
+		newMessage.compression = chunkedContent.originalCompression;
+		return {
+			message: newMessage,
+			state: "Processed",
+		};
+	}
+
+	public clearPartialChunks(clientId: string) {
+		if (this.chunkMap.has(clientId)) {
+			this.chunkMap.delete(clientId);
+		}
+	}
+
+	private addChunk(
+		clientId: string,
+		chunkedContent: IChunkedOp,
+		originalMessage: ISequencedDocumentMessage,
+	) {
+		let map = this.chunkMap.get(clientId);
+		if (map === undefined) {
+			map = [];
+			this.chunkMap.set(clientId, map);
+		}
+
+		if (chunkedContent.chunkId !== map.length + 1) {
+			// We are expecting the chunks to be processed sequentially, in the same order as they are sent.
+			// Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)
+			// holding the existing chunks for that particular clientId.
+			throw new DataCorruptionError("Chunk Id mismatch", {
+				...extractSafePropertiesFromMessage(originalMessage),
+				chunkMapLength: map.length,
+				chunkId: chunkedContent.chunkId,
+				totalChunks: chunkedContent.totalChunks,
+			});
+		}
+
+		map.push(chunkedContent.contents);
+	}
+
+	/**
+	 * Splits the first op of a compressed batch in chunks, sends the chunks separately and
+	 * returns a new batch composed of the last chunk and the rest of the ops in the original batch.
+	 *
+	 * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops
+	 * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original
+	 * uncompressed ops at ingestion in the runtime.
+	 *
+	 * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire
+	 * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.
+	 *
+	 * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch
+	 * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch
+	 * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed
+	 * (as it is the last chunk).
+	 *
+	 * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
+	 * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.
+	 *
+	 * @param batch - the compressed batch which needs to be processed
+	 * @returns A new adjusted batch which can be sent over the wire
+	 */
+	public splitCompressedBatch(batch: IBatch): IBatch {
+		assert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);
+		assert(
+			batch.contentSizeInBytes > 0 && batch.content.length > 0,
+			0x514 /* Batch needs to be non-empty */,
+		);
+		assert(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);
+		assert(
+			this.chunkSizeInBytes < this.maxBatchSizeInBytes,
+			0x516 /* Chunk size needs to be smaller than the max batch size */,
+		);
+
+		const firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split
+		assert(
+			firstMessage.metadata?.compressed === true || firstMessage.compression !== undefined,
+			0x517 /* Batch needs to be compressed */,
+		);
+		assert(
+			(firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,
+			0x518 /* First message in the batch needs to be chunkable */,
+		);
+
+		const restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers
+		const chunks = splitOp(firstMessage, this.chunkSizeInBytes);
+
+		assert(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);
+		// Send the first N-1 chunks immediately
+		for (const chunk of chunks.slice(0, -1)) {
+			this.submitBatchFn([chunkToBatchMessage(chunk, firstMessage.referenceSequenceNumber)]);
+		}
+
+		// The last chunk will be part of the new batch and needs to
+		// preserve the batch metadata of the original batch
+		const lastChunk = chunkToBatchMessage(
+			chunks[chunks.length - 1],
+			firstMessage.referenceSequenceNumber,
+			{ batch: firstMessage.metadata?.batch },
+		);
+
+		this.logger.sendPerformanceEvent({
+			eventName: "Chunked compressed batch",
+			length: batch.content.length,
+			sizeInBytes: batch.contentSizeInBytes,
+			chunks: chunks.length,
+			chunkSizeInBytes: this.chunkSizeInBytes,
+		});
+
+		return {
+			content: [lastChunk, ...restOfMessages],
+			contentSizeInBytes: lastChunk.contents?.length ?? 0,
+		};
+	}
+}
+
+const chunkToBatchMessage = (
+	chunk: IChunkedOp,
+	referenceSequenceNumber: number,
+	metadata: Record<string, unknown> | undefined = undefined,
+): BatchMessage => {
+	const payload: ContainerRuntimeMessage = {
+		type: ContainerMessageType.ChunkedOp,
+		contents: chunk,
+	};
+	return {
+		contents: JSON.stringify(payload),
+		deserializedContent: payload,
+		metadata,
+		localOpMetadata: undefined,
+		referenceSequenceNumber,
+	};
+};
+
+export const splitOp = (op: BatchMessage, chunkSizeInBytes: number): IChunkedOp[] => {
+	const chunks: IChunkedOp[] = [];
+	assert(
+		op.contents !== undefined && op.contents !== null,
+		0x51a /* We should have something to chunk */,
+	);
+
+	const contentLength = op.contents.length;
+	const chunkN = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1;
+	let offset = 0;
+	for (let i = 1; i <= chunkN; i++) {
+		const chunk: IChunkedOp = {
+			chunkId: i,
+			contents: op.contents.substr(offset, chunkSizeInBytes),
+			originalType: op.deserializedContent.type,
+			totalChunks: chunkN,
+		};
+
+		if (i === chunkN) {
+			// We don't need to port these to all the chunks,
+			// as we rebuild the original op when we process the
+			// last chunk, therefore it is the only one that needs it.
+			chunk.originalMetadata = op.metadata;
+			chunk.originalCompression = op.compression;
+		}
+
+		chunks.push(chunk);
+		offset += chunkSizeInBytes;
+	}
+
+	return chunks;
+};