@fluidframework/container-runtime 2.0.0-dev.2.3.0.115467 → 2.0.0-dev.3.1.0.125672

package/src/opLifecycle/batchManager.ts

@@ -3,103 +3,165 @@
  * Licensed under the MIT License.
  */
 
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { UsageError } from "@fluidframework/driver-utils";
+import { ChildLogger } from "@fluidframework/telemetry-utils";
 import { ICompressionRuntimeOptions } from "../containerRuntime";
 import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions";
 
 export interface IBatchManagerOptions {
-    readonly hardLimit: number;
-    readonly softLimit?: number;
-    readonly compressionOptions?: ICompressionRuntimeOptions;
+	readonly enableOpReentryCheck?: boolean;
+	readonly hardLimit: number;
+	readonly softLimit?: number;
+	readonly compressionOptions?: ICompressionRuntimeOptions;
 }
 
 /**
  * Helper class that manages partial batch & rollback.
  */
 export class BatchManager {
-    private pendingBatch: BatchMessage[] = [];
-    private batchContentSize = 0;
-
-    public get length() { return this.pendingBatch.length; }
-    public get contentSizeInBytes() { return this.batchContentSize; }
-
-    constructor(public readonly options: IBatchManagerOptions) { }
-
-    public push(message: BatchMessage): boolean {
-        const contentSize = this.batchContentSize + (message.contents?.length ?? 0);
-        const opCount = this.pendingBatch.length;
-
-        // Attempt to estimate batch size, aka socket message size.
-        // Each op has pretty large envelope, estimating to be 200 bytes.
-        // Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.
-        // Not taking it into account, as compression work should help there - compressed payload will be
-        // initially stored as base64, and that requires only 2 extra escape characters.
-        const socketMessageSize = contentSize + 200 * opCount;
-
-        // If we were provided soft limit, check for exceeding it.
-        // But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)
-        // and start over. That's not an option if we have no ops.
-        // If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.
-        // Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.
-        if (this.options.softLimit !== undefined
-            && this.length > 0
-            && socketMessageSize >= this.options.softLimit) {
-            return false;
-        }
-
-        if (socketMessageSize >= this.options.hardLimit) {
-            return false;
-        }
-
-        this.batchContentSize = contentSize;
-        this.pendingBatch.push(message);
-        return true;
-    }
-
-    public get empty() { return this.pendingBatch.length === 0; }
-
-    public popBatch(): IBatch {
-        const batch: IBatch = {
-            content: this.pendingBatch,
-            contentSizeInBytes: this.batchContentSize,
-        };
-
-        this.pendingBatch = [];
-        this.batchContentSize = 0;
-
-        return addBatchMetadata(batch);
-    }
-
-    /**
-     * Capture the pending state at this point
-     */
-    public checkpoint(): IBatchCheckpoint {
-        const startPoint = this.pendingBatch.length;
-        return {
-            rollback: (process: (message: BatchMessage) => void) => {
-                for (let i = this.pendingBatch.length; i > startPoint;) {
-                    i--;
-                    const message = this.pendingBatch[i];
-                    this.batchContentSize -= message.contents?.length ?? 0;
-                    process(message);
-                }
-
-                this.pendingBatch.length = startPoint;
-            },
-        };
-    }
+	private readonly logger;
+	private pendingBatch: BatchMessage[] = [];
+	private batchContentSize = 0;
+	/**
+	 * Track the number of ops which were detected to have a mismatched
+	 * reference sequence number, in order to self-throttle the telemetry events.
+	 *
+	 * This should be removed as part of ADO:2322
+	 */
+	private readonly maxMismatchedOpsToReport = 5;
+	private mismatchedOpsReported = 0;
+
+	public get length() {
+		return this.pendingBatch.length;
+	}
+	public get contentSizeInBytes() {
+		return this.batchContentSize;
+	}
+
+	constructor(public readonly options: IBatchManagerOptions, logger: ITelemetryLogger) {
+		this.logger = ChildLogger.create(logger, "BatchManager");
+	}
+
+	public push(message: BatchMessage): boolean {
+		this.checkReferenceSequenceNumber(message);
+
+		const contentSize = this.batchContentSize + (message.contents?.length ?? 0);
+		const opCount = this.pendingBatch.length;
+
+		// Attempt to estimate batch size, aka socket message size.
+		// Each op has pretty large envelope, estimating to be 200 bytes.
+		// Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.
+		// Not taking it into account, as compression work should help there - compressed payload will be
+		// initially stored as base64, and that requires only 2 extra escape characters.
+		const socketMessageSize = contentSize + 200 * opCount;
+
+		// If we were provided soft limit, check for exceeding it.
+		// But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)
+		// and start over. That's not an option if we have no ops.
+		// If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.
+		// Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.
+		if (
+			this.options.softLimit !== undefined &&
+			this.length > 0 &&
+			socketMessageSize >= this.options.softLimit
+		) {
+			return false;
+		}
+
+		if (socketMessageSize >= this.options.hardLimit) {
+			return false;
+		}
+
+		this.batchContentSize = contentSize;
+		this.pendingBatch.push(message);
+		return true;
+	}
+
+	public get empty() {
+		return this.pendingBatch.length === 0;
+	}
+
+	public popBatch(): IBatch {
+		const batch: IBatch = {
+			content: this.pendingBatch,
+			contentSizeInBytes: this.batchContentSize,
+		};
+
+		this.pendingBatch = [];
+		this.batchContentSize = 0;
+
+		return addBatchMetadata(batch);
+	}
+
+	/**
+	 * Capture the pending state at this point
+	 */
+	public checkpoint(): IBatchCheckpoint {
+		const startPoint = this.pendingBatch.length;
+		return {
+			rollback: (process: (message: BatchMessage) => void) => {
+				for (let i = this.pendingBatch.length; i > startPoint; ) {
+					i--;
+					const message = this.pendingBatch[i];
+					this.batchContentSize -= message.contents?.length ?? 0;
+					process(message);
+				}
+
+				this.pendingBatch.length = startPoint;
+			},
+		};
+	}
+
+	private checkReferenceSequenceNumber(message: BatchMessage) {
+		if (
+			this.pendingBatch.length === 0 ||
+			message.referenceSequenceNumber === this.pendingBatch[0].referenceSequenceNumber
+		) {
+			// The reference sequence numbers are stable
+			return;
+		}
+
+		const telemetryProperties = {
+			referenceSequenceNumber: this.pendingBatch[0].referenceSequenceNumber,
+			messageReferenceSequenceNumber: message.referenceSequenceNumber,
+			type: message.deserializedContent.type,
+			length: this.pendingBatch.length,
+			enableOpReentryCheck: this.options.enableOpReentryCheck === true,
+		};
+		const error = new UsageError("Submission of an out of order message");
+		const eventName = "ReferenceSequenceNumberMismatch";
+
+		if (this.options.enableOpReentryCheck === true) {
+			this.logger.sendErrorEvent({ eventName, ...telemetryProperties }, error);
+			throw error;
+		}
+
+		if (++this.mismatchedOpsReported <= this.maxMismatchedOpsToReport) {
+			this.logger.sendErrorEvent(
+				{
+					eventName,
+					...telemetryProperties,
+					ops: this.mismatchedOpsReported,
+					maxOps: this.maxMismatchedOpsToReport,
+				},
+				error,
+			);
+		}
+	}
 }
 
 const addBatchMetadata = (batch: IBatch): IBatch => {
-    if (batch.content.length > 1) {
-        batch.content[0].metadata = {
-            ...batch.content[0].metadata,
-            batch: true
-        };
-        batch.content[batch.content.length - 1].metadata = {
-            ...batch.content[batch.content.length - 1].metadata,
-            batch: false
-        };
-    }
-
-    return batch;
+	if (batch.content.length > 1) {
+		batch.content[0].metadata = {
+			...batch.content[0].metadata,
+			batch: true,
+		};
+		batch.content[batch.content.length - 1].metadata = {
+			...batch.content[batch.content.length - 1].metadata,
+			batch: false,
+		};
+	}
+
+	return batch;
 };

package/src/opLifecycle/definitions.ts

@@ -4,41 +4,67 @@
  */
 
 import { IBatchMessage } from "@fluidframework/container-definitions";
-import { MessageType } from "@fluidframework/protocol-definitions";
+import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
 import { CompressionAlgorithms, ContainerMessageType, ContainerRuntimeMessage } from "..";
 
 /**
  * Batch message type used internally by the runtime
  */
 export type BatchMessage = IBatchMessage & {
-    localOpMetadata: unknown;
-    deserializedContent: ContainerRuntimeMessage;
-    referenceSequenceNumber: number;
-    compression?: CompressionAlgorithms;
+	localOpMetadata: unknown;
+	deserializedContent: ContainerRuntimeMessage;
+	referenceSequenceNumber: number;
+	compression?: CompressionAlgorithms;
 };
 
 /**
  * Batch interface used internally by the runtime.
  */
 export interface IBatch {
-    /**
-     * Sum of the in-memory content sizes of all messages in the batch.
-     * If the batch is compressed, this number reflects the post-compression size.
-     */
-    readonly contentSizeInBytes: number;
-    /**
-     * All the messages in the batch
-     */
-    readonly content: BatchMessage[];
+	/**
+	 * Sum of the in-memory content sizes of all messages in the batch.
+	 * If the batch is compressed, this number reflects the post-compression size.
+	 */
+	readonly contentSizeInBytes: number;
+	/**
+	 * All the messages in the batch
+	 */
+	readonly content: BatchMessage[];
 }
 
 export interface IBatchCheckpoint {
-    rollback: (action: (message: BatchMessage) => void) => void;
+	rollback: (action: (message: BatchMessage) => void) => void;
 }
 
 export interface IChunkedOp {
-    chunkId: number;
-    totalChunks: number;
-    contents: string;
-    originalType: MessageType | ContainerMessageType;
+	chunkId: number;
+	totalChunks: number;
+	contents: string;
+	originalType: MessageType | ContainerMessageType;
+	originalMetadata?: Record<string, unknown>;
+	originalCompression?: string;
+}
+
+/**
+ * The state of remote message processing:
+ * `Processed` - the message can be considered processed
+ * `Skipped` - the message was ignored by the processor
+ * `Accepted` - the message was processed partially. Eventually, a message
+ * will make the processor return `Processed`.
+ */
+export type ProcessingState = "Processed" | "Skipped" | "Accepted";
+
+/**
+ * Return type for functions which process remote messages
+ */
+export interface IMessageProcessingResult {
+	/**
+	 * A shallow copy of the input message if processing happened, or
+	 * the original message otherwise
+	 */
+	readonly message: ISequencedDocumentMessage;
+	/**
+	 * Processing result of the input message.
+	 */
+	readonly state: ProcessingState;
 }

package/src/opLifecycle/index.ts

@@ -5,13 +5,14 @@
 
 export { BatchManager } from "./batchManager";
 export {
-    BatchMessage,
-    IBatch,
-    IBatchCheckpoint,
-    IChunkedOp,
+	BatchMessage,
+	IBatch,
+	IBatchCheckpoint,
+	IChunkedOp,
+	IMessageProcessingResult,
 } from "./definitions";
 export { Outbox } from "./outbox";
 export { OpCompressor } from "./opCompressor";
 export { OpDecompressor } from "./opDecompressor";
-export { OpSplitter } from "./opSplitter";
+export { OpSplitter, splitOp } from "./opSplitter";
 export { RemoteMessageProcessor, unpackRuntimeMessage } from "./remoteMessageProcessor";

package/src/opLifecycle/opCompressor.ts

@@ -5,9 +5,10 @@
 
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { IsoBuffer } from "@fluidframework/common-utils";
+import { UsageError } from "@fluidframework/container-utils";
 import { ChildLogger } from "@fluidframework/telemetry-utils";
 import { compress } from "lz4js";
-import { CompressionAlgorithms, ContainerRuntimeMessage } from "../containerRuntime";
+import { CompressionAlgorithms } from "../containerRuntime";
 import { IBatch, BatchMessage } from "./definitions";
 
 /**
@@ -16,49 +17,66 @@ import { IBatch, BatchMessage } from "./definitions";
  * op to reserve sequence numbers.
  */
 export class OpCompressor {
-    private readonly logger;
-    private compressedBatchCount = 0;
+	private readonly logger;
 
-    constructor(logger: ITelemetryLogger) {
-        this.logger = ChildLogger.create(logger, "OpCompressor");
-    }
+	constructor(logger: ITelemetryLogger) {
+		this.logger = ChildLogger.create(logger, "OpCompressor");
+	}
 
-    public compressBatch(batch: IBatch): IBatch {
-        const messages: BatchMessage[] = [];
-        this.compressedBatchCount++;
-        const contentToCompress: ContainerRuntimeMessage[] = [];
-        for (const message of batch.content) {
-            contentToCompress.push(message.deserializedContent);
-        }
+	public compressBatch(batch: IBatch): IBatch {
+		const compressionStart = Date.now();
+		const contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));
+		const compressedContents = compress(contentsAsBuffer);
+		const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
+		const duration = Date.now() - compressionStart;
 
-        const compressionStart = Date.now();
-        const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(contentToCompress));
-        const compressedContents = compress(contentsAsBuffer);
-        const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
-        const duration = Date.now() - compressionStart;
+		if (batch.contentSizeInBytes > 200000) {
+			this.logger.sendPerformanceEvent({
+				eventName: "CompressedBatch",
+				duration,
+				sizeBeforeCompression: batch.contentSizeInBytes,
+				sizeAfterCompression: compressedContent.length,
+			});
+		}
 
-        if (batch.contentSizeInBytes > 200000 || this.compressedBatchCount % 25) {
-            this.logger.sendPerformanceEvent({
-                eventName: "CompressedBatch",
-                duration,
-                sizeBeforeCompression: batch.contentSizeInBytes,
-                sizeAfterCompression: compressedContent.length,
-            });
-        }
+		const messages: BatchMessage[] = [];
+		messages.push({
+			...batch.content[0],
+			contents: JSON.stringify({ packedContents: compressedContent }),
+			metadata: batch.content[0].metadata,
+			compression: CompressionAlgorithms.lz4,
+		});
 
-        messages.push({
-            ...batch.content[0], contents: JSON.stringify({ packedContents: compressedContent }),
-            metadata: { ...batch.content[0].metadata, compressed: true },
-            compression: CompressionAlgorithms.lz4,
-        });
+		for (const message of batch.content.slice(1)) {
+			messages.push({ ...message, contents: undefined });
+		}
 
-        for (const message of batch.content.slice(1)) {
-            messages.push({ ...message, contents: undefined });
-        }
+		return {
+			contentSizeInBytes: compressedContent.length,
+			content: messages,
+		};
+	}
 
-        return {
-            contentSizeInBytes: compressedContent.length,
-            content: messages,
-        };
-    }
+	private serializeBatch(batch: IBatch): string {
+		try {
+			return JSON.stringify(batch.content.map((message) => message.deserializedContent));
+		} catch (e: any) {
+			if (e.message === "Invalid string length") {
+				// This is how JSON.stringify signals that
+				// the content size exceeds its capacity
+				const error = new UsageError("Payload too large");
+				this.logger.sendErrorEvent(
+					{
+						eventName: "BatchTooLarge",
+						size: batch.contentSizeInBytes,
+						length: batch.content.length,
+					},
+					error,
+				);
+				throw error;
+			}
+
+			throw e;
+		}
+	}
 }

package/src/opLifecycle/opDecompressor.ts

@@ -7,6 +7,7 @@ import { decompress } from "lz4js";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { assert, IsoBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
 import { CompressionAlgorithms } from "../containerRuntime";
+import { IMessageProcessingResult } from "./definitions";
 
 /**
  * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
@@ -17,68 +18,107 @@ import { CompressionAlgorithms } from "../containerRuntime";
  * 4. An individually compressed op will have undefined batch metadata and compression set to true
  */
 export class OpDecompressor {
-    private activeBatch = false;
-    private rootMessageContents: any | undefined;
-    private processedCount = 0;
-
-    public processMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {
-        // We're checking for compression = true or top level compression property so
-        // that we can enable compression without waiting on all ordering services
-        // to pick up protocol change. Eventually only the top level property should
-        // be used.
-        if (message.metadata?.batch === true
-            && (message.metadata?.compressed || message.compression !== undefined)) {
-            // Beginning of a compressed batch
-            assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
-            if (message.compression) {
-                // lz4 is the only supported compression algorithm for now
-                assert(message.compression === CompressionAlgorithms.lz4,
-                    0x4b9 /* lz4 is currently the only supported compression algorithm */);
-            }
-
-            this.activeBatch = true;
-
-            const contents = IsoBuffer.from(message.contents.packedContents, "base64");
-            const decompressedMessage = decompress(contents);
-            const intoString = Uint8ArrayToString(decompressedMessage);
-            const asObj = JSON.parse(intoString);
-            this.rootMessageContents = asObj;
-
-            return { ...message, contents: this.rootMessageContents[this.processedCount++] };
-        }
-
-        if (this.rootMessageContents !== undefined && message.metadata?.batch === undefined && this.activeBatch) {
-            // Continuation of compressed batch
-            return { ...message, contents: this.rootMessageContents[this.processedCount++] };
-        }
-
-        if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {
-            // End of compressed batch
-            const returnMessage = {
-                ...message,
-                contents: this.rootMessageContents[this.processedCount++]
-            };
-
-            this.activeBatch = false;
-            this.rootMessageContents = undefined;
-            this.processedCount = 0;
-
-            return returnMessage;
-        }
-
-        if (message.metadata?.batch === undefined &&
-            (message.metadata?.compressed || message.compression === CompressionAlgorithms.lz4)) {
-            // Single compressed message
-            assert(this.activeBatch === false, 0x4ba /* shouldn't receive compressed message in middle of a batch */);
-
-            const contents = IsoBuffer.from(message.contents.packedContents, "base64");
-            const decompressedMessage = decompress(contents);
-            const intoString = new TextDecoder().decode(decompressedMessage);
-            const asObj = JSON.parse(intoString);
-
-            return { ...message, contents: asObj[0] };
-        }
-
-        return message;
-    }
+	private activeBatch = false;
+	private rootMessageContents: any | undefined;
+	private processedCount = 0;
+
+	public processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {
+		assert(
+			message.compression === undefined || message.compression === CompressionAlgorithms.lz4,
+			0x511 /* Only lz4 compression is supported */,
+		);
+
+		if (message.metadata?.batch === true && message.compression === CompressionAlgorithms.lz4) {
+			// Beginning of a compressed batch
+			assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
+			if (message.compression) {
+				// lz4 is the only supported compression algorithm for now
+				assert(
+					message.compression === CompressionAlgorithms.lz4,
+					0x4b9 /* lz4 is currently the only supported compression algorithm */,
+				);
+			}
+
+			this.activeBatch = true;
+
+			const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+			const decompressedMessage = decompress(contents);
+			const intoString = Uint8ArrayToString(decompressedMessage);
+			const asObj = JSON.parse(intoString);
+			this.rootMessageContents = asObj;
+
+			return {
+				message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+				state: "Accepted",
+			};
+		}
+
+		if (
+			this.rootMessageContents !== undefined &&
+			message.metadata?.batch === undefined &&
+			this.activeBatch
+		) {
+			assert(message.contents === undefined, 0x512 /* Expecting empty message */);
+
+			// Continuation of compressed batch
+			return {
+				message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+				state: "Accepted",
+			};
+		}
+
+		if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {
+			// End of compressed batch
+			const returnMessage = newMessage(
+				message,
+				this.rootMessageContents[this.processedCount++],
+			);
+
+			this.activeBatch = false;
+			this.rootMessageContents = undefined;
+			this.processedCount = 0;
+
+			return {
+				message: returnMessage,
+				state: "Processed",
+			};
+		}
+
+		if (
+			message.metadata?.batch === undefined &&
+			message.compression === CompressionAlgorithms.lz4
+		) {
+			// Single compressed message
+			assert(
+				this.activeBatch === false,
+				0x4ba /* shouldn't receive compressed message in middle of a batch */,
+			);
+
+			const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+			const decompressedMessage = decompress(contents);
+			const intoString = new TextDecoder().decode(decompressedMessage);
+			const asObj = JSON.parse(intoString);
+
+			return {
+				message: newMessage(message, asObj[0]),
+				state: "Processed",
+			};
+		}
+
+		return {
+			message,
+			state: "Skipped",
+		};
+	}
 }
+
+// We should not be mutating the input message nor its metadata
+const newMessage = (
+	originalMessage: ISequencedDocumentMessage,
+	contents: any,
+): ISequencedDocumentMessage => ({
+	...originalMessage,
+	contents,
+	compression: undefined,
+	metadata: { ...originalMessage.metadata },
+});