@fluidframework/container-runtime 2.0.0-dev.2.3.0.115467 → 2.0.0-dev.4.1.0.148229

package/src/opLifecycle/opCompressor.ts

@@ -4,10 +4,12 @@
  */
 
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { IsoBuffer } from "@fluidframework/common-utils";
+import { assert, IsoBuffer } from "@fluidframework/common-utils";
+import { UsageError } from "@fluidframework/container-utils";
 import { ChildLogger } from "@fluidframework/telemetry-utils";
 import { compress } from "lz4js";
-import { CompressionAlgorithms, ContainerRuntimeMessage } from "../containerRuntime";
+import { CompressionAlgorithms } from "../containerRuntime";
+import { estimateSocketSize } from "./batchManager";
 import { IBatch, BatchMessage } from "./definitions";
 
 /**
@@ -16,49 +18,85 @@ import { IBatch, BatchMessage } from "./definitions";
  * op to reserve sequence numbers.
  */
 export class OpCompressor {
-    private readonly logger;
-    private compressedBatchCount = 0;
+	private readonly logger;
 
-    constructor(logger: ITelemetryLogger) {
-        this.logger = ChildLogger.create(logger, "OpCompressor");
-    }
+	constructor(logger: ITelemetryLogger) {
+		this.logger = ChildLogger.create(logger, "OpCompressor");
+	}
 
-    public compressBatch(batch: IBatch): IBatch {
-        const messages: BatchMessage[] = [];
-        this.compressedBatchCount++;
-        const contentToCompress: ContainerRuntimeMessage[] = [];
-        for (const message of batch.content) {
-            contentToCompress.push(message.deserializedContent);
-        }
+	public compressBatch(batch: IBatch): IBatch {
+		assert(
+			batch.contentSizeInBytes > 0 && batch.content.length > 0,
+			0x5a4 /* Batch should not be empty */,
+		);
 
-        const compressionStart = Date.now();
-        const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(contentToCompress));
-        const compressedContents = compress(contentsAsBuffer);
-        const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
-        const duration = Date.now() - compressionStart;
+		const compressionStart = Date.now();
+		const contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));
+		const compressedContents = compress(contentsAsBuffer);
+		const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
+		const duration = Date.now() - compressionStart;
 
-        if (batch.contentSizeInBytes > 200000 || this.compressedBatchCount % 25) {
-            this.logger.sendPerformanceEvent({
-                eventName: "CompressedBatch",
-                duration,
-                sizeBeforeCompression: batch.contentSizeInBytes,
-                sizeAfterCompression: compressedContent.length,
-            });
-        }
+		const messages: BatchMessage[] = [];
+		messages.push({
+			...batch.content[0],
+			contents: JSON.stringify({ packedContents: compressedContent }),
+			metadata: batch.content[0].metadata,
+			compression: CompressionAlgorithms.lz4,
+		});
 
-        messages.push({
-            ...batch.content[0], contents: JSON.stringify({ packedContents: compressedContent }),
-            metadata: { ...batch.content[0].metadata, compressed: true },
-            compression: CompressionAlgorithms.lz4,
-        });
+		// Add empty placeholder messages to reserve the sequence numbers
+		for (const message of batch.content.slice(1)) {
+			messages.push({
+				deserializedContent: {
+					contents: undefined,
+					type: message.deserializedContent.type,
+				},
+				localOpMetadata: message.localOpMetadata,
+				metadata: message.metadata,
+				referenceSequenceNumber: message.referenceSequenceNumber,
+			});
+		}
 
-        for (const message of batch.content.slice(1)) {
-            messages.push({ ...message, contents: undefined });
-        }
+		const compressedBatch: IBatch = {
+			contentSizeInBytes: compressedContent.length,
+			content: messages,
+			referenceSequenceNumber: batch.referenceSequenceNumber,
+		};
 
-        return {
-            contentSizeInBytes: compressedContent.length,
-            content: messages,
-        };
-    }
+		if (batch.contentSizeInBytes > 200000) {
+			this.logger.sendPerformanceEvent({
+				eventName: "CompressedBatch",
+				duration,
+				sizeBeforeCompression: batch.contentSizeInBytes,
+				sizeAfterCompression: compressedBatch.contentSizeInBytes,
+				opCount: compressedBatch.content.length,
+				socketSize: estimateSocketSize(compressedBatch),
+			});
+		}
+
+		return compressedBatch;
+	}
+
+	private serializeBatch(batch: IBatch): string {
+		try {
+			return JSON.stringify(batch.content.map((message) => message.deserializedContent));
+		} catch (e: any) {
+			if (e.message === "Invalid string length") {
+				// This is how JSON.stringify signals that
+				// the content size exceeds its capacity
+				const error = new UsageError("Payload too large");
+				this.logger.sendErrorEvent(
+					{
+						eventName: "BatchTooLarge",
+						size: batch.contentSizeInBytes,
+						length: batch.content.length,
+					},
+					error,
+				);
+				throw error;
+			}
+
+			throw e;
+		}
+	}
 }

package/src/opLifecycle/opDecompressor.ts

@@ -6,7 +6,10 @@
 import { decompress } from "lz4js";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { assert, IsoBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
+import { ChildLogger } from "@fluidframework/telemetry-utils";
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { CompressionAlgorithms } from "../containerRuntime";
+import { IMessageProcessingResult } from "./definitions";
 
 /**
  * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
@@ -17,68 +20,149 @@ import { CompressionAlgorithms } from "../containerRuntime";
  * 4. An individually compressed op will have undefined batch metadata and compression set to true
  */
 export class OpDecompressor {
-    private activeBatch = false;
-    private rootMessageContents: any | undefined;
-    private processedCount = 0;
-
-    public processMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {
-        // We're checking for compression = true or top level compression property so
-        // that we can enable compression without waiting on all ordering services
-        // to pick up protocol change. Eventually only the top level property should
-        // be used.
-        if (message.metadata?.batch === true
-            && (message.metadata?.compressed || message.compression !== undefined)) {
-            // Beginning of a compressed batch
-            assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
-            if (message.compression) {
-                // lz4 is the only supported compression algorithm for now
-                assert(message.compression === CompressionAlgorithms.lz4,
-                    0x4b9 /* lz4 is currently the only supported compression algorithm */);
-            }
-
-            this.activeBatch = true;
-
-            const contents = IsoBuffer.from(message.contents.packedContents, "base64");
-            const decompressedMessage = decompress(contents);
-            const intoString = Uint8ArrayToString(decompressedMessage);
-            const asObj = JSON.parse(intoString);
-            this.rootMessageContents = asObj;
-
-            return { ...message, contents: this.rootMessageContents[this.processedCount++] };
-        }
-
-        if (this.rootMessageContents !== undefined && message.metadata?.batch === undefined && this.activeBatch) {
-            // Continuation of compressed batch
-            return { ...message, contents: this.rootMessageContents[this.processedCount++] };
-        }
-
-        if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {
-            // End of compressed batch
-            const returnMessage = {
-                ...message,
-                contents: this.rootMessageContents[this.processedCount++]
-            };
-
-            this.activeBatch = false;
-            this.rootMessageContents = undefined;
-            this.processedCount = 0;
-
-            return returnMessage;
-        }
-
-        if (message.metadata?.batch === undefined &&
-            (message.metadata?.compressed || message.compression === CompressionAlgorithms.lz4)) {
-            // Single compressed message
-            assert(this.activeBatch === false, 0x4ba /* shouldn't receive compressed message in middle of a batch */);
-
-            const contents = IsoBuffer.from(message.contents.packedContents, "base64");
-            const decompressedMessage = decompress(contents);
-            const intoString = new TextDecoder().decode(decompressedMessage);
-            const asObj = JSON.parse(intoString);
-
-            return { ...message, contents: asObj[0] };
-        }
-
-        return message;
-    }
+	private activeBatch = false;
+	private rootMessageContents: any | undefined;
+	private processedCount = 0;
+	private readonly logger;
+
+	constructor(logger: ITelemetryLogger) {
+		this.logger = ChildLogger.create(logger, "OpDecompressor");
+	}
+
+	public processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {
+		assert(
+			message.compression === undefined || message.compression === CompressionAlgorithms.lz4,
+			0x511 /* Only lz4 compression is supported */,
+		);
+
+		if (message.metadata?.batch === true && this.isCompressed(message)) {
+			// Beginning of a compressed batch
+			assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
+			if (message.compression) {
+				// lz4 is the only supported compression algorithm for now
+				assert(
+					message.compression === CompressionAlgorithms.lz4,
+					0x4b9 /* lz4 is currently the only supported compression algorithm */,
+				);
+			}
+
+			this.activeBatch = true;
+
+			const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+			const decompressedMessage = decompress(contents);
+			const intoString = Uint8ArrayToString(decompressedMessage);
+			const asObj = JSON.parse(intoString);
+			this.rootMessageContents = asObj;
+
+			return {
+				message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+				state: "Accepted",
+			};
+		}
+
+		if (
+			this.rootMessageContents !== undefined &&
+			message.metadata?.batch === undefined &&
+			this.activeBatch
+		) {
+			assert(message.contents === undefined, 0x512 /* Expecting empty message */);
+
+			// Continuation of compressed batch
+			return {
+				message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+				state: "Accepted",
+			};
+		}
+
+		if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {
+			// End of compressed batch
+			const returnMessage = newMessage(
+				message,
+				this.rootMessageContents[this.processedCount++],
+			);
+
+			this.activeBatch = false;
+			this.rootMessageContents = undefined;
+			this.processedCount = 0;
+
+			return {
+				message: returnMessage,
+				state: "Processed",
+			};
+		}
+
+		if (message.metadata?.batch === undefined && this.isCompressed(message)) {
+			// Single compressed message
+			assert(
+				this.activeBatch === false,
+				0x4ba /* shouldn't receive compressed message in middle of a batch */,
+			);
+
+			const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+			const decompressedMessage = decompress(contents);
+			const intoString = new TextDecoder().decode(decompressedMessage);
+			const asObj = JSON.parse(intoString);
+
+			return {
+				message: newMessage(message, asObj[0]),
+				state: "Processed",
+			};
+		}
+
+		return {
+			message,
+			state: "Skipped",
+		};
+	}
+
+	private isCompressed(message: ISequencedDocumentMessage) {
+		if (message.compression === CompressionAlgorithms.lz4) {
+			return true;
+		}
+
+		/**
+		 * Back-compat self healing mechanism for ADO:3538, as loaders from
+		 * version client_v2.0.0-internal.1.2.0 to client_v2.0.0-internal.2.2.0 do not
+		 * support adding the proper compression metadata to compressed messages submitted
+		 * by the runtime. Should be removed after the loader reaches sufficient saturation
+		 * for a version greater or equal than client_v2.0.0-internal.2.2.0.
+		 *
+		 * The condition holds true for compressed messages, regardless of metadata. We are ultimately
+		 * looking for a message with a single property `packedContents` inside `contents`, of type 'string'
+		 * with a base64 encoded value.
+		 */
+		try {
+			if (
+				typeof message.contents === "object" &&
+				Object.keys(message.contents).length === 1 &&
+				message.contents?.packedContents !== undefined &&
+				typeof message.contents?.packedContents === "string" &&
+				message.contents.packedContents.length > 0 &&
+				IsoBuffer.from(message.contents.packedContents, "base64").toString("base64") ===
+					message.contents.packedContents
+			) {
+				this.logger.sendTelemetryEvent({
+					eventName: "LegacyCompression",
+					type: message.type,
+					batch: message.metadata?.batch,
+				});
+				return true;
+			}
+		} catch (err) {
+			return false;
+		}
+
+		return false;
+	}
 }
+
+// We should not be mutating the input message nor its metadata
+const newMessage = (
+	originalMessage: ISequencedDocumentMessage,
+	contents: any,
+): ISequencedDocumentMessage => ({
+	...originalMessage,
+	contents,
+	compression: undefined,
+	metadata: { ...originalMessage.metadata },
+});