@fluidframework/container-runtime 1.4.0-115997 → 2.0.0-dev-rc.1.0.0.224419

package/dist/opLifecycle/opCompressor.cjs

@@ -0,0 +1,84 @@
+"use strict";
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpCompressor = void 0;
+const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
+const core_utils_1 = require("@fluidframework/core-utils");
+const client_utils_1 = require("@fluid-internal/client-utils");
+const lz4js_1 = require("lz4js");
+const containerRuntime_1 = require("../containerRuntime.cjs");
+const batchManager_1 = require("./batchManager.cjs");
+/**
+ * Compresses batches of ops. It generates a single compressed op that contains
+ * the contents of each op in the batch. It then submits empty ops for each original
+ * op to reserve sequence numbers.
+ */
+class OpCompressor {
+    constructor(logger) {
+        this.logger = (0, telemetry_utils_1.createChildLogger)({ logger, namespace: "OpCompressor" });
+    }
+    compressBatch(batch) {
+        (0, core_utils_1.assert)(batch.contentSizeInBytes > 0 && batch.content.length > 0, 0x5a4 /* Batch should not be empty */);
+        const compressionStart = Date.now();
+        const contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));
+        const compressedContents = (0, lz4js_1.compress)(contentsAsBuffer);
+        const compressedContent = client_utils_1.IsoBuffer.from(compressedContents).toString("base64");
+        const duration = Date.now() - compressionStart;
+        const messages = [];
+        messages.push({
+            ...batch.content[0],
+            contents: JSON.stringify({ packedContents: compressedContent }),
+            metadata: batch.content[0].metadata,
+            compression: containerRuntime_1.CompressionAlgorithms.lz4,
+        });
+        // Add empty placeholder messages to reserve the sequence numbers
+        for (const message of batch.content.slice(1)) {
+            messages.push({
+                type: message.type,
+                localOpMetadata: message.localOpMetadata,
+                metadata: message.metadata,
+                referenceSequenceNumber: message.referenceSequenceNumber,
+            });
+        }
+        const compressedBatch = {
+            contentSizeInBytes: compressedContent.length,
+            content: messages,
+            referenceSequenceNumber: batch.referenceSequenceNumber,
+        };
+        if (batch.contentSizeInBytes > 200000) {
+            this.logger.sendPerformanceEvent({
+                eventName: "CompressedBatch",
+                duration,
+                sizeBeforeCompression: batch.contentSizeInBytes,
+                sizeAfterCompression: compressedBatch.contentSizeInBytes,
+                opCount: compressedBatch.content.length,
+                socketSize: (0, batchManager_1.estimateSocketSize)(compressedBatch),
+            });
+        }
+        return compressedBatch;
+    }
+    serializeBatch(batch) {
+        try {
+            return `[${batch.content.map((message) => message.contents).join(",")}]`;
+        }
+        catch (e) {
+            if (e.message === "Invalid string length") {
+                // This is how JSON.stringify signals that
+                // the content size exceeds its capacity
+                const error = new telemetry_utils_1.UsageError("Payload too large");
+                this.logger.sendErrorEvent({
+                    eventName: "BatchTooLarge",
+                    size: batch.contentSizeInBytes,
+                    length: batch.content.length,
+                }, error);
+                throw error;
+            }
+            throw e;
+        }
+    }
+}
+exports.OpCompressor = OpCompressor;
+//# sourceMappingURL=opCompressor.cjs.map

package/dist/opLifecycle/opCompressor.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"opCompressor.cjs","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,qEAAgF;AAChF,2DAAoD;AACpD,+DAAyD;AACzD,iCAAiC;AAEjC,8DAA4D;AAC5D,qDAAoD;AAGpD;;;;GAIG;AACH,MAAa,YAAY;IAGxB,YAAY,MAA4B;QACvC,IAAI,CAAC,MAAM,GAAG,IAAA,mCAAiB,EAAC,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,CAAC;IACxE,CAAC;IAEM,aAAa,CAAC,KAAa;QACjC,IAAA,mBAAM,EACL,KAAK,CAAC,kBAAkB,GAAG,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EACxD,KAAK,CAAC,+BAA+B,CACrC,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9E,MAAM,kBAAkB,GAAG,IAAA,gBAAQ,EAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,iBAAiB,GAAG,wBAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;QAE/C,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,QAAQ,CAAC,IAAI,CAAC;YACb,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YACnB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC;YAC/D,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ;YACnC,WAAW,EAAE,wCAAqB,CAAC,GAAG;SACtC,CAAC,CAAC;QAEH,iEAAiE;QACjE,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACb,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,eAAe,EAAE,OAAO,CAAC,eAAe;gBACxC,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;aACxD,CAAC,CAAC;SACH;QAED,MAAM,eAAe,GAAW;YAC/B,kBAAkB,EAAE,iBAAiB,CAAC,MAAM;YAC5C,OAAO,EAAE,QAAQ;YACjB,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;SACtD,CAAC;QAEF,IAAI,KAAK,CAAC,kBAAkB,GAAG,MAAM,EAAE;YACtC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAChC,SAAS,EAAE,iBAAiB;gBAC5B,QAAQ;gBACR,qBAAqB,EAAE,KAAK,CAAC,kBAAkB;gBAC/C,oBAAoB,EAAE,eAAe,CAAC,kBAAkB;gBACxD,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,MAAM;gBACvC,UAAU,EAAE,IAAA,iCAAkB,EAAC,eAAe,CAAC;aAC/C,CAAC,CAAC;SACH;QAED,OAAO,eAAe,CAAC;IACxB,CAAC;IAEO,cAAc,CAAC,KAAa;QACnC,IAAI;YACH,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;SACzE;QAAC,OAAO,CAAM,EAAE;YAChB,IAAI,CAAC,CAAC,OAAO,KAAK,uBAAuB,EAAE;gBAC1C,0CAA0C;gBAC1C,wCAAwC;gBACxC,MAAM,KAAK,GAAG,IAAI,4BAAU,CAAC,mBAAmB,CAAC,CAAC;gBAClD,IAAI,CAAC,MAAM,CAAC,cAAc,CACzB;oBACC,SAAS,EAAE,eAAe;oBAC1B,IAAI,EAAE,KAAK,CAAC,kBAAkB;oBAC9B,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;iBAC5B,EACD,KAAK,CACL,CAAC;gBACF,MAAM,KAAK,CAAC;aACZ;YAED,MAAM,CAAC,CAAC;SACR;IACF,CAAC;CACD;AA/ED,oCA+EC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { createChildLogger, UsageError } from \"@fluidframework/telemetry-utils\";\nimport { assert } from \"@fluidframework/core-utils\";\nimport { IsoBuffer } from \"@fluid-internal/client-utils\";\nimport { compress } from \"lz4js\";\nimport { ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport { CompressionAlgorithms } from \"../containerRuntime\";\nimport { estimateSocketSize } from \"./batchManager\";\nimport { IBatch, BatchMessage } from \"./definitions\";\n\n/**\n * Compresses batches of ops. It generates a single compressed op that contains\n * the contents of each op in the batch. It then submits empty ops for each original\n * op to reserve sequence numbers.\n */\nexport class OpCompressor {\n\tprivate readonly logger;\n\n\tconstructor(logger: ITelemetryBaseLogger) {\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpCompressor\" });\n\t}\n\n\tpublic compressBatch(batch: IBatch): IBatch {\n\t\tassert(\n\t\t\tbatch.contentSizeInBytes > 0 && batch.content.length > 0,\n\t\t\t0x5a4 /* Batch should not be empty */,\n\t\t);\n\n\t\tconst compressionStart = Date.now();\n\t\tconst contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));\n\t\tconst compressedContents = compress(contentsAsBuffer);\n\t\tconst compressedContent = IsoBuffer.from(compressedContents).toString(\"base64\");\n\t\tconst duration = Date.now() - compressionStart;\n\n\t\tconst messages: BatchMessage[] = [];\n\t\tmessages.push({\n\t\t\t...batch.content[0],\n\t\t\tcontents: JSON.stringify({ packedContents: compressedContent }),\n\t\t\tmetadata: batch.content[0].metadata,\n\t\t\tcompression: CompressionAlgorithms.lz4,\n\t\t});\n\n\t\t// Add empty placeholder messages to reserve the sequence numbers\n\t\tfor (const message of batch.content.slice(1)) {\n\t\t\tmessages.push({\n\t\t\t\ttype: message.type,\n\t\t\t\tlocalOpMetadata: message.localOpMetadata,\n\t\t\t\tmetadata: message.metadata,\n\t\t\t\treferenceSequenceNumber: message.referenceSequenceNumber,\n\t\t\t});\n\t\t}\n\n\t\tconst compressedBatch: IBatch = {\n\t\t\tcontentSizeInBytes: compressedContent.length,\n\t\t\tcontent: messages,\n\t\t\treferenceSequenceNumber: batch.referenceSequenceNumber,\n\t\t};\n\n\t\tif (batch.contentSizeInBytes > 200000) {\n\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\teventName: \"CompressedBatch\",\n\t\t\t\tduration,\n\t\t\t\tsizeBeforeCompression: batch.contentSizeInBytes,\n\t\t\t\tsizeAfterCompression: compressedBatch.contentSizeInBytes,\n\t\t\t\topCount: compressedBatch.content.length,\n\t\t\t\tsocketSize: estimateSocketSize(compressedBatch),\n\t\t\t});\n\t\t}\n\n\t\treturn compressedBatch;\n\t}\n\n\tprivate serializeBatch(batch: IBatch): string {\n\t\ttry {\n\t\t\treturn `[${batch.content.map((message) => message.contents).join(\",\")}]`;\n\t\t} catch (e: any) {\n\t\t\tif (e.message === \"Invalid string length\") {\n\t\t\t\t// This is how JSON.stringify signals that\n\t\t\t\t// the content size exceeds its capacity\n\t\t\t\tconst error = new UsageError(\"Payload too large\");\n\t\t\t\tthis.logger.sendErrorEvent(\n\t\t\t\t\t{\n\t\t\t\t\t\teventName: \"BatchTooLarge\",\n\t\t\t\t\t\tsize: batch.contentSizeInBytes,\n\t\t\t\t\t\tlength: batch.content.length,\n\t\t\t\t\t},\n\t\t\t\t\terror,\n\t\t\t\t);\n\t\t\t\tthrow error;\n\t\t\t}\n\n\t\t\tthrow e;\n\t\t}\n\t}\n}\n"]}

package/dist/opLifecycle/opCompressor.d.ts

@@ -0,0 +1,18 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { IBatch } from "./definitions";
+/**
+ * Compresses batches of ops. It generates a single compressed op that contains
+ * the contents of each op in the batch. It then submits empty ops for each original
+ * op to reserve sequence numbers.
+ */
+export declare class OpCompressor {
+    private readonly logger;
+    constructor(logger: ITelemetryBaseLogger);
+    compressBatch(batch: IBatch): IBatch;
+    private serializeBatch;
+}
+//# sourceMappingURL=opCompressor.d.ts.map

package/dist/opLifecycle/opCompressor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"opCompressor.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AAGvE,OAAO,EAAE,MAAM,EAAgB,MAAM,eAAe,CAAC;AAErD;;;;GAIG;AACH,qBAAa,YAAY;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAEZ,MAAM,EAAE,oBAAoB;IAIjC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAkD3C,OAAO,CAAC,cAAc;CAsBtB"}

package/dist/opLifecycle/opDecompressor.cjs

@@ -0,0 +1,132 @@
+"use strict";
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpDecompressor = void 0;
+const lz4js_1 = require("lz4js");
+const core_utils_1 = require("@fluidframework/core-utils");
+const client_utils_1 = require("@fluid-internal/client-utils");
+const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
+const containerRuntime_1 = require("../containerRuntime.cjs");
+/**
+ * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
+ * This class relies on some implicit contracts defined below:
+ * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true
+ * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
+ * 3. The final message of a batch will have batch metadata set to false
+ * 4. An individually compressed op will have undefined batch metadata and compression set to true
+ */
+class OpDecompressor {
+    constructor(logger) {
+        this.activeBatch = false;
+        this.processedCount = 0;
+        this.logger = (0, telemetry_utils_1.createChildLogger)({ logger, namespace: "OpDecompressor" });
+    }
+    processMessage(message) {
+        (0, core_utils_1.assert)(message.compression === undefined || message.compression === containerRuntime_1.CompressionAlgorithms.lz4, 0x511 /* Only lz4 compression is supported */);
+        if (message.metadata?.batch === true &&
+            this.isCompressed(message)) {
+            // Beginning of a compressed batch
+            (0, core_utils_1.assert)(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
+            this.activeBatch = true;
+            const contents = client_utils_1.IsoBuffer.from(message.contents.packedContents, "base64");
+            const decompressedMessage = (0, lz4js_1.decompress)(contents);
+            const intoString = (0, client_utils_1.Uint8ArrayToString)(decompressedMessage);
+            const asObj = JSON.parse(intoString);
+            this.rootMessageContents = asObj;
+            return {
+                message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+                state: "Accepted",
+            };
+        }
+        if (this.rootMessageContents !== undefined &&
+            message.metadata?.batch === undefined &&
+            this.activeBatch) {
+            (0, core_utils_1.assert)(message.contents === undefined, 0x512 /* Expecting empty message */);
+            // Continuation of compressed batch
+            return {
+                message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+                state: "Accepted",
+            };
+        }
+        if (this.rootMessageContents !== undefined &&
+            message.metadata?.batch === false) {
+            // End of compressed batch
+            const returnMessage = newMessage(message, this.rootMessageContents[this.processedCount++]);
+            this.activeBatch = false;
+            this.rootMessageContents = undefined;
+            this.processedCount = 0;
+            return {
+                message: returnMessage,
+                state: "Processed",
+            };
+        }
+        if (message.metadata?.batch === undefined &&
+            this.isCompressed(message)) {
+            // Single compressed message
+            (0, core_utils_1.assert)(this.activeBatch === false, 0x4ba /* shouldn't receive compressed message in middle of a batch */);
+            const contents = client_utils_1.IsoBuffer.from(message.contents.packedContents, "base64");
+            const decompressedMessage = (0, lz4js_1.decompress)(contents);
+            const intoString = new TextDecoder().decode(decompressedMessage);
+            const asObj = JSON.parse(intoString);
+            return {
+                message: newMessage(message, asObj[0]),
+                state: "Processed",
+            };
+        }
+        return {
+            message,
+            state: "Skipped",
+        };
+    }
+    isCompressed(message) {
+        if (message.compression === containerRuntime_1.CompressionAlgorithms.lz4) {
+            return true;
+        }
+        /**
+         * Back-compat self healing mechanism for ADO:3538, as loaders from
+         * version client_v2.0.0-internal.1.2.0 to client_v2.0.0-internal.2.2.0 do not
+         * support adding the proper compression metadata to compressed messages submitted
+         * by the runtime. Should be removed after the loader reaches sufficient saturation
+         * for a version greater or equal than client_v2.0.0-internal.2.2.0.
+         *
+         * The condition holds true for compressed messages, regardless of metadata. We are ultimately
+         * looking for a message with a single property `packedContents` inside `contents`, of type 'string'
+         * with a base64 encoded value.
+         */
+        try {
+            if (message.contents !== null &&
+                typeof message.contents === "object" &&
+                Object.keys(message.contents).length === 1 &&
+                typeof message.contents.packedContents ===
+                    "string" &&
+                message.contents.packedContents.length > 0 &&
+                client_utils_1.IsoBuffer.from(message.contents.packedContents, "base64").toString("base64") ===
+                    message.contents.packedContents) {
+                this.logger.sendTelemetryEvent({
+                    eventName: "LegacyCompression",
+                    type: message.type,
+                    batch: message.metadata?.batch,
+                });
+                return true;
+            }
+        }
+        catch (err) {
+            return false;
+        }
+        return false;
+    }
+}
+exports.OpDecompressor = OpDecompressor;
+// We should not be mutating the input message nor its metadata
+const newMessage = (originalMessage, contents) => ({
+    ...originalMessage,
+    contents,
+    compression: undefined,
+    // TODO: It should already be the case that we're not modifying any metadata, not clear if/why this shallow clone should be required.
+    // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+    metadata: { ...originalMessage.metadata },
+});
+//# sourceMappingURL=opDecompressor.cjs.map

package/dist/opLifecycle/opDecompressor.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"opDecompressor.cjs","sourceRoot":"","sources":["../../src/opLifecycle/opDecompressor.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,iCAAmC;AAEnC,2DAAoD;AACpD,+DAA6E;AAC7E,qEAAoE;AAEpE,8DAA4D;AAW5D;;;;;;;GAOG;AACH,MAAa,cAAc;IAM1B,YAAY,MAA4B;QALhC,gBAAW,GAAG,KAAK,CAAC;QAEpB,mBAAc,GAAG,CAAC,CAAC;QAI1B,IAAI,CAAC,MAAM,GAAG,IAAA,mCAAiB,EAAC,EAAE,MAAM,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAC1E,CAAC;IAEM,cAAc,CAAC,OAAkC;QACvD,IAAA,mBAAM,EACL,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,EACtF,KAAK,CAAC,uCAAuC,CAC7C,CAAC;QAEF,IACE,OAAO,CAAC,QAAuC,EAAE,KAAK,KAAK,IAAI;YAChE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EACzB;YACD,kCAAkC;YAClC,IAAA,mBAAM,EAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,KAAK,CAAC,4CAA4C,CAAC,CAAC;YACvF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YAExB,MAAM,QAAQ,GAAG,wBAAS,CAAC,IAAI,CAC7B,OAAO,CAAC,QAAoC,CAAC,cAAc,EAC5D,QAAQ,CACR,CAAC;YACF,MAAM,mBAAmB,GAAG,IAAA,kBAAU,EAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAA,iCAAkB,EAAC,mBAAmB,CAAC,CAAC;YAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACrC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;YAEjC,OAAO;gBACN,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;gBAC7E,KAAK,EAAE,UAAU;aACjB,CAAC;SACF;QAED,IACC,IAAI,CAAC,mBAAmB,KAAK,SAAS;YACrC,OAAO,CAAC,QAAuC,EAAE,KAAK,KAAK,SAAS;YACrE,IAAI,CAAC,WAAW,EACf;YACD,IAAA,mBAAM,EAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAE5E,mCAAmC;YACnC,OAAO;gBACN,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;gBAC7E,KAAK,EAAE,UAAU;aACjB,CAAC;SACF;QAED,IACC,IAAI,CAAC,mBAAmB,KAAK,SAAS;YACrC,OAAO,CAAC,QAAuC,EAAE,KAAK,KAAK,KAAK,EAChE;YACD,0BAA0B;YAC1B,MAAM,aAAa,GAAG,UAAU,CAC/B,OAAO,EACP,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAC/C,CAAC;YAEF,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,IAAI,CAAC,mBAAmB,GAAG,SAAS,CAAC;YACrC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;YAExB,OAAO;gBACN,OAAO,EAAE,aAAa;gBACtB,KAAK,EAAE,WAAW;aAClB,CAAC;SACF;QAED,IACE,OAAO,CAAC,QAAuC,EAAE,KAAK,KAAK,SAAS;YACrE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EACzB;YACD,4BAA4B;YAC5B,IAAA,mBAAM,EACL,IAAI,CAAC,WAAW,KAAK,KAAK,EAC1B,KAAK,CAAC,+DAA+D,CACrE,CAAC;YAEF,MAAM,QAAQ,GAAG,wBAAS,CAAC,IAAI,CAC7B,OAAO,CAAC,QAAoC,CAAC,cAAc,EAC5D,QAAQ,CACR,CAAC;YACF,MAAM,mBAAmB,GAAG,IAAA,kBAAU,EAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAErC,OAAO;gBACN,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;gBACtC,KAAK,EAAE,WAAW;aAClB,CAAC;SACF;QAED,OAAO;YACN,OAAO;YACP,KAAK,EAAE,SAAS;SAChB,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,OAAkC;QACtD,IAAI,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,EAAE;YACtD,OAAO,IAAI,CAAC;SACZ;QAED;;;;;;;;;;WAUG;QACH,IAAI;YACH,IACC,OAAO,CAAC,QAAQ,KAAK,IAAI;gBACzB,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,KAAK,CAAC;gBAC1C,OAAQ,OAAO,CAAC,QAAyC,CAAC,cAAc;oBACvE,QAAQ;gBACR,OAAO,CAAC,QAAoC,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC;gBACvE,wBAAS,CAAC,IAAI,CACZ,OAAO,CAAC,QAAoC,CAAC,cAAc,EAC5D,QAAQ,CACR,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBAClB,OAAO,CAAC,QAAoC,CAAC,cAAc,EAC5D;gBACD,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;oBAC9B,SAAS,EAAE,mBAAmB;oBAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,KAAK,EAAG,OAAO,CAAC,QAAuC,EAAE,KAAK;iBAC9D,CAAC,CAAC;gBACH,OAAO,IAAI,CAAC;aACZ;SACD;QAAC,OAAO,GAAG,EAAE;YACb,OAAO,KAAK,CAAC;SACb;QAED,OAAO,KAAK,CAAC;IACd,CAAC;CACD;AAlJD,wCAkJC;AAED,+DAA+D;AAC/D,MAAM,UAAU,GAAG,CAClB,eAA0C,EAC1C,QAAa,EACe,EAAE,CAAC,CAAC;IAChC,GAAG,eAAe;IAClB,QAAQ;IACR,WAAW,EAAE,SAAS;IACtB,qIAAqI;IACrI,4EAA4E;IAC5E,QAAQ,EAAE,EAAE,GAAI,eAAe,CAAC,QAAgB,EAAE;CAClD,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { decompress } from \"lz4js\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { assert } from \"@fluidframework/core-utils\";\nimport { IsoBuffer, Uint8ArrayToString } from \"@fluid-internal/client-utils\";\nimport { createChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport { CompressionAlgorithms } from \"../containerRuntime\";\nimport { IBatchMetadata } from \"../metadata\";\nimport { IMessageProcessingResult } from \"./definitions\";\n\n/**\n * Compression makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.\n */\ninterface IPackedContentsContents {\n\tpackedContents: string;\n}\n\n/**\n * State machine that \"unrolls\" contents of compressed batches of ops after decompressing them.\n * This class relies on some implicit contracts defined below:\n * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true\n * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set\n * 3. The final message of a batch will have batch metadata set to false\n * 4. An individually compressed op will have undefined batch metadata and compression set to true\n */\nexport class OpDecompressor {\n\tprivate activeBatch = false;\n\tprivate rootMessageContents: any | undefined;\n\tprivate processedCount = 0;\n\tprivate readonly logger;\n\n\tconstructor(logger: ITelemetryBaseLogger) {\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpDecompressor\" });\n\t}\n\n\tpublic processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {\n\t\tassert(\n\t\t\tmessage.compression === undefined || message.compression === CompressionAlgorithms.lz4,\n\t\t\t0x511 /* Only lz4 compression is supported */,\n\t\t);\n\n\t\tif (\n\t\t\t(message.metadata as IBatchMetadata | undefined)?.batch === true &&\n\t\t\tthis.isCompressed(message)\n\t\t) {\n\t\t\t// Beginning of a compressed batch\n\t\t\tassert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);\n\t\t\tthis.activeBatch = true;\n\n\t\t\tconst contents = IsoBuffer.from(\n\t\t\t\t(message.contents as IPackedContentsContents).packedContents,\n\t\t\t\t\"base64\",\n\t\t\t);\n\t\t\tconst decompressedMessage = decompress(contents);\n\t\t\tconst intoString = Uint8ArrayToString(decompressedMessage);\n\t\t\tconst asObj = JSON.parse(intoString);\n\t\t\tthis.rootMessageContents = asObj;\n\n\t\t\treturn {\n\t\t\t\tmessage: newMessage(message, this.rootMessageContents[this.processedCount++]),\n\t\t\t\tstate: \"Accepted\",\n\t\t\t};\n\t\t}\n\n\t\tif (\n\t\t\tthis.rootMessageContents !== undefined &&\n\t\t\t(message.metadata as IBatchMetadata | undefined)?.batch === undefined &&\n\t\t\tthis.activeBatch\n\t\t) {\n\t\t\tassert(message.contents === undefined, 0x512 /* Expecting empty message */);\n\n\t\t\t// Continuation of compressed batch\n\t\t\treturn {\n\t\t\t\tmessage: newMessage(message, this.rootMessageContents[this.processedCount++]),\n\t\t\t\tstate: \"Accepted\",\n\t\t\t};\n\t\t}\n\n\t\tif (\n\t\t\tthis.rootMessageContents !== undefined &&\n\t\t\t(message.metadata as IBatchMetadata | undefined)?.batch === false\n\t\t) {\n\t\t\t// End of compressed batch\n\t\t\tconst returnMessage = newMessage(\n\t\t\t\tmessage,\n\t\t\t\tthis.rootMessageContents[this.processedCount++],\n\t\t\t);\n\n\t\t\tthis.activeBatch = false;\n\t\t\tthis.rootMessageContents = undefined;\n\t\t\tthis.processedCount = 0;\n\n\t\t\treturn {\n\t\t\t\tmessage: returnMessage,\n\t\t\t\tstate: \"Processed\",\n\t\t\t};\n\t\t}\n\n\t\tif (\n\t\t\t(message.metadata as IBatchMetadata | undefined)?.batch === undefined &&\n\t\t\tthis.isCompressed(message)\n\t\t) {\n\t\t\t// Single compressed message\n\t\t\tassert(\n\t\t\t\tthis.activeBatch === false,\n\t\t\t\t0x4ba /* shouldn't receive compressed message in middle of a batch */,\n\t\t\t);\n\n\t\t\tconst contents = IsoBuffer.from(\n\t\t\t\t(message.contents as IPackedContentsContents).packedContents,\n\t\t\t\t\"base64\",\n\t\t\t);\n\t\t\tconst decompressedMessage = decompress(contents);\n\t\t\tconst intoString = new TextDecoder().decode(decompressedMessage);\n\t\t\tconst asObj = JSON.parse(intoString);\n\n\t\t\treturn {\n\t\t\t\tmessage: newMessage(message, asObj[0]),\n\t\t\t\tstate: \"Processed\",\n\t\t\t};\n\t\t}\n\n\t\treturn {\n\t\t\tmessage,\n\t\t\tstate: \"Skipped\",\n\t\t};\n\t}\n\n\tprivate isCompressed(message: ISequencedDocumentMessage) {\n\t\tif (message.compression === CompressionAlgorithms.lz4) {\n\t\t\treturn true;\n\t\t}\n\n\t\t/**\n\t\t * Back-compat self healing mechanism for ADO:3538, as loaders from\n\t\t * version client_v2.0.0-internal.1.2.0 to client_v2.0.0-internal.2.2.0 do not\n\t\t * support adding the proper compression metadata to compressed messages submitted\n\t\t * by the runtime. Should be removed after the loader reaches sufficient saturation\n\t\t * for a version greater or equal than client_v2.0.0-internal.2.2.0.\n\t\t *\n\t\t * The condition holds true for compressed messages, regardless of metadata. We are ultimately\n\t\t * looking for a message with a single property `packedContents` inside `contents`, of type 'string'\n\t\t * with a base64 encoded value.\n\t\t */\n\t\ttry {\n\t\t\tif (\n\t\t\t\tmessage.contents !== null &&\n\t\t\t\ttypeof message.contents === \"object\" &&\n\t\t\t\tObject.keys(message.contents).length === 1 &&\n\t\t\t\ttypeof (message.contents as { packedContents?: unknown }).packedContents ===\n\t\t\t\t\t\"string\" &&\n\t\t\t\t(message.contents as IPackedContentsContents).packedContents.length > 0 &&\n\t\t\t\tIsoBuffer.from(\n\t\t\t\t\t(message.contents as IPackedContentsContents).packedContents,\n\t\t\t\t\t\"base64\",\n\t\t\t\t).toString(\"base64\") ===\n\t\t\t\t\t(message.contents as IPackedContentsContents).packedContents\n\t\t\t) {\n\t\t\t\tthis.logger.sendTelemetryEvent({\n\t\t\t\t\teventName: \"LegacyCompression\",\n\t\t\t\t\ttype: message.type,\n\t\t\t\t\tbatch: (message.metadata as IBatchMetadata | undefined)?.batch,\n\t\t\t\t});\n\t\t\t\treturn true;\n\t\t\t}\n\t\t} catch (err) {\n\t\t\treturn false;\n\t\t}\n\n\t\treturn false;\n\t}\n}\n\n// We should not be mutating the input message nor its metadata\nconst newMessage = (\n\toriginalMessage: ISequencedDocumentMessage,\n\tcontents: any,\n): ISequencedDocumentMessage => ({\n\t...originalMessage,\n\tcontents,\n\tcompression: undefined,\n\t// TODO: It should already be the case that we're not modifying any metadata, not clear if/why this shallow clone should be required.\n\t// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion\n\tmetadata: { ...(originalMessage.metadata as any) },\n});\n"]}

package/dist/opLifecycle/opDecompressor.d.ts

@@ -0,0 +1,25 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { IMessageProcessingResult } from "./definitions";
+/**
+ * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
+ * This class relies on some implicit contracts defined below:
+ * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true
+ * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
+ * 3. The final message of a batch will have batch metadata set to false
+ * 4. An individually compressed op will have undefined batch metadata and compression set to true
+ */
+export declare class OpDecompressor {
+    private activeBatch;
+    private rootMessageContents;
+    private processedCount;
+    private readonly logger;
+    constructor(logger: ITelemetryBaseLogger);
+    processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult;
+    private isCompressed;
+}
+//# sourceMappingURL=opDecompressor.d.ts.map

package/dist/opLifecycle/opDecompressor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"opDecompressor.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opDecompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAIjF,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AAGvE,OAAO,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AASzD;;;;;;;GAOG;AACH,qBAAa,cAAc;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,mBAAmB,CAAkB;IAC7C,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAEZ,MAAM,EAAE,oBAAoB;IAIjC,cAAc,CAAC,OAAO,EAAE,yBAAyB,GAAG,wBAAwB;IA6FnF,OAAO,CAAC,YAAY;CA2CpB"}

package/dist/opLifecycle/opGroupingManager.cjs

@@ -0,0 +1,95 @@
+"use strict";
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpGroupingManager = void 0;
+const core_utils_1 = require("@fluidframework/core-utils");
+const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
+function isGroupContents(opContents) {
+    return opContents?.type === OpGroupingManager.groupedBatchOp;
+}
+class OpGroupingManager {
+    constructor(config, logger) {
+        this.config = config;
+        this.logger = (0, telemetry_utils_1.createChildLogger)({ logger, namespace: "OpGroupingManager" });
+    }
+    groupBatch(batch) {
+        if (!this.shouldGroup(batch)) {
+            return batch;
+        }
+        if (batch.content.length >= 1000) {
+            this.logger.sendTelemetryEvent({
+                eventName: "GroupLargeBatch",
+                length: batch.content.length,
+                threshold: this.config.opCountThreshold,
+                reentrant: batch.hasReentrantOps,
+                referenceSequenceNumber: batch.content[0].referenceSequenceNumber,
+            });
+        }
+        for (const message of batch.content) {
+            if (message.metadata) {
+                const keys = Object.keys(message.metadata);
+                (0, core_utils_1.assert)(keys.length < 2, 0x5dd /* cannot group ops with metadata */);
+                (0, core_utils_1.assert)(keys.length === 0 || keys[0] === "batch", 0x5de /* unexpected op metadata */);
+            }
+        }
+        const serializedContent = JSON.stringify({
+            type: OpGroupingManager.groupedBatchOp,
+            contents: batch.content.map((message) => ({
+                contents: message.contents === undefined ? undefined : JSON.parse(message.contents),
+                metadata: message.metadata,
+                compression: message.compression,
+            })),
+        });
+        const groupedBatch = {
+            ...batch,
+            content: [
+                {
+                    localOpMetadata: undefined,
+                    metadata: undefined,
+                    referenceSequenceNumber: batch.content[0].referenceSequenceNumber,
+                    contents: serializedContent,
+                    type: OpGroupingManager.groupedBatchOp,
+                },
+            ],
+        };
+        return groupedBatch;
+    }
+    ungroupOp(op) {
+        let fakeCsn = 1;
+        if (!isGroupContents(op.contents)) {
+            // Align the worlds of what clientSequenceNumber represents when grouped batching is enabled
+            if (this.config.groupedBatchingEnabled) {
+                return [
+                    {
+                        ...op,
+                        clientSequenceNumber: fakeCsn,
+                    },
+                ];
+            }
+            return [op];
+        }
+        const messages = op.contents.contents;
+        return messages.map((subMessage) => ({
+            ...op,
+            clientSequenceNumber: fakeCsn++,
+            contents: subMessage.contents,
+            metadata: subMessage.metadata,
+            compression: subMessage.compression,
+        }));
+    }
+    shouldGroup(batch) {
+        return (
+        // Grouped batching must be enabled
+        this.config.groupedBatchingEnabled &&
+            // The number of ops in the batch must surpass the configured threshold
+            batch.content.length >= this.config.opCountThreshold &&
+            // Support for reentrant batches must be explicitly enabled
+            (this.config.reentrantBatchGroupingEnabled || batch.hasReentrantOps !== true));
+    }
+}
+exports.OpGroupingManager = OpGroupingManager;
+OpGroupingManager.groupedBatchOp = "groupedBatch";
+//# sourceMappingURL=opGroupingManager.cjs.map

package/dist/opLifecycle/opGroupingManager.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"opGroupingManager.cjs","sourceRoot":"","sources":["../../src/opLifecycle/opGroupingManager.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,2DAAoD;AAEpD,qEAAoE;AAmBpE,SAAS,eAAe,CAAC,UAAe;IACvC,OAAO,UAAU,EAAE,IAAI,KAAK,iBAAiB,CAAC,cAAc,CAAC;AAC9D,CAAC;AAQD,MAAa,iBAAiB;IAI7B,YACkB,MAA+B,EAChD,MAA4B;QADX,WAAM,GAAN,MAAM,CAAyB;QAGhD,IAAI,CAAC,MAAM,GAAG,IAAA,mCAAiB,EAAC,EAAE,MAAM,EAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAC7E,CAAC;IAEM,UAAU,CAAC,KAAa;QAC9B,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE;YAC7B,OAAO,KAAK,CAAC;SACb;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,EAAE;YACjC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;gBAC9B,SAAS,EAAE,iBAAiB;gBAC5B,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;gBAC5B,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB;gBACvC,SAAS,EAAE,KAAK,CAAC,eAAe;gBAChC,uBAAuB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,uBAAuB;aACjE,CAAC,CAAC;SACH;QAED,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE;YACpC,IAAI,OAAO,CAAC,QAAQ,EAAE;gBACrB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAC3C,IAAA,mBAAM,EAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,CAAC,oCAAoC,CAAC,CAAC;gBACpE,IAAA,mBAAM,EACL,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,OAAO,EACxC,KAAK,CAAC,4BAA4B,CAClC,CAAC;aACF;SACD;QAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,SAAS,CAAC;YACxC,IAAI,EAAE,iBAAiB,CAAC,cAAc;YACtC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAkB,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;gBACnF,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;aAChC,CAAC,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,YAAY,GAAW;YAC5B,GAAG,KAAK;YACR,OAAO,EAAE;gBACR;oBACC,eAAe,EAAE,SAAS;oBAC1B,QAAQ,EAAE,SAAS;oBACnB,uBAAuB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,uBAAuB;oBACjE,QAAQ,EAAE,iBAAiB;oBAC3B,IAAI,EAAE,iBAAiB,CAAC,cAAsC;iBAC9D;aACD;SACD,CAAC;QACF,OAAO,YAAY,CAAC;IACrB,CAAC;IAEM,SAAS,CAAC,EAA6B;QAC7C,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE;YAClC,4FAA4F;YAC5F,IAAI,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE;gBACvC,OAAO;oBACN;wBACC,GAAG,EAAE;wBACL,oBAAoB,EAAE,OAAO;qBAC7B;iBACD,CAAC;aACF;YACD,OAAO,CAAC,EAAE,CAAC,CAAC;SACZ;QAED,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACtC,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACpC,GAAG,EAAE;YACL,oBAAoB,EAAE,OAAO,EAAE;YAC/B,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,WAAW,EAAE,UAAU,CAAC,WAAW;SACnC,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,WAAW,CAAC,KAAa;QAC/B,OAAO;QACN,mCAAmC;QACnC,IAAI,CAAC,MAAM,CAAC,sBAAsB;YAClC,uEAAuE;YACvE,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB;YACpD,2DAA2D;YAC3D,CAAC,IAAI,CAAC,MAAM,CAAC,6BAA6B,IAAI,KAAK,CAAC,eAAe,KAAK,IAAI,CAAC,CAC7E,CAAC;IACH,CAAC;;AA/FF,8CAgGC;AA/FgB,gCAAc,GAAG,cAAc,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { assert } from \"@fluidframework/core-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { createChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport { ContainerMessageType } from \"../messageTypes\";\nimport { IBatch } from \"./definitions\";\n\n/**\n * Grouping makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.\n */\ninterface IGroupedBatchMessageContents {\n\ttype: typeof OpGroupingManager.groupedBatchOp;\n\tcontents: IGroupedMessage[];\n}\n\ninterface IGroupedMessage {\n\tcontents?: unknown;\n\tmetadata?: Record<string, unknown>;\n\tcompression?: string;\n}\n\nfunction isGroupContents(opContents: any): opContents is IGroupedBatchMessageContents {\n\treturn opContents?.type === OpGroupingManager.groupedBatchOp;\n}\n\nexport interface OpGroupingManagerConfig {\n\treadonly groupedBatchingEnabled: boolean;\n\treadonly opCountThreshold: number;\n\treadonly reentrantBatchGroupingEnabled: boolean;\n}\n\nexport class OpGroupingManager {\n\tstatic readonly groupedBatchOp = \"groupedBatch\";\n\tprivate readonly logger;\n\n\tconstructor(\n\t\tprivate readonly config: OpGroupingManagerConfig,\n\t\tlogger: ITelemetryBaseLogger,\n\t) {\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpGroupingManager\" });\n\t}\n\n\tpublic groupBatch(batch: IBatch): IBatch {\n\t\tif (!this.shouldGroup(batch)) {\n\t\t\treturn batch;\n\t\t}\n\n\t\tif (batch.content.length >= 1000) {\n\t\t\tthis.logger.sendTelemetryEvent({\n\t\t\t\teventName: \"GroupLargeBatch\",\n\t\t\t\tlength: batch.content.length,\n\t\t\t\tthreshold: this.config.opCountThreshold,\n\t\t\t\treentrant: batch.hasReentrantOps,\n\t\t\t\treferenceSequenceNumber: batch.content[0].referenceSequenceNumber,\n\t\t\t});\n\t\t}\n\n\t\tfor (const message of batch.content) {\n\t\t\tif (message.metadata) {\n\t\t\t\tconst keys = Object.keys(message.metadata);\n\t\t\t\tassert(keys.length < 2, 0x5dd /* cannot group ops with metadata */);\n\t\t\t\tassert(\n\t\t\t\t\tkeys.length === 0 || keys[0] === \"batch\",\n\t\t\t\t\t0x5de /* unexpected op metadata */,\n\t\t\t\t);\n\t\t\t}\n\t\t}\n\n\t\tconst serializedContent = JSON.stringify({\n\t\t\ttype: OpGroupingManager.groupedBatchOp,\n\t\t\tcontents: batch.content.map<IGroupedMessage>((message) => ({\n\t\t\t\tcontents: message.contents === undefined ? undefined : JSON.parse(message.contents),\n\t\t\t\tmetadata: message.metadata,\n\t\t\t\tcompression: message.compression,\n\t\t\t})),\n\t\t});\n\n\t\tconst groupedBatch: IBatch = {\n\t\t\t...batch,\n\t\t\tcontent: [\n\t\t\t\t{\n\t\t\t\t\tlocalOpMetadata: undefined,\n\t\t\t\t\tmetadata: undefined,\n\t\t\t\t\treferenceSequenceNumber: batch.content[0].referenceSequenceNumber,\n\t\t\t\t\tcontents: serializedContent,\n\t\t\t\t\ttype: OpGroupingManager.groupedBatchOp as ContainerMessageType,\n\t\t\t\t},\n\t\t\t],\n\t\t};\n\t\treturn groupedBatch;\n\t}\n\n\tpublic ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[] {\n\t\tlet fakeCsn = 1;\n\t\tif (!isGroupContents(op.contents)) {\n\t\t\t// Align the worlds of what clientSequenceNumber represents when grouped batching is enabled\n\t\t\tif (this.config.groupedBatchingEnabled) {\n\t\t\t\treturn [\n\t\t\t\t\t{\n\t\t\t\t\t\t...op,\n\t\t\t\t\t\tclientSequenceNumber: fakeCsn,\n\t\t\t\t\t},\n\t\t\t\t];\n\t\t\t}\n\t\t\treturn [op];\n\t\t}\n\n\t\tconst messages = op.contents.contents;\n\t\treturn messages.map((subMessage) => ({\n\t\t\t...op,\n\t\t\tclientSequenceNumber: fakeCsn++,\n\t\t\tcontents: subMessage.contents,\n\t\t\tmetadata: subMessage.metadata,\n\t\t\tcompression: subMessage.compression,\n\t\t}));\n\t}\n\n\tpublic shouldGroup(batch: IBatch): boolean {\n\t\treturn (\n\t\t\t// Grouped batching must be enabled\n\t\t\tthis.config.groupedBatchingEnabled &&\n\t\t\t// The number of ops in the batch must surpass the configured threshold\n\t\t\tbatch.content.length >= this.config.opCountThreshold &&\n\t\t\t// Support for reentrant batches must be explicitly enabled\n\t\t\t(this.config.reentrantBatchGroupingEnabled || batch.hasReentrantOps !== true)\n\t\t);\n\t}\n}\n"]}

package/dist/opLifecycle/opGroupingManager.d.ts

@@ -0,0 +1,22 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { IBatch } from "./definitions";
+export interface OpGroupingManagerConfig {
+    readonly groupedBatchingEnabled: boolean;
+    readonly opCountThreshold: number;
+    readonly reentrantBatchGroupingEnabled: boolean;
+}
+export declare class OpGroupingManager {
+    private readonly config;
+    static readonly groupedBatchOp = "groupedBatch";
+    private readonly logger;
+    constructor(config: OpGroupingManagerConfig, logger: ITelemetryBaseLogger);
+    groupBatch(batch: IBatch): IBatch;
+    ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[];
+    shouldGroup(batch: IBatch): boolean;
+}
+//# sourceMappingURL=opGroupingManager.d.ts.map

package/dist/opLifecycle/opGroupingManager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"opGroupingManager.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opGroupingManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAEjF,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AAEvE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAoBvC,MAAM,WAAW,uBAAuB;IACvC,QAAQ,CAAC,sBAAsB,EAAE,OAAO,CAAC;IACzC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,6BAA6B,EAAE,OAAO,CAAC;CAChD;AAED,qBAAa,iBAAiB;IAK5B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJxB,MAAM,CAAC,QAAQ,CAAC,cAAc,kBAAkB;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAGN,MAAM,EAAE,uBAAuB,EAChD,MAAM,EAAE,oBAAoB;IAKtB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAkDjC,SAAS,CAAC,EAAE,EAAE,yBAAyB,GAAG,yBAAyB,EAAE;IAyBrE,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;CAU1C"}

package/dist/opLifecycle/opSplitter.cjs

@@ -0,0 +1,202 @@
+"use strict";
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.splitOp = exports.OpSplitter = void 0;
+const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
+const core_utils_1 = require("@fluidframework/core-utils");
+const messageTypes_1 = require("../messageTypes.cjs");
+const batchManager_1 = require("./batchManager.cjs");
+/**
+ * Responsible for creating and reconstructing chunked messages.
+ */
+class OpSplitter {
+    constructor(chunks, submitBatchFn, chunkSizeInBytes, maxBatchSizeInBytes, logger) {
+        this.submitBatchFn = submitBatchFn;
+        this.chunkSizeInBytes = chunkSizeInBytes;
+        this.maxBatchSizeInBytes = maxBatchSizeInBytes;
+        this.chunkMap = new Map(chunks);
+        this.logger = (0, telemetry_utils_1.createChildLogger)({ logger, namespace: "OpSplitter" });
+    }
+    get isBatchChunkingEnabled() {
+        return this.chunkSizeInBytes < Number.POSITIVE_INFINITY && this.submitBatchFn !== undefined;
+    }
+    get chunks() {
+        return this.chunkMap;
+    }
+    processRemoteMessage(message) {
+        if (message.type !== messageTypes_1.ContainerMessageType.ChunkedOp) {
+            return {
+                message,
+                state: "Skipped",
+            };
+        }
+        // TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
+        // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+        const clientId = message.clientId;
+        const chunkedContent = message.contents;
+        this.addChunk(clientId, chunkedContent, message);
+        if (chunkedContent.chunkId < chunkedContent.totalChunks) {
+            // We are processing the op in chunks but haven't reached
+            // the last chunk yet in order to reconstruct the original op
+            return {
+                message,
+                state: "Accepted",
+            };
+        }
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const serializedContent = this.chunkMap.get(clientId).join("");
+        this.clearPartialChunks(clientId);
+        const newMessage = { ...message };
+        newMessage.contents = serializedContent === "" ? undefined : JSON.parse(serializedContent);
+        newMessage.type = chunkedContent.originalType;
+        newMessage.metadata = chunkedContent.originalMetadata;
+        newMessage.compression = chunkedContent.originalCompression;
+        return {
+            message: newMessage,
+            state: "Processed",
+        };
+    }
+    clearPartialChunks(clientId) {
+        if (this.chunkMap.has(clientId)) {
+            this.chunkMap.delete(clientId);
+        }
+    }
+    addChunk(clientId, chunkedContent, originalMessage) {
+        let map = this.chunkMap.get(clientId);
+        if (map === undefined) {
+            map = [];
+            this.chunkMap.set(clientId, map);
+        }
+        if (chunkedContent.chunkId !== map.length + 1) {
+            // We are expecting the chunks to be processed sequentially, in the same order as they are sent.
+            // Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)
+            // holding the existing chunks for that particular clientId.
+            throw new telemetry_utils_1.DataCorruptionError("Chunk Id mismatch", {
+                ...(0, telemetry_utils_1.extractSafePropertiesFromMessage)(originalMessage),
+                chunkMapLength: map.length,
+                chunkId: chunkedContent.chunkId,
+                totalChunks: chunkedContent.totalChunks,
+            });
+        }
+        map.push(chunkedContent.contents);
+    }
+    /**
+     * Splits the first op of a compressed batch in chunks, sends the chunks separately and
+     * returns a new batch composed of the last chunk and the rest of the ops in the original batch.
+     *
+     * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops
+     * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original
+     * uncompressed ops at ingestion in the runtime.
+     *
+     * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire
+     * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.
+     *
+     * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch
+     * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch
+     * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed
+     * (as it is the last chunk).
+     *
+     * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
+     * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.
+     *
+     * @param batch - the compressed batch which needs to be processed
+     * @returns A new adjusted batch which can be sent over the wire
+     */
+    splitFirstBatchMessage(batch) {
+        (0, core_utils_1.assert)(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);
+        (0, core_utils_1.assert)(batch.contentSizeInBytes > 0 && batch.content.length > 0, 0x514 /* Batch needs to be non-empty */);
+        (0, core_utils_1.assert)(batch.referenceSequenceNumber !== undefined, 0x58a /* Batch must have a reference sequence number if non-empty */);
+        (0, core_utils_1.assert)(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);
+        (0, core_utils_1.assert)(this.chunkSizeInBytes < this.maxBatchSizeInBytes, 0x516 /* Chunk size needs to be smaller than the max batch size */);
+        const firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split
+        (0, core_utils_1.assert)((firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes, 0x518 /* First message in the batch needs to be chunkable */);
+        const restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers
+        const socketSize = (0, batchManager_1.estimateSocketSize)(batch);
+        const chunks = (0, exports.splitOp)(firstMessage, this.chunkSizeInBytes, 
+        // If we estimate that the socket batch size will exceed the batch limit
+        // we will inject an empty op to minimize the risk of the payload failing due to
+        // the overhead from the trailing empty ops in the batch.
+        socketSize >= this.maxBatchSizeInBytes);
+        (0, core_utils_1.assert)(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);
+        // Send the first N-1 chunks immediately
+        for (const chunk of chunks.slice(0, -1)) {
+            this.submitBatchFn([chunkToBatchMessage(chunk, batch.referenceSequenceNumber)], batch.referenceSequenceNumber);
+        }
+        // The last chunk will be part of the new batch and needs to
+        // preserve the batch metadata of the original batch
+        const lastChunk = chunkToBatchMessage(chunks[chunks.length - 1], batch.referenceSequenceNumber, { batch: firstMessage.metadata?.batch });
+        this.logger.sendPerformanceEvent({
+            // Used to be "Chunked compressed batch"
+            eventName: "CompressedChunkedBatch",
+            length: batch.content.length,
+            sizeInBytes: batch.contentSizeInBytes,
+            chunks: chunks.length,
+            chunkSizeInBytes: this.chunkSizeInBytes,
+            socketSize,
+        });
+        return {
+            content: [lastChunk, ...restOfMessages],
+            contentSizeInBytes: lastChunk.contents?.length ?? 0,
+            referenceSequenceNumber: batch.referenceSequenceNumber,
+        };
+    }
+}
+exports.OpSplitter = OpSplitter;
+const chunkToBatchMessage = (chunk, referenceSequenceNumber, metadata = undefined) => {
+    const payload = {
+        type: messageTypes_1.ContainerMessageType.ChunkedOp,
+        contents: chunk,
+    };
+    return {
+        contents: JSON.stringify(payload),
+        type: payload.type,
+        metadata,
+        localOpMetadata: undefined,
+        referenceSequenceNumber,
+    };
+};
+/**
+ * Splits an op into smaller ops (chunks), based on the size of the op and the `chunkSizeInBytes` parameter.
+ *
+ * The last op of the result will be bundled with empty ops in the same batch. There is a risk of the batch payload
+ * exceeding the 1MB limit due to the overhead from the empty ops. If the last op is large, the risk is even higher.
+ * To minimize the odds, an extra empty op can be added to the result using the `extraOp` parameter.
+ *
+ * @param op - the op to be split
+ * @param chunkSizeInBytes - how large should the chunks be
+ * @param extraOp - should an extra empty op be added to the result
+ * @returns an array of chunked ops
+ */
+const splitOp = (op, chunkSizeInBytes, extraOp = false) => {
+    const chunks = [];
+    (0, core_utils_1.assert)(op.contents !== undefined && op.contents !== null, 0x51a /* We should have something to chunk */);
+    const contentLength = op.contents.length;
+    const chunkCount = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1 + (extraOp ? 1 : 0);
+    let offset = 0;
+    for (let chunkId = 1; chunkId <= chunkCount; chunkId++) {
+        const chunk = {
+            chunkId,
+            contents: op.contents.substr(offset, chunkSizeInBytes),
+            originalType: op.type,
+            totalChunks: chunkCount,
+        };
+        if (chunkId === chunkCount) {
+            // We don't need to port these to all the chunks,
+            // as we rebuild the original op when we process the
+            // last chunk, therefore it is the only one that needs it.
+            chunk.originalMetadata = op.metadata;
+            chunk.originalCompression = op.compression;
+        }
+        chunks.push(chunk);
+        offset += chunkSizeInBytes;
+        (0, core_utils_1.assert)(chunkId >= chunkCount - 1 || offset <= contentLength, 0x58b /* Content offset within bounds */);
+    }
+    (0, core_utils_1.assert)(offset >= contentLength, 0x58c /* Content offset equal or larger than content length */);
+    (0, core_utils_1.assert)(chunks.length === chunkCount, 0x5a5 /* Expected number of chunks */);
+    return chunks;
+};
+exports.splitOp = splitOp;
+//# sourceMappingURL=opSplitter.cjs.map