@fluidframework/container-runtime 2.0.0-rc.1.0.4 → 2.0.0-rc.2.0.0

package/lib/test/opLifecycle/opDecompressor.spec.js

@@ -0,0 +1,218 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { strict as assert } from "assert";
+import { compress } from "lz4js";
+import { IsoBuffer } from "@fluid-internal/client-utils";
+import { MockLogger } from "@fluidframework/telemetry-utils";
+import { ContainerMessageType } from "../../index.js";
+import { OpDecompressor } from "../../opLifecycle/index.js";
+function generateCompressedBatchMessage(length) {
+    const batch = [];
+    for (let i = 0; i < length; i++) {
+        // Actual Op and contents aren't important. Values are not realistic.
+        batch.push({
+            type: ContainerMessageType.FluidDataStoreOp,
+            contents: `value${i}`,
+        });
+    }
+    const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(batch));
+    const compressedContents = compress(contentsAsBuffer);
+    const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
+    const messageBase = {
+        contents: { packedContents: compressedContent },
+        metadata: { meta: "data" },
+        clientId: "clientId",
+        sequenceNumber: 1,
+        minimumSequenceNumber: 1,
+        clientSequenceNumber: 1,
+        referenceSequenceNumber: 1,
+        type: "type",
+        timestamp: 1,
+        compression: "lz4",
+    };
+    // Single compressed message won't have batch metadata
+    if (length === 1) {
+        return messageBase;
+    }
+    return {
+        ...messageBase,
+        // TODO: It's not clear if this shallow clone is required, as opposed to just setting "batch" to false.
+        // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+        metadata: { ...messageBase.metadata, batch: true },
+    };
+}
+const emptyMessage = {
+    contents: undefined,
+    clientId: "clientId",
+    sequenceNumber: 1,
+    minimumSequenceNumber: 1,
+    clientSequenceNumber: 1,
+    referenceSequenceNumber: 1,
+    type: "type",
+    timestamp: 1,
+};
+const endBatchEmptyMessage = {
+    contents: {},
+    metadata: { batch: false },
+    clientId: "clientId",
+    sequenceNumber: 1,
+    minimumSequenceNumber: 1,
+    clientSequenceNumber: 1,
+    referenceSequenceNumber: 1,
+    type: "type",
+    timestamp: 1,
+};
+describe("OpDecompressor", () => {
+    const mockLogger = new MockLogger();
+    let decompressor;
+    beforeEach(() => {
+        mockLogger.clear();
+        decompressor = new OpDecompressor(mockLogger);
+    });
+    it("Processes single compressed op", () => {
+        const result = decompressor.processMessage(generateCompressedBatchMessage(1));
+        assert.equal(result.state, "Processed");
+        assert.strictEqual(result.message.contents.contents, "value0");
+        assert.strictEqual(result.message.metadata?.compressed, undefined);
+        assert.strictEqual(result.message.compression, undefined);
+    });
+    // Back-compat self healing mechanism for ADO:3538
+    it("Processes single compressed op without compression markers", () => {
+        const result = decompressor.processMessage({
+            ...generateCompressedBatchMessage(1),
+            compression: undefined,
+        });
+        assert.equal(result.state, "Processed");
+        assert.strictEqual(result.message.contents.contents, "value0");
+        assert.strictEqual(result.message.metadata?.compressed, undefined);
+        assert.strictEqual(result.message.compression, undefined);
+        mockLogger.assertMatch([
+            {
+                eventName: "OpDecompressor:LegacyCompression",
+                category: "generic",
+            },
+        ]);
+    });
+    it("Expecting only lz4 compression", () => {
+        assert.throws(() => decompressor.processMessage({
+            ...generateCompressedBatchMessage(5),
+            compression: "gzip",
+        }));
+    });
+    it("Processes multiple compressed ops", () => {
+        const rootMessage = generateCompressedBatchMessage(5);
+        const firstMessageResult = decompressor.processMessage(rootMessage);
+        assert.equal(firstMessageResult.state, "Accepted");
+        assert.strictEqual(firstMessageResult.message.contents.contents, "value0");
+        assert.strictEqual(firstMessageResult.message.metadata
+            ?.compressed, undefined);
+        assert.strictEqual(firstMessageResult.message.compression, undefined);
+        for (let i = 1; i < 4; i++) {
+            const result = decompressor.processMessage(emptyMessage);
+            assert.equal(result.state, "Accepted");
+            assert.strictEqual(result.message.contents.contents, `value${i}`);
+            assert.strictEqual(result.message.metadata?.compressed, undefined);
+            assert.strictEqual(result.message.compression, undefined);
+        }
+        const lastMessageResult = decompressor.processMessage(endBatchEmptyMessage);
+        assert.equal(lastMessageResult.state, "Processed");
+        assert.strictEqual(lastMessageResult.message.contents.contents, "value4");
+        assert.strictEqual(lastMessageResult.message.metadata
+            ?.compressed, undefined);
+        assert.strictEqual(lastMessageResult.message.compression, undefined);
+    });
+    it("Expecting empty messages in the middle of the compressed batch", () => {
+        const rootMessage = generateCompressedBatchMessage(5);
+        const firstMessageResult = decompressor.processMessage(rootMessage);
+        assert.equal(firstMessageResult.state, "Accepted");
+        assert.strictEqual(firstMessageResult.message.contents.contents, "value0");
+        assert.throws(() => decompressor.processMessage({ ...emptyMessage, contents: {} }));
+    });
+    it("Processes multiple batches of compressed ops", () => {
+        const rootMessage = generateCompressedBatchMessage(5);
+        const firstMessageResult = decompressor.processMessage(rootMessage);
+        assert.equal(firstMessageResult.state, "Accepted");
+        assert.strictEqual(firstMessageResult.message.contents.contents, "value0");
+        for (let i = 1; i < 4; i++) {
+            const result = decompressor.processMessage(emptyMessage);
+            assert.equal(result.state, "Accepted");
+            assert.strictEqual(result.message.contents.contents, `value${i}`);
+        }
+        const lastMessageResult = decompressor.processMessage(endBatchEmptyMessage);
+        assert.equal(lastMessageResult.state, "Processed");
+        assert.strictEqual(lastMessageResult.message.contents.contents, "value4");
+        const nextRootMessage = generateCompressedBatchMessage(3);
+        const nextFirstMessageResult = decompressor.processMessage(nextRootMessage);
+        assert.equal(nextFirstMessageResult.state, "Accepted");
+        assert.strictEqual(nextFirstMessageResult.message.contents.contents, "value0");
+        const middleMessageResult = decompressor.processMessage(emptyMessage);
+        assert.equal(middleMessageResult.state, "Accepted");
+        assert.strictEqual(middleMessageResult.message.contents.contents, "value1");
+        const endBatchEmptyMessageResult = decompressor.processMessage(endBatchEmptyMessage);
+        assert.equal(endBatchEmptyMessageResult.state, "Processed");
+        assert.strictEqual(endBatchEmptyMessageResult.message.contents.contents, "value2");
+    });
+    it("Ignores ops without compression", () => {
+        const rootMessages = [
+            {
+                // Back-compat self healing mechanism for ADO:3538,
+                // the message should have a `packedContents` property.
+                contents: { some: "contents" },
+                metadata: { meta: "data" },
+                clientId: "clientId",
+                sequenceNumber: 1,
+                term: 1,
+                minimumSequenceNumber: 1,
+                clientSequenceNumber: 1,
+                referenceSequenceNumber: 1,
+                type: "type",
+                timestamp: 1,
+            },
+            {
+                // Back-compat self healing mechanism for ADO:3538,
+                contents: { packedContents: "packedContents is not base64 encoded" },
+                metadata: { meta: "data" },
+                clientId: "clientId",
+                sequenceNumber: 1,
+                term: 1,
+                minimumSequenceNumber: 1,
+                clientSequenceNumber: 1,
+                referenceSequenceNumber: 1,
+                type: "type",
+                timestamp: 1,
+            },
+            {
+                // Back-compat self healing mechanism for ADO:3538,
+                contents: { packedContents: "YmFzZTY0IGNvbnRlbnQ=", some: "contents" },
+                metadata: { meta: "data" },
+                clientId: "clientId",
+                sequenceNumber: 1,
+                term: 1,
+                minimumSequenceNumber: 1,
+                clientSequenceNumber: 1,
+                referenceSequenceNumber: 1,
+                type: "type",
+                timestamp: 1,
+            },
+            {
+                metadata: { meta: "data" },
+                clientId: "clientId",
+                sequenceNumber: 1,
+                term: 1,
+                minimumSequenceNumber: 1,
+                clientSequenceNumber: 1,
+                referenceSequenceNumber: 1,
+                type: "type",
+                timestamp: 1,
+            },
+        ];
+        for (const rootMessage of rootMessages) {
+            const firstMessageResult = decompressor.processMessage(rootMessage);
+            assert.equal(firstMessageResult.state, "Skipped");
+            assert.deepStrictEqual(firstMessageResult.message, rootMessage);
+        }
+    });
+});
+//# sourceMappingURL=opDecompressor.spec.js.map

package/lib/test/opLifecycle/opDecompressor.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"opDecompressor.spec.js","sourceRoot":"","sources":["../../../src/test/opLifecycle/opDecompressor.spec.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AAEjC,OAAO,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAEzD,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAU5D,SAAS,8BAA8B,CAAC,MAAc;IACrD,MAAM,KAAK,GAAqC,EAAE,CAAC;IACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE;QAChC,qEAAqE;QACrE,KAAK,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,oBAAoB,CAAC,gBAAgB;YAC3C,QAAQ,EAAE,QAAQ,CAAC,EAA0B;SAC7C,CAAC,CAAC;KACH;IAED,MAAM,gBAAgB,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACzE,MAAM,kBAAkB,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IACtD,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEhF,MAAM,WAAW,GAA8B;QAC9C,QAAQ,EAAE,EAAE,cAAc,EAAE,iBAAiB,EAAE;QAC/C,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;QAC1B,QAAQ,EAAE,UAAU;QACpB,cAAc,EAAE,CAAC;QACjB,qBAAqB,EAAE,CAAC;QACxB,oBAAoB,EAAE,CAAC;QACvB,uBAAuB,EAAE,CAAC;QAC1B,IAAI,EAAE,MAAM;QACZ,SAAS,EAAE,CAAC;QACZ,WAAW,EAAE,KAAK;KAClB,CAAC;IAEF,sDAAsD;IACtD,IAAI,MAAM,KAAK,CAAC,EAAE;QACjB,OAAO,WAAW,CAAC;KACnB;IAED,OAAO;QACN,GAAG,WAAW;QACd,uGAAuG;QACvG,4EAA4E;QAC5E,QAAQ,EAAE,EAAE,GAAI,WAAW,CAAC,QAAgB,EAAE,KAAK,EAAE,IAAI,EAAE;KAC3D,CAAC;AACH,CAAC;AAED,MAAM,YAAY,GAA8B;IAC/C,QAAQ,EAAE,SAAS;IACnB,QAAQ,EAAE,UAAU;IACpB,cAAc,EAAE,CAAC;IACjB,qBAAqB,EAAE,CAAC;IACxB,oBAAoB,EAAE,CAAC;IACvB,uBAAuB,EAAE,CAAC;IAC1B,IAAI,EAAE,MAAM;IACZ,SAAS,EAAE,CAAC;CACZ,CAAC;AAEF,MAAM,oBAAoB,GAA8B;IACvD,QAAQ,EAAE,EAAE;IACZ,QAAQ,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;IAC1B,QAAQ,EAAE,UAAU;IACpB,cAAc,EAAE,CAAC;IACjB,qBAAqB,EAAE,CAAC;IACxB,oBAAoB,EAAE,CAAC;IACvB,uBAAuB,EAAE,CAAC;IAC1B,IAAI,EAAE,MAAM;IACZ,SAAS,EAAE,CAAC;CACZ,CAAC;AAEF,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC/B,MAAM,UAAU,GAAG,IAAI,UAAU,EAAE,CAAC;IACpC,IAAI,YAA4B,CAAC;IACjC,UAAU,CAAC,GAAG,EAAE;QACf,UAAU,CAAC,KAAK,EAAE,CAAC;QACnB,YAAY,GAAG,IAAI,cAAc,CAAC,UAAU,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACzC,MAAM,MAAM,GAAG,YAAY,CAAC,cAAc,CAAC,8BAA8B,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9E,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACxC,MAAM,CAAC,WAAW,CAAE,MAAM,CAAC,OAAO,CAAC,QAAiC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACzF,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,QAAiD,EAAE,UAAU,EAC7E,SAAS,CACT,CAAC;QACF,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,kDAAkD;IAClD,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACrE,MAAM,MAAM,GAAG,YAAY,CAAC,cAAc,CAAC;YAC1C,GAAG,8BAA8B,CAAC,CAAC,CAAC;YACpC,WAAW,EAAE,SAAS;SACtB,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACxC,MAAM,CAAC,WAAW,CAAE,MAAM,CAAC,OAAO,CAAC,QAAiC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACzF,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,QAAiD,EAAE,UAAU,EAC7E,SAAS,CACT,CAAC;QACF,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QAE1D,UAAU,CAAC,WAAW,CAAC;YACtB;gBACC,SAAS,EAAE,kCAAkC;gBAC7C,QAAQ,EAAE,SAAS;aACnB;SACD,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACzC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAClB,YAAY,CAAC,cAAc,CAAC;YAC3B,GAAG,8BAA8B,CAAC,CAAC,CAAC;YACpC,WAAW,EAAE,MAAM;SACnB,CAAC,CACF,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC5C,MAAM,WAAW,GAAG,8BAA8B,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,kBAAkB,GAAG,YAAY,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEpE,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACnD,MAAM,CAAC,WAAW,CAChB,kBAAkB,CAAC,OAAO,CAAC,QAAiC,CAAC,QAAQ,EACtE,QAAQ,CACR,CAAC;QACF,MAAM,CAAC,WAAW,CAChB,kBAAkB,CAAC,OAAO,CAAC,QAAiD;YAC5E,EAAE,UAAU,EACb,SAAS,CACT,CAAC;QACF,MAAM,CAAC,WAAW,CAAC,kBAAkB,CAAC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QAEtE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;YAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;YACzD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;YACvC,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,QAAiC,CAAC,QAAQ,EAC1D,QAAQ,CAAC,EAAE,CACX,CAAC;YACF,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,QAAiD,EAAE,UAAU,EAC7E,SAAS,CACT,CAAC;YACF,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;SAC1D;QAED,MAAM,iBAAiB,GAAG,YAAY,CAAC,cAAc,CAAC,oBAAoB,CAAC,CAAC;QAC5E,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACnD,MAAM,CAAC,WAAW,CAChB,iBAAiB,CAAC,OAAO,CAAC,QAAiC,CAAC,QAAQ,EACrE,QAAQ,CACR,CAAC;QACF,MAAM,CAAC,WAAW,CAChB,iBAAiB,CAAC,OAAO,CAAC,QAAiD;YAC3E,EAAE,UAAU,EACb,SAAS,CACT,CAAC;QACF,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACtE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACzE,MAAM,WAAW,GAAG,8BAA8B,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,kBAAkB,GAAG,YAAY,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEpE,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACnD,MAAM,CAAC,WAAW,CAChB,kBAAkB,CAAC,OAAO,CAAC,QAAiC,CAAC,QAAQ,EACtE,QAAQ,CACR,CAAC;QAEF,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,GAAG,YAAY,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;IACrF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACvD,MAAM,WAAW,GAAG,8BAA8B,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,kBAAkB,GAAG,YAAY,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEpE,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACnD,MAAM,CAAC,WAAW,CAChB,kBAAkB,CAAC,OAAO,CAAC,QAAiC,CAAC,QAAQ,EACtE,QAAQ,CACR,CAAC;QAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;YAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;YACzD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;YACvC,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,QAAiC,CAAC,QAAQ,EAC1D,QAAQ,CAAC,EAAE,CACX,CAAC;SACF;QAED,MAAM,iBAAiB,GAAG,YAAY,CAAC,cAAc,CAAC,oBAAoB,CAAC,CAAC;QAC5E,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACnD,MAAM,CAAC,WAAW,CAChB,iBAAiB,CAAC,OAAO,CAAC,QAAiC,CAAC,QAAQ,EACrE,QAAQ,CACR,CAAC;QAEF,MAAM,eAAe,GAAG,8BAA8B,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,sBAAsB,GAAG,YAAY,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;QAC5E,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACvD,MAAM,CAAC,WAAW,CAChB,sBAAsB,CAAC,OAAO,CAAC,QAAiC,CAAC,QAAQ,EAC1E,QAAQ,CACR,CAAC;QAEF,MAAM,mBAAmB,GAAG,YAAY,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;QACtE,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QACpD,MAAM,CAAC,WAAW,CAChB,mBAAmB,CAAC,OAAO,CAAC,QAAiC,CAAC,QAAQ,EACvE,QAAQ,CACR,CAAC;QAEF,MAAM,0BAA0B,GAAG,YAAY,CAAC,cAAc,CAAC,oBAAoB,CAAC,CAAC;QACrF,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC5D,MAAM,CAAC,WAAW,CAChB,0BAA0B,CAAC,OAAO,CAAC,QAAiC,CAAC,QAAQ,EAC9E,QAAQ,CACR,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QAC1C,MAAM,YAAY,GAAG;YACpB;gBACC,mDAAmD;gBACnD,uDAAuD;gBACvD,QAAQ,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE;gBAC9B,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;gBAC1B,QAAQ,EAAE,UAAU;gBACpB,cAAc,EAAE,CAAC;gBACjB,IAAI,EAAE,CAAC;gBACP,qBAAqB,EAAE,CAAC;gBACxB,oBAAoB,EAAE,CAAC;gBACvB,uBAAuB,EAAE,CAAC;gBAC1B,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,CAAC;aACZ;YACD;gBACC,mDAAmD;gBACnD,QAAQ,EAAE,EAAE,cAAc,EAAE,sCAAsC,EAAE;gBACpE,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;gBAC1B,QAAQ,EAAE,UAAU;gBACpB,cAAc,EAAE,CAAC;gBACjB,IAAI,EAAE,CAAC;gBACP,qBAAqB,EAAE,CAAC;gBACxB,oBAAoB,EAAE,CAAC;gBACvB,uBAAuB,EAAE,CAAC;gBAC1B,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,CAAC;aACZ;YACD;gBACC,mDAAmD;gBACnD,QAAQ,EAAE,EAAE,cAAc,EAAE,sBAAsB,EAAE,IAAI,EAAE,UAAU,EAAE;gBACtE,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;gBAC1B,QAAQ,EAAE,UAAU;gBACpB,cAAc,EAAE,CAAC;gBACjB,IAAI,EAAE,CAAC;gBACP,qBAAqB,EAAE,CAAC;gBACxB,oBAAoB,EAAE,CAAC;gBACvB,uBAAuB,EAAE,CAAC;gBAC1B,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,CAAC;aACZ;YACD;gBACC,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;gBAC1B,QAAQ,EAAE,UAAU;gBACpB,cAAc,EAAE,CAAC;gBACjB,IAAI,EAAE,CAAC;gBACP,qBAAqB,EAAE,CAAC;gBACxB,oBAAoB,EAAE,CAAC;gBACvB,uBAAuB,EAAE,CAAC;gBAC1B,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,CAAC;aACZ;SACD,CAAC;QAEF,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE;YACvC,MAAM,kBAAkB,GAAG,YAAY,CAAC,cAAc,CACrD,WAAwC,CACxC,CAAC;YAEF,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YAClD,MAAM,CAAC,eAAe,CAAC,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;SAChE;IACF,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { strict as assert } from \"assert\";\nimport { compress } from \"lz4js\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { IsoBuffer } from \"@fluid-internal/client-utils\";\nimport type { IEnvelope } from \"@fluidframework/runtime-definitions\";\nimport { MockLogger } from \"@fluidframework/telemetry-utils\";\nimport { ContainerMessageType } from \"../../index.js\";\nimport { OpDecompressor } from \"../../opLifecycle/index.js\";\nimport type { InboundContainerRuntimeMessage } from \"../../messageTypes.js\";\n\n/**\n * Format of test messages generated in this test.\n */\ninterface ITestMessageContents {\n\tcontents: string;\n}\n\nfunction generateCompressedBatchMessage(length: number): ISequencedDocumentMessage {\n\tconst batch: InboundContainerRuntimeMessage[] = [];\n\tfor (let i = 0; i < length; i++) {\n\t\t// Actual Op and contents aren't important. Values are not realistic.\n\t\tbatch.push({\n\t\t\ttype: ContainerMessageType.FluidDataStoreOp,\n\t\t\tcontents: `value${i}` as unknown as IEnvelope,\n\t\t});\n\t}\n\n\tconst contentsAsBuffer = new TextEncoder().encode(JSON.stringify(batch));\n\tconst compressedContents = compress(contentsAsBuffer);\n\tconst compressedContent = IsoBuffer.from(compressedContents).toString(\"base64\");\n\n\tconst messageBase: ISequencedDocumentMessage = {\n\t\tcontents: { packedContents: compressedContent },\n\t\tmetadata: { meta: \"data\" },\n\t\tclientId: \"clientId\",\n\t\tsequenceNumber: 1,\n\t\tminimumSequenceNumber: 1,\n\t\tclientSequenceNumber: 1,\n\t\treferenceSequenceNumber: 1,\n\t\ttype: \"type\",\n\t\ttimestamp: 1,\n\t\tcompression: \"lz4\",\n\t};\n\n\t// Single compressed message won't have batch metadata\n\tif (length === 1) {\n\t\treturn messageBase;\n\t}\n\n\treturn {\n\t\t...messageBase,\n\t\t// TODO: It's not clear if this shallow clone is required, as opposed to just setting \"batch\" to false.\n\t\t// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion\n\t\tmetadata: { ...(messageBase.metadata as any), batch: true },\n\t};\n}\n\nconst emptyMessage: ISequencedDocumentMessage = {\n\tcontents: undefined,\n\tclientId: \"clientId\",\n\tsequenceNumber: 1,\n\tminimumSequenceNumber: 1,\n\tclientSequenceNumber: 1,\n\treferenceSequenceNumber: 1,\n\ttype: \"type\",\n\ttimestamp: 1,\n};\n\nconst endBatchEmptyMessage: ISequencedDocumentMessage = {\n\tcontents: {},\n\tmetadata: { batch: false },\n\tclientId: \"clientId\",\n\tsequenceNumber: 1,\n\tminimumSequenceNumber: 1,\n\tclientSequenceNumber: 1,\n\treferenceSequenceNumber: 1,\n\ttype: \"type\",\n\ttimestamp: 1,\n};\n\ndescribe(\"OpDecompressor\", () => {\n\tconst mockLogger = new MockLogger();\n\tlet decompressor: OpDecompressor;\n\tbeforeEach(() => {\n\t\tmockLogger.clear();\n\t\tdecompressor = new OpDecompressor(mockLogger);\n\t});\n\n\tit(\"Processes single compressed op\", () => {\n\t\tconst result = decompressor.processMessage(generateCompressedBatchMessage(1));\n\t\tassert.equal(result.state, \"Processed\");\n\t\tassert.strictEqual((result.message.contents as ITestMessageContents).contents, \"value0\");\n\t\tassert.strictEqual(\n\t\t\t(result.message.metadata as { compressed?: unknown } | undefined)?.compressed,\n\t\t\tundefined,\n\t\t);\n\t\tassert.strictEqual(result.message.compression, undefined);\n\t});\n\n\t// Back-compat self healing mechanism for ADO:3538\n\tit(\"Processes single compressed op without compression markers\", () => {\n\t\tconst result = decompressor.processMessage({\n\t\t\t...generateCompressedBatchMessage(1),\n\t\t\tcompression: undefined,\n\t\t});\n\t\tassert.equal(result.state, \"Processed\");\n\t\tassert.strictEqual((result.message.contents as ITestMessageContents).contents, \"value0\");\n\t\tassert.strictEqual(\n\t\t\t(result.message.metadata as { compressed?: unknown } | undefined)?.compressed,\n\t\t\tundefined,\n\t\t);\n\t\tassert.strictEqual(result.message.compression, undefined);\n\n\t\tmockLogger.assertMatch([\n\t\t\t{\n\t\t\t\teventName: \"OpDecompressor:LegacyCompression\",\n\t\t\t\tcategory: \"generic\",\n\t\t\t},\n\t\t]);\n\t});\n\n\tit(\"Expecting only lz4 compression\", () => {\n\t\tassert.throws(() =>\n\t\t\tdecompressor.processMessage({\n\t\t\t\t...generateCompressedBatchMessage(5),\n\t\t\t\tcompression: \"gzip\",\n\t\t\t}),\n\t\t);\n\t});\n\n\tit(\"Processes multiple compressed ops\", () => {\n\t\tconst rootMessage = generateCompressedBatchMessage(5);\n\t\tconst firstMessageResult = decompressor.processMessage(rootMessage);\n\n\t\tassert.equal(firstMessageResult.state, \"Accepted\");\n\t\tassert.strictEqual(\n\t\t\t(firstMessageResult.message.contents as ITestMessageContents).contents,\n\t\t\t\"value0\",\n\t\t);\n\t\tassert.strictEqual(\n\t\t\t(firstMessageResult.message.metadata as { compressed?: unknown } | undefined)\n\t\t\t\t?.compressed,\n\t\t\tundefined,\n\t\t);\n\t\tassert.strictEqual(firstMessageResult.message.compression, undefined);\n\n\t\tfor (let i = 1; i < 4; i++) {\n\t\t\tconst result = decompressor.processMessage(emptyMessage);\n\t\t\tassert.equal(result.state, \"Accepted\");\n\t\t\tassert.strictEqual(\n\t\t\t\t(result.message.contents as ITestMessageContents).contents,\n\t\t\t\t`value${i}`,\n\t\t\t);\n\t\t\tassert.strictEqual(\n\t\t\t\t(result.message.metadata as { compressed?: unknown } | undefined)?.compressed,\n\t\t\t\tundefined,\n\t\t\t);\n\t\t\tassert.strictEqual(result.message.compression, undefined);\n\t\t}\n\n\t\tconst lastMessageResult = decompressor.processMessage(endBatchEmptyMessage);\n\t\tassert.equal(lastMessageResult.state, \"Processed\");\n\t\tassert.strictEqual(\n\t\t\t(lastMessageResult.message.contents as ITestMessageContents).contents,\n\t\t\t\"value4\",\n\t\t);\n\t\tassert.strictEqual(\n\t\t\t(lastMessageResult.message.metadata as { compressed?: unknown } | undefined)\n\t\t\t\t?.compressed,\n\t\t\tundefined,\n\t\t);\n\t\tassert.strictEqual(lastMessageResult.message.compression, undefined);\n\t});\n\n\tit(\"Expecting empty messages in the middle of the compressed batch\", () => {\n\t\tconst rootMessage = generateCompressedBatchMessage(5);\n\t\tconst firstMessageResult = decompressor.processMessage(rootMessage);\n\n\t\tassert.equal(firstMessageResult.state, \"Accepted\");\n\t\tassert.strictEqual(\n\t\t\t(firstMessageResult.message.contents as ITestMessageContents).contents,\n\t\t\t\"value0\",\n\t\t);\n\n\t\tassert.throws(() => decompressor.processMessage({ ...emptyMessage, contents: {} }));\n\t});\n\n\tit(\"Processes multiple batches of compressed ops\", () => {\n\t\tconst rootMessage = generateCompressedBatchMessage(5);\n\t\tconst firstMessageResult = decompressor.processMessage(rootMessage);\n\n\t\tassert.equal(firstMessageResult.state, \"Accepted\");\n\t\tassert.strictEqual(\n\t\t\t(firstMessageResult.message.contents as ITestMessageContents).contents,\n\t\t\t\"value0\",\n\t\t);\n\n\t\tfor (let i = 1; i < 4; i++) {\n\t\t\tconst result = decompressor.processMessage(emptyMessage);\n\t\t\tassert.equal(result.state, \"Accepted\");\n\t\t\tassert.strictEqual(\n\t\t\t\t(result.message.contents as ITestMessageContents).contents,\n\t\t\t\t`value${i}`,\n\t\t\t);\n\t\t}\n\n\t\tconst lastMessageResult = decompressor.processMessage(endBatchEmptyMessage);\n\t\tassert.equal(lastMessageResult.state, \"Processed\");\n\t\tassert.strictEqual(\n\t\t\t(lastMessageResult.message.contents as ITestMessageContents).contents,\n\t\t\t\"value4\",\n\t\t);\n\n\t\tconst nextRootMessage = generateCompressedBatchMessage(3);\n\t\tconst nextFirstMessageResult = decompressor.processMessage(nextRootMessage);\n\t\tassert.equal(nextFirstMessageResult.state, \"Accepted\");\n\t\tassert.strictEqual(\n\t\t\t(nextFirstMessageResult.message.contents as ITestMessageContents).contents,\n\t\t\t\"value0\",\n\t\t);\n\n\t\tconst middleMessageResult = decompressor.processMessage(emptyMessage);\n\t\tassert.equal(middleMessageResult.state, \"Accepted\");\n\t\tassert.strictEqual(\n\t\t\t(middleMessageResult.message.contents as ITestMessageContents).contents,\n\t\t\t\"value1\",\n\t\t);\n\n\t\tconst endBatchEmptyMessageResult = decompressor.processMessage(endBatchEmptyMessage);\n\t\tassert.equal(endBatchEmptyMessageResult.state, \"Processed\");\n\t\tassert.strictEqual(\n\t\t\t(endBatchEmptyMessageResult.message.contents as ITestMessageContents).contents,\n\t\t\t\"value2\",\n\t\t);\n\t});\n\n\tit(\"Ignores ops without compression\", () => {\n\t\tconst rootMessages = [\n\t\t\t{\n\t\t\t\t// Back-compat self healing mechanism for ADO:3538,\n\t\t\t\t// the message should have a `packedContents` property.\n\t\t\t\tcontents: { some: \"contents\" },\n\t\t\t\tmetadata: { meta: \"data\" },\n\t\t\t\tclientId: \"clientId\",\n\t\t\t\tsequenceNumber: 1,\n\t\t\t\tterm: 1,\n\t\t\t\tminimumSequenceNumber: 1,\n\t\t\t\tclientSequenceNumber: 1,\n\t\t\t\treferenceSequenceNumber: 1,\n\t\t\t\ttype: \"type\",\n\t\t\t\ttimestamp: 1,\n\t\t\t},\n\t\t\t{\n\t\t\t\t// Back-compat self healing mechanism for ADO:3538,\n\t\t\t\tcontents: { packedContents: \"packedContents is not base64 encoded\" },\n\t\t\t\tmetadata: { meta: \"data\" },\n\t\t\t\tclientId: \"clientId\",\n\t\t\t\tsequenceNumber: 1,\n\t\t\t\tterm: 1,\n\t\t\t\tminimumSequenceNumber: 1,\n\t\t\t\tclientSequenceNumber: 1,\n\t\t\t\treferenceSequenceNumber: 1,\n\t\t\t\ttype: \"type\",\n\t\t\t\ttimestamp: 1,\n\t\t\t},\n\t\t\t{\n\t\t\t\t// Back-compat self healing mechanism for ADO:3538,\n\t\t\t\tcontents: { packedContents: \"YmFzZTY0IGNvbnRlbnQ=\", some: \"contents\" },\n\t\t\t\tmetadata: { meta: \"data\" },\n\t\t\t\tclientId: \"clientId\",\n\t\t\t\tsequenceNumber: 1,\n\t\t\t\tterm: 1,\n\t\t\t\tminimumSequenceNumber: 1,\n\t\t\t\tclientSequenceNumber: 1,\n\t\t\t\treferenceSequenceNumber: 1,\n\t\t\t\ttype: \"type\",\n\t\t\t\ttimestamp: 1,\n\t\t\t},\n\t\t\t{\n\t\t\t\tmetadata: { meta: \"data\" },\n\t\t\t\tclientId: \"clientId\",\n\t\t\t\tsequenceNumber: 1,\n\t\t\t\tterm: 1,\n\t\t\t\tminimumSequenceNumber: 1,\n\t\t\t\tclientSequenceNumber: 1,\n\t\t\t\treferenceSequenceNumber: 1,\n\t\t\t\ttype: \"type\",\n\t\t\t\ttimestamp: 1,\n\t\t\t},\n\t\t];\n\n\t\tfor (const rootMessage of rootMessages) {\n\t\t\tconst firstMessageResult = decompressor.processMessage(\n\t\t\t\trootMessage as ISequencedDocumentMessage,\n\t\t\t);\n\n\t\t\tassert.equal(firstMessageResult.state, \"Skipped\");\n\t\t\tassert.deepStrictEqual(firstMessageResult.message, rootMessage);\n\t\t}\n\t});\n});\n"]}

package/lib/test/opLifecycle/opSplitter.spec.js

@@ -0,0 +1,272 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+import * as crypto from "crypto";
+import { strict as assert } from "assert";
+import { ContainerMessageType } from "@fluidframework/container-runtime-previous";
+import { MockLogger } from "@fluidframework/telemetry-utils";
+import { OpSplitter, splitOp } from "../../opLifecycle/index.js";
+import { CompressionAlgorithms } from "../../containerRuntime.js";
+describe("OpSplitter", () => {
+    const batchesSubmitted = [];
+    const mockSubmitBatchFn = (batch, referenceSequenceNumber) => {
+        batchesSubmitted.push({ messages: batch, referenceSequenceNumber });
+        return batchesSubmitted.length;
+    };
+    beforeEach(() => {
+        batchesSubmitted.splice(0);
+        mockLogger.clear();
+    });
+    const chunkSizeInBytes = 50 * 1024;
+    const maxBatchSizeInBytes = 10 * 50 * 1024;
+    const mockLogger = new MockLogger();
+    it("Reconstruct original chunked op", () => {
+        const op1 = generateChunkableOp(chunkSizeInBytes * 2);
+        const op2 = generateChunkableOp(chunkSizeInBytes * 3);
+        const chunks1 = wrapChunkedOps(splitOp(op1, chunkSizeInBytes), "testClient1");
+        const chunks2 = wrapChunkedOps(splitOp(op2, chunkSizeInBytes), "testClient2");
+        const opSplitter = new OpSplitter([], mockSubmitBatchFn, 0, maxBatchSizeInBytes, mockLogger);
+        assert.equal(opSplitter.processRemoteMessage(chunks1[0]).state, "Accepted");
+        assert.equal(opSplitter.processRemoteMessage(chunks2[0]).state, "Accepted");
+        assert.equal(opSplitter.processRemoteMessage(chunks1[1]).state, "Accepted");
+        assert.equal(opSplitter.processRemoteMessage(chunks2[1]).state, "Accepted");
+        const chunks1LastResult = opSplitter.processRemoteMessage(chunks1[2]);
+        // The last chunk will reconstruct the original message
+        assert.equal(chunks1LastResult.state, "Processed");
+        assertSameMessage(chunks1LastResult.message, op1);
+        assert.equal(opSplitter.chunks.size, 1);
+        assert.equal(opSplitter.processRemoteMessage(chunks2[2]).state, "Accepted");
+        const chunks2LastResult = opSplitter.processRemoteMessage(chunks2[3]);
+        // The last chunk will reconstruct the original message
+        assert.equal(chunks2LastResult.state, "Processed");
+        assertSameMessage(chunks2LastResult.message, op2);
+        assert.equal(opSplitter.chunks.size, 0);
+    });
+    it("Reconstruct original chunked op with extra empty op", () => {
+        const op1 = generateChunkableOp(chunkSizeInBytes * 2);
+        const op2 = generateChunkableOp(chunkSizeInBytes * 3);
+        const chunks1 = wrapChunkedOps(splitOp(op1, chunkSizeInBytes, true), "testClient1");
+        const chunks2 = wrapChunkedOps(splitOp(op2, chunkSizeInBytes, true), "testClient2");
+        const opSplitter = new OpSplitter([], mockSubmitBatchFn, 0, maxBatchSizeInBytes, mockLogger);
+        assert.equal(opSplitter.processRemoteMessage(chunks1[0]).state, "Accepted");
+        assert.equal(opSplitter.processRemoteMessage(chunks2[0]).state, "Accepted");
+        assert.equal(opSplitter.processRemoteMessage(chunks1[1]).state, "Accepted");
+        assert.equal(opSplitter.processRemoteMessage(chunks2[1]).state, "Accepted");
+        assert.equal(opSplitter.processRemoteMessage(chunks1[2]).state, "Accepted");
+        const chunks1LastResult = opSplitter.processRemoteMessage(chunks1[3]);
+        // The last chunk will reconstruct the original message
+        assert.equal(chunks1LastResult.state, "Processed");
+        assertSameMessage(chunks1LastResult.message, op1);
+        assert.equal(opSplitter.chunks.size, 1);
+        assert.equal(opSplitter.processRemoteMessage(chunks2[2]).state, "Accepted");
+        assert.equal(opSplitter.processRemoteMessage(chunks2[3]).state, "Accepted");
+        const chunks2LastResult = opSplitter.processRemoteMessage(chunks2[4]);
+        // The last chunk will reconstruct the original message
+        assert.equal(chunks2LastResult.state, "Processed");
+        assertSameMessage(chunks2LastResult.message, op2);
+        assert.equal(opSplitter.chunks.size, 0);
+    });
+    it("Reconstruct original chunked op with initial chunks", () => {
+        const op = generateChunkableOp(chunkSizeInBytes * 3);
+        const chunks = wrapChunkedOps(splitOp(op, chunkSizeInBytes), "testClient");
+        const opSplitter = new OpSplitter([], mockSubmitBatchFn, 0, maxBatchSizeInBytes, mockLogger);
+        opSplitter.processRemoteMessage(chunks[0]);
+        opSplitter.processRemoteMessage(chunks[1]);
+        const otherOpSplitter = new OpSplitter(Array.from(opSplitter.chunks), mockSubmitBatchFn, 0, maxBatchSizeInBytes, mockLogger);
+        opSplitter.clearPartialChunks("testClient");
+        otherOpSplitter.processRemoteMessage(chunks[2]);
+        assertSameMessage(otherOpSplitter.processRemoteMessage(chunks[3]).message, op);
+    });
+    it("Clear chunks", () => {
+        const chunks = wrapChunkedOps(splitOp(generateChunkableOp(chunkSizeInBytes * 3), chunkSizeInBytes), "testClient");
+        const opSplitter = new OpSplitter([], mockSubmitBatchFn, 0, maxBatchSizeInBytes, mockLogger);
+        opSplitter.processRemoteMessage(chunks[0]);
+        assert.equal(opSplitter.chunks.size, 1);
+        opSplitter.clearPartialChunks("noClient");
+        assert.equal(opSplitter.chunks.size, 1);
+        opSplitter.clearPartialChunks("testClient");
+        assert.equal(opSplitter.chunks.size, 0);
+    });
+    it("Throw when processing out-of-order chunks", () => {
+        const chunks = wrapChunkedOps(splitOp(generateChunkableOp(chunkSizeInBytes * 3), chunkSizeInBytes), "testClient1");
+        const opSplitter = new OpSplitter([], mockSubmitBatchFn, 0, maxBatchSizeInBytes, mockLogger);
+        assert.throws(() => opSplitter.processRemoteMessage(chunks[2]));
+    });
+    it("Don't accept non-chunked ops", () => {
+        const chunks = wrapChunkedOps(splitOp(generateChunkableOp(chunkSizeInBytes * 3), chunkSizeInBytes), "testClient1").map((op) => ({ ...op, type: ContainerMessageType.FluidDataStoreOp }));
+        const opSplitter = new OpSplitter([], mockSubmitBatchFn, 0, maxBatchSizeInBytes, mockLogger);
+        for (const op of chunks) {
+            assert.deepStrictEqual(opSplitter.processRemoteMessage(op), {
+                message: op,
+                state: "Skipped",
+            });
+        }
+    });
+    it("Chunk metadata", () => {
+        const originalOp = generateChunkableOp(chunkSizeInBytes * 4);
+        const chunks = splitOp(originalOp, chunkSizeInBytes);
+        assert.equal(chunks
+            .slice(0, -1)
+            .every((chunk) => chunk.originalCompression === undefined &&
+            chunk.originalMetadata === undefined), true);
+        const lastChunk = chunks[chunks.length - 1];
+        assert.deepStrictEqual(lastChunk.originalMetadata, originalOp.metadata);
+        assert.equal(lastChunk.originalCompression, originalOp.compression);
+    });
+    it("Batch split invariants", () => {
+        const opSplitter = new OpSplitter([], mockSubmitBatchFn, 50, maxBatchSizeInBytes, mockLogger);
+        const regularMessage = generateChunkableOp(20);
+        const compressedMessage = { ...regularMessage, metadata: { compressed: true } };
+        // Empty batch
+        assert.throws(() => opSplitter.splitFirstBatchMessage({
+            content: [compressedMessage],
+            contentSizeInBytes: 0,
+            referenceSequenceNumber: 0,
+        }));
+        // Empty batch
+        assert.throws(() => opSplitter.splitFirstBatchMessage({
+            content: [],
+            contentSizeInBytes: 1,
+            referenceSequenceNumber: 0,
+        }));
+        // Batch is too small to be chunked
+        assert.throws(() => opSplitter.splitFirstBatchMessage({
+            content: [compressedMessage],
+            contentSizeInBytes: 1,
+            referenceSequenceNumber: 0,
+        }));
+        // Batch is not compressed
+        assert.throws(() => opSplitter.splitFirstBatchMessage({
+            content: [regularMessage],
+            contentSizeInBytes: 3,
+            referenceSequenceNumber: 0,
+        }));
+        // Misconfigured op splitter
+        assert.throws(() => new OpSplitter([], mockSubmitBatchFn, 0, maxBatchSizeInBytes, mockLogger).splitFirstBatchMessage({
+            content: [compressedMessage],
+            contentSizeInBytes: 3,
+            referenceSequenceNumber: 0,
+        }));
+        // Old loader
+        assert.throws(() => new OpSplitter([], undefined, 0, maxBatchSizeInBytes, mockLogger).splitFirstBatchMessage({
+            content: [compressedMessage],
+            contentSizeInBytes: 3,
+            referenceSequenceNumber: 0,
+        }));
+        // Misconfigured op splitter
+        assert.throws(() => new OpSplitter([], mockSubmitBatchFn, 2, 1, mockLogger).splitFirstBatchMessage({
+            content: [compressedMessage],
+            contentSizeInBytes: 3,
+            referenceSequenceNumber: 0,
+        }));
+        // Not enabled
+        assert.throws(() => new OpSplitter([], mockSubmitBatchFn, Number.POSITIVE_INFINITY, maxBatchSizeInBytes, mockLogger).splitFirstBatchMessage({
+            content: [compressedMessage],
+            contentSizeInBytes: 3,
+            referenceSequenceNumber: 0,
+        }));
+    });
+    describe("Compressed batches", () => {
+        [false, true].forEach((extraOp) => {
+            it(`Split compressed batch with multiple messages with${extraOp ? "" : "out"} extra empty op.`, () => {
+                const chunkSize = 20;
+                const opSplitter = new OpSplitter([], mockSubmitBatchFn, chunkSize, extraOp ? chunkSize * 2 : maxBatchSizeInBytes, mockLogger);
+                const largeMessage = generateChunkableOp(100);
+                const emptyMessage = generateChunkableOp(0);
+                const result = opSplitter.splitFirstBatchMessage({
+                    content: [largeMessage, emptyMessage, emptyMessage, emptyMessage],
+                    contentSizeInBytes: largeMessage.contents?.length ?? 0,
+                    referenceSequenceNumber: 0,
+                });
+                assert.equal(batchesSubmitted.length, 5 + (extraOp ? 1 : 0));
+                for (const batch of batchesSubmitted) {
+                    assert.equal(batch.messages.length, 1);
+                    assert.equal(batch.messages[0].type, ContainerMessageType.ChunkedOp);
+                    assert.equal(batch.referenceSequenceNumber, 0);
+                }
+                assert.equal(result.content.length, 4);
+                const lastChunk = JSON.parse(result.content[0].contents).contents;
+                assert.equal(lastChunk.chunkId, lastChunk.totalChunks);
+                assert.deepStrictEqual(result.content.slice(1), new Array(3).fill(emptyMessage));
+                assert.equal(!extraOp ||
+                    JSON.parse(result.content[0].contents).contents?.contents?.length === 0, true);
+                assert.notEqual(result.contentSizeInBytes, largeMessage.contents?.length ?? 0);
+                const contentSentSeparately = batchesSubmitted.map((x) => JSON.parse(x.messages[0].contents)
+                    .contents.contents);
+                const sentContent = [...contentSentSeparately, lastChunk.contents].reduce((accumulator, current) => `${accumulator}${current}`);
+                assert.equal(sentContent, largeMessage.contents);
+                assert(mockLogger.matchEvents([
+                    {
+                        eventName: "OpSplitter:CompressedChunkedBatch",
+                        length: result.content.length,
+                        chunks: 100 / 20 + 1 + (extraOp ? 1 : 0),
+                        chunkSizeInBytes: 20,
+                    },
+                ]));
+            });
+            it(`Split compressed batch with single message with${extraOp ? "" : "out"} extra empty op.`, () => {
+                const chunkSize = 20;
+                const opSplitter = new OpSplitter([], mockSubmitBatchFn, 20, extraOp ? chunkSize * 2 : maxBatchSizeInBytes, mockLogger);
+                const largeMessage = generateChunkableOp(100);
+                const result = opSplitter.splitFirstBatchMessage({
+                    content: [largeMessage],
+                    contentSizeInBytes: largeMessage.contents?.length ?? 0,
+                    referenceSequenceNumber: 0,
+                });
+                assert.equal(batchesSubmitted.length, 5 + (extraOp ? 1 : 0));
+                for (const batch of batchesSubmitted) {
+                    assert.equal(batch.messages.length, 1);
+                    assert.equal(batch.messages[0].type, ContainerMessageType.ChunkedOp);
+                    assert.equal(batch.referenceSequenceNumber, 0);
+                }
+                assert.equal(result.content.length, 1);
+                assert.notEqual(result.contentSizeInBytes, largeMessage.contents?.length ?? 0);
+                const lastChunk = JSON.parse(result.content[0].contents).contents;
+                assert.equal(lastChunk.chunkId, lastChunk.totalChunks);
+                assert.equal(!extraOp ||
+                    JSON.parse(result.content[0].contents).contents?.contents?.length === 0, true);
+                assert.notEqual(result.contentSizeInBytes, largeMessage.contents?.length ?? 0);
+                const contentSentSeparately = batchesSubmitted.map((x) => JSON.parse(x.messages[0].contents)
+                    .contents.contents);
+                const sentContent = [...contentSentSeparately, lastChunk.contents].reduce((accumulator, current) => `${accumulator}${current}`);
+                assert.equal(sentContent, largeMessage.contents);
+                assert(mockLogger.matchEvents([
+                    {
+                        eventName: "OpSplitter:CompressedChunkedBatch",
+                        length: result.content.length,
+                        chunks: 100 / 20 + 1 + (extraOp ? 1 : 0),
+                        chunkSizeInBytes: 20,
+                    },
+                ]));
+            });
+        });
+    });
+    const assertSameMessage = (result, original) => {
+        assert.deepStrictEqual(result.contents, JSON.parse(original.contents));
+        assert.strictEqual(result.type, original.type);
+        assert.strictEqual(result.metadata, original.metadata);
+        assert.strictEqual(result.compression, original.compression);
+    };
+    const generateChunkableOp = (contentSizeInBytes) => {
+        const contents = { value: crypto.randomBytes(contentSizeInBytes / 2).toString("hex") };
+        return {
+            localOpMetadata: undefined,
+            type: ContainerMessageType.FluidDataStoreOp,
+            referenceSequenceNumber: Infinity,
+            metadata: { meta: "data" },
+            compression: CompressionAlgorithms.lz4,
+            contents: JSON.stringify(contents),
+        };
+    };
+    const wrapChunkedOps = (ops, clientId) => ops.map((op) => {
+        const result = {
+            contents: op,
+            clientId,
+            type: ContainerMessageType.ChunkedOp,
+        };
+        return result;
+    });
+});
+//# sourceMappingURL=opSplitter.spec.js.map

package/lib/test/opLifecycle/opSplitter.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"opSplitter.spec.js","sourceRoot":"","sources":["../../../src/test/opLifecycle/opSplitter.spec.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,6DAA6D;AAE7D,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,4CAA4C,CAAC;AAGlF,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAC7D,OAAO,EAA4B,UAAU,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAC3F,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAElE,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC3B,MAAM,gBAAgB,GAAsE,EAAE,CAAC;IAE/F,MAAM,iBAAiB,GAAG,CACzB,KAAsB,EACtB,uBAAgC,EACvB,EAAE;QACX,gBAAgB,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;QACpE,OAAO,gBAAgB,CAAC,MAAM,CAAC;IAChC,CAAC,CAAC;IAEF,UAAU,CAAC,GAAG,EAAE;QACf,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,MAAM,gBAAgB,GAAG,EAAE,GAAG,IAAI,CAAC;IACnC,MAAM,mBAAmB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC3C,MAAM,UAAU,GAAG,IAAI,UAAU,EAAE,CAAC;IAEpC,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QAC1C,MAAM,GAAG,GAAG,mBAAmB,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;QACtD,MAAM,GAAG,GAAG,mBAAmB,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,aAAa,CAAC,CAAC;QAC9E,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,aAAa,CAAC,CAAC;QAC9E,MAAM,UAAU,GAAG,IAAI,UAAU,CAChC,EAAE,EACF,iBAAiB,EACjB,CAAC,EACD,mBAAmB,EACnB,UAAU,CACV,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC5E,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAE5E,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC5E,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAE5E,MAAM,iBAAiB,GAAG,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,uDAAuD;QACvD,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACnD,iBAAiB,CAAC,iBAAiB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAClD,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAExC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAE5E,MAAM,iBAAiB,GAAG,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,uDAAuD;QACvD,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACnD,iBAAiB,CAAC,iBAAiB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAElD,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC9D,MAAM,GAAG,GAAG,mBAAmB,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;QACtD,MAAM,GAAG,GAAG,mBAAmB,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,aAAa,CAAC,CAAC;QACpF,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,aAAa,CAAC,CAAC;QACpF,MAAM,UAAU,GAAG,IAAI,UAAU,CAChC,EAAE,EACF,iBAAiB,EACjB,CAAC,EACD,mBAAmB,EACnB,UAAU,CACV,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC5E,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAE5E,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC5E,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAE5E,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAE5E,MAAM,iBAAiB,GAAG,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,uDAAuD;QACvD,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACnD,iBAAiB,CAAC,iBAAiB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAClD,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAExC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC5E,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAE5E,MAAM,iBAAiB,GAAG,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,uDAAuD;QACvD,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACnD,iBAAiB,CAAC,iBAAiB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAElD,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC9D,MAAM,EAAE,GAAG,mBAAmB,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,EAAE,EAAE,gBAAgB,CAAC,EAAE,YAAY,CAAC,CAAC;QAC3E,MAAM,UAAU,GAAG,IAAI,UAAU,CAChC,EAAE,EACF,iBAAiB,EACjB,CAAC,EACD,mBAAmB,EACnB,UAAU,CACV,CAAC;QACF,UAAU,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3C,UAAU,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,eAAe,GAAG,IAAI,UAAU,CACrC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAC7B,iBAAiB,EACjB,CAAC,EACD,mBAAmB,EACnB,UAAU,CACV,CAAC;QACF,UAAU,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QAE5C,eAAe,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,iBAAiB,CAAC,eAAe,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;QACvB,MAAM,MAAM,GAAG,cAAc,CAC5B,OAAO,CAAC,mBAAmB,CAAC,gBAAgB,GAAG,CAAC,CAAC,EAAE,gBAAgB,CAAC,EACpE,YAAY,CACZ,CAAC;QACF,MAAM,UAAU,GAAG,IAAI,UAAU,CAChC,EAAE,EACF,iBAAiB,EACjB,CAAC,EACD,mBAAmB,EACnB,UAAU,CACV,CAAC;QACF,UAAU,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxC,UAAU,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAC1C,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxC,UAAU,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QAC5C,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACpD,MAAM,MAAM,GAAG,cAAc,CAC5B,OAAO,CAAC,mBAAmB,CAAC,gBAAgB,GAAG,CAAC,CAAC,EAAE,gBAAgB,CAAC,EACpE,aAAa,CACb,CAAC;QACF,MAAM,UAAU,GAAG,IAAI,UAAU,CAChC,EAAE,EACF,iBAAiB,EACjB,CAAC,EACD,mBAAmB,EACnB,UAAU,CACV,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACvC,MAAM,MAAM,GAAG,cAAc,CAC5B,OAAO,CAAC,mBAAmB,CAAC,gBAAgB,GAAG,CAAC,CAAC,EAAE,gBAAgB,CAAC,EACpE,aAAa,CACb,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,oBAAoB,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,IAAI,UAAU,CAChC,EAAE,EACF,iBAAiB,EACjB,CAAC,EACD,mBAAmB,EACnB,UAAU,CACV,CAAC;QACF,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE;YACxB,MAAM,CAAC,eAAe,CAAC,UAAU,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAAE;gBAC3D,OAAO,EAAE,EAAE;gBACX,KAAK,EAAE,SAAS;aAChB,CAAC,CAAC;SACH;IACF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gBAAgB,EAAE,GAAG,EAAE;QACzB,MAAM,UAAU,GAAG,mBAAmB,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC;QACrD,MAAM,CAAC,KAAK,CACX,MAAM;aACJ,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;aACZ,KAAK,CACL,CAAC,KAAK,EAAE,EAAE,CACT,KAAK,CAAC,mBAAmB,KAAK,SAAS;YACvC,KAAK,CAAC,gBAAgB,KAAK,SAAS,CACrC,EACF,IAAI,CACJ,CAAC;QAEF,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC5C,MAAM,CAAC,eAAe,CAAC,SAAS,CAAC,gBAAgB,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;QACxE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,mBAAmB,EAAE,UAAU,CAAC,WAAW,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACjC,MAAM,UAAU,GAAG,IAAI,UAAU,CAChC,EAAE,EACF,iBAAiB,EACjB,EAAE,EACF,mBAAmB,EACnB,UAAU,CACV,CAAC;QACF,MAAM,cAAc,GAAG,mBAAmB,CAAC,EAAE,CAAC,CAAC;QAC/C,MAAM,iBAAiB,GAAG,EAAE,GAAG,cAAc,EAAE,QAAQ,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC;QAEhF,cAAc;QACd,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAClB,UAAU,CAAC,sBAAsB,CAAC;YACjC,OAAO,EAAE,CAAC,iBAAiB,CAAC;YAC5B,kBAAkB,EAAE,CAAC;YACrB,uBAAuB,EAAE,CAAC;SAC1B,CAAC,CACF,CAAC;QAEF,cAAc;QACd,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAClB,UAAU,CAAC,sBAAsB,CAAC;YACjC,OAAO,EAAE,EAAE;YACX,kBAAkB,EAAE,CAAC;YACrB,uBAAuB,EAAE,CAAC;SAC1B,CAAC,CACF,CAAC;QAEF,mCAAmC;QACnC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAClB,UAAU,CAAC,sBAAsB,CAAC;YACjC,OAAO,EAAE,CAAC,iBAAiB,CAAC;YAC5B,kBAAkB,EAAE,CAAC;YACrB,uBAAuB,EAAE,CAAC;SAC1B,CAAC,CACF,CAAC;QAEF,0BAA0B;QAC1B,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAClB,UAAU,CAAC,sBAAsB,CAAC;YACjC,OAAO,EAAE,CAAC,cAAc,CAAC;YACzB,kBAAkB,EAAE,CAAC;YACrB,uBAAuB,EAAE,CAAC;SAC1B,CAAC,CACF,CAAC;QAEF,4BAA4B;QAC5B,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAClB,IAAI,UAAU,CACb,EAAE,EACF,iBAAiB,EACjB,CAAC,EACD,mBAAmB,EACnB,UAAU,CACV,CAAC,sBAAsB,CAAC;YACxB,OAAO,EAAE,CAAC,iBAAiB,CAAC;YAC5B,kBAAkB,EAAE,CAAC;YACrB,uBAAuB,EAAE,CAAC;SAC1B,CAAC,CACF,CAAC;QAEF,aAAa;QACb,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAClB,IAAI,UAAU,CACb,EAAE,EACF,SAAS,EACT,CAAC,EACD,mBAAmB,EACnB,UAAU,CACV,CAAC,sBAAsB,CAAC;YACxB,OAAO,EAAE,CAAC,iBAAiB,CAAC;YAC5B,kBAAkB,EAAE,CAAC;YACrB,uBAAuB,EAAE,CAAC;SAC1B,CAAC,CACF,CAAC;QAEF,4BAA4B;QAC5B,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAClB,IAAI,UAAU,CAAC,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC,sBAAsB,CAAC;YAC9E,OAAO,EAAE,CAAC,iBAAiB,CAAC;YAC5B,kBAAkB,EAAE,CAAC;YACrB,uBAAuB,EAAE,CAAC;SAC1B,CAAC,CACF,CAAC;QAEF,cAAc;QACd,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAClB,IAAI,UAAU,CACb,EAAE,EACF,iBAAiB,EACjB,MAAM,CAAC,iBAAiB,EACxB,mBAAmB,EACnB,UAAU,CACV,CAAC,sBAAsB,CAAC;YACxB,OAAO,EAAE,CAAC,iBAAiB,CAAC;YAC5B,kBAAkB,EAAE,CAAC;YACrB,uBAAuB,EAAE,CAAC;SAC1B,CAAC,CACF,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QACnC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YACjC,EAAE,CAAC,qDACF,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAChB,kBAAkB,EAAE,GAAG,EAAE;gBACxB,MAAM,SAAS,GAAG,EAAE,CAAC;gBACrB,MAAM,UAAU,GAAG,IAAI,UAAU,CAChC,EAAE,EACF,iBAAiB,EACjB,SAAS,EACT,OAAO,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,mBAAmB,EAC7C,UAAU,CACV,CAAC;gBACF,MAAM,YAAY,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;gBAE5C,MAAM,MAAM,GAAG,UAAU,CAAC,sBAAsB,CAAC;oBAChD,OAAO,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,CAAC;oBACjE,kBAAkB,EAAE,YAAY,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC;oBACtD,uBAAuB,EAAE,CAAC;iBAC1B,CAAC,CAAC;gBAEH,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC7D,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE;oBACrC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBACvC,MAAM,CAAC,KAAK,CACV,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAkB,CAAC,IAAI,EACxC,oBAAoB,CAAC,SAAS,CAC9B,CAAC;oBACF,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC;iBAC/C;gBAED,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBACvC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAS,CAAC,CAAC,QAAsB,CAAC;gBACjF,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;gBACvD,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;gBACjF,MAAM,CAAC,KAAK,CACX,CAAC,OAAO;oBACP,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAS,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,KAAK,CAAC,EACzE,IAAI,CACJ,CAAC;gBACF,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,EAAE,YAAY,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;gBAC/E,MAAM,qBAAqB,GAAG,gBAAgB,CAAC,GAAG,CACjD,CAAC,CAAC,EAAE,EAAE,CAEJ,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAkB,CAAC,QAAS,CAAC;qBACnD,QACF,CAAC,QAAQ,CACX,CAAC;gBACF,MAAM,WAAW,GAAG,CAAC,GAAG,qBAAqB,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC,MAAM,CACxE,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,WAAW,GAAG,OAAO,EAAE,CACpD,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;gBAEjD,MAAM,CACL,UAAU,CAAC,WAAW,CAAC;oBACtB;wBACC,SAAS,EAAE,mCAAmC;wBAC9C,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;wBAC7B,MAAM,EAAE,GAAG,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;wBACxC,gBAAgB,EAAE,EAAE;qBACpB;iBACD,CAAC,CACF,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,kDACF,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAChB,kBAAkB,EAAE,GAAG,EAAE;gBACxB,MAAM,SAAS,GAAG,EAAE,CAAC;gBACrB,MAAM,UAAU,GAAG,IAAI,UAAU,CAChC,EAAE,EACF,iBAAiB,EACjB,EAAE,EACF,OAAO,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,mBAAmB,EAC7C,UAAU,CACV,CAAC;gBACF,MAAM,YAAY,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;gBAE9C,MAAM,MAAM,GAAG,UAAU,CAAC,sBAAsB,CAAC;oBAChD,OAAO,EAAE,CAAC,YAAY,CAAC;oBACvB,kBAAkB,EAAE,YAAY,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC;oBACtD,uBAAuB,EAAE,CAAC;iBAC1B,CAAC,CAAC;gBAEH,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC7D,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE;oBACrC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBACvC,MAAM,CAAC,KAAK,CACV,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAkB,CAAC,IAAI,EACxC,oBAAoB,CAAC,SAAS,CAC9B,CAAC;oBACF,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC;iBAC/C;gBAED,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBACvC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,EAAE,YAAY,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;gBAC/E,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAS,CAAC,CAAC,QAAsB,CAAC;gBACjF,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;gBACvD,MAAM,CAAC,KAAK,CACX,CAAC,OAAO;oBACP,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAS,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,KAAK,CAAC,EACzE,IAAI,CACJ,CAAC;gBACF,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,EAAE,YAAY,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;gBAC/E,MAAM,qBAAqB,GAAG,gBAAgB,CAAC,GAAG,CACjD,CAAC,CAAC,EAAE,EAAE,CAEJ,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAkB,CAAC,QAAS,CAAC;qBACnD,QACF,CAAC,QAAQ,CACX,CAAC;gBACF,MAAM,WAAW,GAAG,CAAC,GAAG,qBAAqB,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC,MAAM,CACxE,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,WAAW,GAAG,OAAO,EAAE,CACpD,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;gBAEjD,MAAM,CACL,UAAU,CAAC,WAAW,CAAC;oBACtB;wBACC,SAAS,EAAE,mCAAmC;wBAC9C,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;wBAC7B,MAAM,EAAE,GAAG,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;wBACxC,gBAAgB,EAAE,EAAE;qBACpB;iBACD,CAAC,CACF,CAAC;YACH,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,iBAAiB,GAAG,CAAC,MAAiC,EAAE,QAAsB,EAAE,EAAE;QACvF,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAS,CAAC,CAAC,CAAC;QACxE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACvD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC9D,CAAC,CAAC;IAEF,MAAM,mBAAmB,GAAG,CAAC,kBAA0B,EAAgB,EAAE;QACxE,MAAM,QAAQ,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,kBAAkB,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACvF,OAAO;YACN,eAAe,EAAE,SAAS;YAC1B,IAAI,EAAE,oBAAoB,CAAC,gBAAgB;YAC3C,uBAAuB,EAAE,QAAQ;YACjC,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;YAC1B,WAAW,EAAE,qBAAqB,CAAC,GAAG;YACtC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;SAClC,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,cAAc,GAAG,CAAC,GAAiB,EAAE,QAAgB,EAA+B,EAAE,CAC3F,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACd,MAAM,MAAM,GAAG;YACd,QAAQ,EAAE,EAAE;YACZ,QAAQ;YACR,IAAI,EAAE,oBAAoB,CAAC,SAAS;SACpC,CAAC;QAEF,OAAO,MAAmC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n/* eslint-disable @typescript-eslint/no-non-null-assertion */\n\nimport * as crypto from \"crypto\";\nimport { strict as assert } from \"assert\";\nimport { ContainerMessageType } from \"@fluidframework/container-runtime-previous\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport { MockLogger } from \"@fluidframework/telemetry-utils\";\nimport { BatchMessage, IChunkedOp, OpSplitter, splitOp } from \"../../opLifecycle/index.js\";\nimport { CompressionAlgorithms } from \"../../containerRuntime.js\";\n\ndescribe(\"OpSplitter\", () => {\n\tconst batchesSubmitted: { messages: IBatchMessage[]; referenceSequenceNumber?: number }[] = [];\n\n\tconst mockSubmitBatchFn = (\n\t\tbatch: IBatchMessage[],\n\t\treferenceSequenceNumber?: number,\n\t): number => {\n\t\tbatchesSubmitted.push({ messages: batch, referenceSequenceNumber });\n\t\treturn batchesSubmitted.length;\n\t};\n\n\tbeforeEach(() => {\n\t\tbatchesSubmitted.splice(0);\n\t\tmockLogger.clear();\n\t});\n\n\tconst chunkSizeInBytes = 50 * 1024;\n\tconst maxBatchSizeInBytes = 10 * 50 * 1024;\n\tconst mockLogger = new MockLogger();\n\n\tit(\"Reconstruct original chunked op\", () => {\n\t\tconst op1 = generateChunkableOp(chunkSizeInBytes * 2);\n\t\tconst op2 = generateChunkableOp(chunkSizeInBytes * 3);\n\t\tconst chunks1 = wrapChunkedOps(splitOp(op1, chunkSizeInBytes), \"testClient1\");\n\t\tconst chunks2 = wrapChunkedOps(splitOp(op2, chunkSizeInBytes), \"testClient2\");\n\t\tconst opSplitter = new OpSplitter(\n\t\t\t[],\n\t\t\tmockSubmitBatchFn,\n\t\t\t0,\n\t\t\tmaxBatchSizeInBytes,\n\t\t\tmockLogger,\n\t\t);\n\n\t\tassert.equal(opSplitter.processRemoteMessage(chunks1[0]).state, \"Accepted\");\n\t\tassert.equal(opSplitter.processRemoteMessage(chunks2[0]).state, \"Accepted\");\n\n\t\tassert.equal(opSplitter.processRemoteMessage(chunks1[1]).state, \"Accepted\");\n\t\tassert.equal(opSplitter.processRemoteMessage(chunks2[1]).state, \"Accepted\");\n\n\t\tconst chunks1LastResult = opSplitter.processRemoteMessage(chunks1[2]);\n\t\t// The last chunk will reconstruct the original message\n\t\tassert.equal(chunks1LastResult.state, \"Processed\");\n\t\tassertSameMessage(chunks1LastResult.message, op1);\n\t\tassert.equal(opSplitter.chunks.size, 1);\n\n\t\tassert.equal(opSplitter.processRemoteMessage(chunks2[2]).state, \"Accepted\");\n\n\t\tconst chunks2LastResult = opSplitter.processRemoteMessage(chunks2[3]);\n\t\t// The last chunk will reconstruct the original message\n\t\tassert.equal(chunks2LastResult.state, \"Processed\");\n\t\tassertSameMessage(chunks2LastResult.message, op2);\n\n\t\tassert.equal(opSplitter.chunks.size, 0);\n\t});\n\n\tit(\"Reconstruct original chunked op with extra empty op\", () => {\n\t\tconst op1 = generateChunkableOp(chunkSizeInBytes * 2);\n\t\tconst op2 = generateChunkableOp(chunkSizeInBytes * 3);\n\t\tconst chunks1 = wrapChunkedOps(splitOp(op1, chunkSizeInBytes, true), \"testClient1\");\n\t\tconst chunks2 = wrapChunkedOps(splitOp(op2, chunkSizeInBytes, true), \"testClient2\");\n\t\tconst opSplitter = new OpSplitter(\n\t\t\t[],\n\t\t\tmockSubmitBatchFn,\n\t\t\t0,\n\t\t\tmaxBatchSizeInBytes,\n\t\t\tmockLogger,\n\t\t);\n\n\t\tassert.equal(opSplitter.processRemoteMessage(chunks1[0]).state, \"Accepted\");\n\t\tassert.equal(opSplitter.processRemoteMessage(chunks2[0]).state, \"Accepted\");\n\n\t\tassert.equal(opSplitter.processRemoteMessage(chunks1[1]).state, \"Accepted\");\n\t\tassert.equal(opSplitter.processRemoteMessage(chunks2[1]).state, \"Accepted\");\n\n\t\tassert.equal(opSplitter.processRemoteMessage(chunks1[2]).state, \"Accepted\");\n\n\t\tconst chunks1LastResult = opSplitter.processRemoteMessage(chunks1[3]);\n\t\t// The last chunk will reconstruct the original message\n\t\tassert.equal(chunks1LastResult.state, \"Processed\");\n\t\tassertSameMessage(chunks1LastResult.message, op1);\n\t\tassert.equal(opSplitter.chunks.size, 1);\n\n\t\tassert.equal(opSplitter.processRemoteMessage(chunks2[2]).state, \"Accepted\");\n\t\tassert.equal(opSplitter.processRemoteMessage(chunks2[3]).state, \"Accepted\");\n\n\t\tconst chunks2LastResult = opSplitter.processRemoteMessage(chunks2[4]);\n\t\t// The last chunk will reconstruct the original message\n\t\tassert.equal(chunks2LastResult.state, \"Processed\");\n\t\tassertSameMessage(chunks2LastResult.message, op2);\n\n\t\tassert.equal(opSplitter.chunks.size, 0);\n\t});\n\n\tit(\"Reconstruct original chunked op with initial chunks\", () => {\n\t\tconst op = generateChunkableOp(chunkSizeInBytes * 3);\n\t\tconst chunks = wrapChunkedOps(splitOp(op, chunkSizeInBytes), \"testClient\");\n\t\tconst opSplitter = new OpSplitter(\n\t\t\t[],\n\t\t\tmockSubmitBatchFn,\n\t\t\t0,\n\t\t\tmaxBatchSizeInBytes,\n\t\t\tmockLogger,\n\t\t);\n\t\topSplitter.processRemoteMessage(chunks[0]);\n\t\topSplitter.processRemoteMessage(chunks[1]);\n\n\t\tconst otherOpSplitter = new OpSplitter(\n\t\t\tArray.from(opSplitter.chunks),\n\t\t\tmockSubmitBatchFn,\n\t\t\t0,\n\t\t\tmaxBatchSizeInBytes,\n\t\t\tmockLogger,\n\t\t);\n\t\topSplitter.clearPartialChunks(\"testClient\");\n\n\t\totherOpSplitter.processRemoteMessage(chunks[2]);\n\t\tassertSameMessage(otherOpSplitter.processRemoteMessage(chunks[3]).message, op);\n\t});\n\n\tit(\"Clear chunks\", () => {\n\t\tconst chunks = wrapChunkedOps(\n\t\t\tsplitOp(generateChunkableOp(chunkSizeInBytes * 3), chunkSizeInBytes),\n\t\t\t\"testClient\",\n\t\t);\n\t\tconst opSplitter = new OpSplitter(\n\t\t\t[],\n\t\t\tmockSubmitBatchFn,\n\t\t\t0,\n\t\t\tmaxBatchSizeInBytes,\n\t\t\tmockLogger,\n\t\t);\n\t\topSplitter.processRemoteMessage(chunks[0]);\n\n\t\tassert.equal(opSplitter.chunks.size, 1);\n\t\topSplitter.clearPartialChunks(\"noClient\");\n\t\tassert.equal(opSplitter.chunks.size, 1);\n\t\topSplitter.clearPartialChunks(\"testClient\");\n\t\tassert.equal(opSplitter.chunks.size, 0);\n\t});\n\n\tit(\"Throw when processing out-of-order chunks\", () => {\n\t\tconst chunks = wrapChunkedOps(\n\t\t\tsplitOp(generateChunkableOp(chunkSizeInBytes * 3), chunkSizeInBytes),\n\t\t\t\"testClient1\",\n\t\t);\n\t\tconst opSplitter = new OpSplitter(\n\t\t\t[],\n\t\t\tmockSubmitBatchFn,\n\t\t\t0,\n\t\t\tmaxBatchSizeInBytes,\n\t\t\tmockLogger,\n\t\t);\n\t\tassert.throws(() => opSplitter.processRemoteMessage(chunks[2]));\n\t});\n\n\tit(\"Don't accept non-chunked ops\", () => {\n\t\tconst chunks = wrapChunkedOps(\n\t\t\tsplitOp(generateChunkableOp(chunkSizeInBytes * 3), chunkSizeInBytes),\n\t\t\t\"testClient1\",\n\t\t).map((op) => ({ ...op, type: ContainerMessageType.FluidDataStoreOp }));\n\t\tconst opSplitter = new OpSplitter(\n\t\t\t[],\n\t\t\tmockSubmitBatchFn,\n\t\t\t0,\n\t\t\tmaxBatchSizeInBytes,\n\t\t\tmockLogger,\n\t\t);\n\t\tfor (const op of chunks) {\n\t\t\tassert.deepStrictEqual(opSplitter.processRemoteMessage(op), {\n\t\t\t\tmessage: op,\n\t\t\t\tstate: \"Skipped\",\n\t\t\t});\n\t\t}\n\t});\n\n\tit(\"Chunk metadata\", () => {\n\t\tconst originalOp = generateChunkableOp(chunkSizeInBytes * 4);\n\t\tconst chunks = splitOp(originalOp, chunkSizeInBytes);\n\t\tassert.equal(\n\t\t\tchunks\n\t\t\t\t.slice(0, -1)\n\t\t\t\t.every(\n\t\t\t\t\t(chunk) =>\n\t\t\t\t\t\tchunk.originalCompression === undefined &&\n\t\t\t\t\t\tchunk.originalMetadata === undefined,\n\t\t\t\t),\n\t\t\ttrue,\n\t\t);\n\n\t\tconst lastChunk = chunks[chunks.length - 1];\n\t\tassert.deepStrictEqual(lastChunk.originalMetadata, originalOp.metadata);\n\t\tassert.equal(lastChunk.originalCompression, originalOp.compression);\n\t});\n\n\tit(\"Batch split invariants\", () => {\n\t\tconst opSplitter = new OpSplitter(\n\t\t\t[],\n\t\t\tmockSubmitBatchFn,\n\t\t\t50,\n\t\t\tmaxBatchSizeInBytes,\n\t\t\tmockLogger,\n\t\t);\n\t\tconst regularMessage = generateChunkableOp(20);\n\t\tconst compressedMessage = { ...regularMessage, metadata: { compressed: true } };\n\n\t\t// Empty batch\n\t\tassert.throws(() =>\n\t\t\topSplitter.splitFirstBatchMessage({\n\t\t\t\tcontent: [compressedMessage],\n\t\t\t\tcontentSizeInBytes: 0,\n\t\t\t\treferenceSequenceNumber: 0,\n\t\t\t}),\n\t\t);\n\n\t\t// Empty batch\n\t\tassert.throws(() =>\n\t\t\topSplitter.splitFirstBatchMessage({\n\t\t\t\tcontent: [],\n\t\t\t\tcontentSizeInBytes: 1,\n\t\t\t\treferenceSequenceNumber: 0,\n\t\t\t}),\n\t\t);\n\n\t\t// Batch is too small to be chunked\n\t\tassert.throws(() =>\n\t\t\topSplitter.splitFirstBatchMessage({\n\t\t\t\tcontent: [compressedMessage],\n\t\t\t\tcontentSizeInBytes: 1,\n\t\t\t\treferenceSequenceNumber: 0,\n\t\t\t}),\n\t\t);\n\n\t\t// Batch is not compressed\n\t\tassert.throws(() =>\n\t\t\topSplitter.splitFirstBatchMessage({\n\t\t\t\tcontent: [regularMessage],\n\t\t\t\tcontentSizeInBytes: 3,\n\t\t\t\treferenceSequenceNumber: 0,\n\t\t\t}),\n\t\t);\n\n\t\t// Misconfigured op splitter\n\t\tassert.throws(() =>\n\t\t\tnew OpSplitter(\n\t\t\t\t[],\n\t\t\t\tmockSubmitBatchFn,\n\t\t\t\t0,\n\t\t\t\tmaxBatchSizeInBytes,\n\t\t\t\tmockLogger,\n\t\t\t).splitFirstBatchMessage({\n\t\t\t\tcontent: [compressedMessage],\n\t\t\t\tcontentSizeInBytes: 3,\n\t\t\t\treferenceSequenceNumber: 0,\n\t\t\t}),\n\t\t);\n\n\t\t// Old loader\n\t\tassert.throws(() =>\n\t\t\tnew OpSplitter(\n\t\t\t\t[],\n\t\t\t\tundefined,\n\t\t\t\t0,\n\t\t\t\tmaxBatchSizeInBytes,\n\t\t\t\tmockLogger,\n\t\t\t).splitFirstBatchMessage({\n\t\t\t\tcontent: [compressedMessage],\n\t\t\t\tcontentSizeInBytes: 3,\n\t\t\t\treferenceSequenceNumber: 0,\n\t\t\t}),\n\t\t);\n\n\t\t// Misconfigured op splitter\n\t\tassert.throws(() =>\n\t\t\tnew OpSplitter([], mockSubmitBatchFn, 2, 1, mockLogger).splitFirstBatchMessage({\n\t\t\t\tcontent: [compressedMessage],\n\t\t\t\tcontentSizeInBytes: 3,\n\t\t\t\treferenceSequenceNumber: 0,\n\t\t\t}),\n\t\t);\n\n\t\t// Not enabled\n\t\tassert.throws(() =>\n\t\t\tnew OpSplitter(\n\t\t\t\t[],\n\t\t\t\tmockSubmitBatchFn,\n\t\t\t\tNumber.POSITIVE_INFINITY,\n\t\t\t\tmaxBatchSizeInBytes,\n\t\t\t\tmockLogger,\n\t\t\t).splitFirstBatchMessage({\n\t\t\t\tcontent: [compressedMessage],\n\t\t\t\tcontentSizeInBytes: 3,\n\t\t\t\treferenceSequenceNumber: 0,\n\t\t\t}),\n\t\t);\n\t});\n\n\tdescribe(\"Compressed batches\", () => {\n\t\t[false, true].forEach((extraOp) => {\n\t\t\tit(`Split compressed batch with multiple messages with${\n\t\t\t\textraOp ? \"\" : \"out\"\n\t\t\t} extra empty op.`, () => {\n\t\t\t\tconst chunkSize = 20;\n\t\t\t\tconst opSplitter = new OpSplitter(\n\t\t\t\t\t[],\n\t\t\t\t\tmockSubmitBatchFn,\n\t\t\t\t\tchunkSize,\n\t\t\t\t\textraOp ? chunkSize * 2 : maxBatchSizeInBytes,\n\t\t\t\t\tmockLogger,\n\t\t\t\t);\n\t\t\t\tconst largeMessage = generateChunkableOp(100);\n\t\t\t\tconst emptyMessage = generateChunkableOp(0);\n\n\t\t\t\tconst result = opSplitter.splitFirstBatchMessage({\n\t\t\t\t\tcontent: [largeMessage, emptyMessage, emptyMessage, emptyMessage],\n\t\t\t\t\tcontentSizeInBytes: largeMessage.contents?.length ?? 0,\n\t\t\t\t\treferenceSequenceNumber: 0,\n\t\t\t\t});\n\n\t\t\t\tassert.equal(batchesSubmitted.length, 5 + (extraOp ? 1 : 0));\n\t\t\t\tfor (const batch of batchesSubmitted) {\n\t\t\t\t\tassert.equal(batch.messages.length, 1);\n\t\t\t\t\tassert.equal(\n\t\t\t\t\t\t(batch.messages[0] as BatchMessage).type,\n\t\t\t\t\t\tContainerMessageType.ChunkedOp,\n\t\t\t\t\t);\n\t\t\t\t\tassert.equal(batch.referenceSequenceNumber, 0);\n\t\t\t\t}\n\n\t\t\t\tassert.equal(result.content.length, 4);\n\t\t\t\tconst lastChunk = JSON.parse(result.content[0].contents!).contents as IChunkedOp;\n\t\t\t\tassert.equal(lastChunk.chunkId, lastChunk.totalChunks);\n\t\t\t\tassert.deepStrictEqual(result.content.slice(1), new Array(3).fill(emptyMessage));\n\t\t\t\tassert.equal(\n\t\t\t\t\t!extraOp ||\n\t\t\t\t\t\tJSON.parse(result.content[0].contents!).contents?.contents?.length === 0,\n\t\t\t\t\ttrue,\n\t\t\t\t);\n\t\t\t\tassert.notEqual(result.contentSizeInBytes, largeMessage.contents?.length ?? 0);\n\t\t\t\tconst contentSentSeparately = batchesSubmitted.map(\n\t\t\t\t\t(x) =>\n\t\t\t\t\t\t(\n\t\t\t\t\t\t\tJSON.parse((x.messages[0] as BatchMessage).contents!)\n\t\t\t\t\t\t\t\t.contents as IChunkedOp\n\t\t\t\t\t\t).contents,\n\t\t\t\t);\n\t\t\t\tconst sentContent = [...contentSentSeparately, lastChunk.contents].reduce(\n\t\t\t\t\t(accumulator, current) => `${accumulator}${current}`,\n\t\t\t\t);\n\t\t\t\tassert.equal(sentContent, largeMessage.contents);\n\n\t\t\t\tassert(\n\t\t\t\t\tmockLogger.matchEvents([\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\teventName: \"OpSplitter:CompressedChunkedBatch\",\n\t\t\t\t\t\t\tlength: result.content.length,\n\t\t\t\t\t\t\tchunks: 100 / 20 + 1 + (extraOp ? 1 : 0),\n\t\t\t\t\t\t\tchunkSizeInBytes: 20,\n\t\t\t\t\t\t},\n\t\t\t\t\t]),\n\t\t\t\t);\n\t\t\t});\n\n\t\t\tit(`Split compressed batch with single message with${\n\t\t\t\textraOp ? \"\" : \"out\"\n\t\t\t} extra empty op.`, () => {\n\t\t\t\tconst chunkSize = 20;\n\t\t\t\tconst opSplitter = new OpSplitter(\n\t\t\t\t\t[],\n\t\t\t\t\tmockSubmitBatchFn,\n\t\t\t\t\t20,\n\t\t\t\t\textraOp ? chunkSize * 2 : maxBatchSizeInBytes,\n\t\t\t\t\tmockLogger,\n\t\t\t\t);\n\t\t\t\tconst largeMessage = generateChunkableOp(100);\n\n\t\t\t\tconst result = opSplitter.splitFirstBatchMessage({\n\t\t\t\t\tcontent: [largeMessage],\n\t\t\t\t\tcontentSizeInBytes: largeMessage.contents?.length ?? 0,\n\t\t\t\t\treferenceSequenceNumber: 0,\n\t\t\t\t});\n\n\t\t\t\tassert.equal(batchesSubmitted.length, 5 + (extraOp ? 1 : 0));\n\t\t\t\tfor (const batch of batchesSubmitted) {\n\t\t\t\t\tassert.equal(batch.messages.length, 1);\n\t\t\t\t\tassert.equal(\n\t\t\t\t\t\t(batch.messages[0] as BatchMessage).type,\n\t\t\t\t\t\tContainerMessageType.ChunkedOp,\n\t\t\t\t\t);\n\t\t\t\t\tassert.equal(batch.referenceSequenceNumber, 0);\n\t\t\t\t}\n\n\t\t\t\tassert.equal(result.content.length, 1);\n\t\t\t\tassert.notEqual(result.contentSizeInBytes, largeMessage.contents?.length ?? 0);\n\t\t\t\tconst lastChunk = JSON.parse(result.content[0].contents!).contents as IChunkedOp;\n\t\t\t\tassert.equal(lastChunk.chunkId, lastChunk.totalChunks);\n\t\t\t\tassert.equal(\n\t\t\t\t\t!extraOp ||\n\t\t\t\t\t\tJSON.parse(result.content[0].contents!).contents?.contents?.length === 0,\n\t\t\t\t\ttrue,\n\t\t\t\t);\n\t\t\t\tassert.notEqual(result.contentSizeInBytes, largeMessage.contents?.length ?? 0);\n\t\t\t\tconst contentSentSeparately = batchesSubmitted.map(\n\t\t\t\t\t(x) =>\n\t\t\t\t\t\t(\n\t\t\t\t\t\t\tJSON.parse((x.messages[0] as BatchMessage).contents!)\n\t\t\t\t\t\t\t\t.contents as IChunkedOp\n\t\t\t\t\t\t).contents,\n\t\t\t\t);\n\t\t\t\tconst sentContent = [...contentSentSeparately, lastChunk.contents].reduce(\n\t\t\t\t\t(accumulator, current) => `${accumulator}${current}`,\n\t\t\t\t);\n\t\t\t\tassert.equal(sentContent, largeMessage.contents);\n\n\t\t\t\tassert(\n\t\t\t\t\tmockLogger.matchEvents([\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\teventName: \"OpSplitter:CompressedChunkedBatch\",\n\t\t\t\t\t\t\tlength: result.content.length,\n\t\t\t\t\t\t\tchunks: 100 / 20 + 1 + (extraOp ? 1 : 0),\n\t\t\t\t\t\t\tchunkSizeInBytes: 20,\n\t\t\t\t\t\t},\n\t\t\t\t\t]),\n\t\t\t\t);\n\t\t\t});\n\t\t});\n\t});\n\n\tconst assertSameMessage = (result: ISequencedDocumentMessage, original: BatchMessage) => {\n\t\tassert.deepStrictEqual(result.contents, JSON.parse(original.contents!));\n\t\tassert.strictEqual(result.type, original.type);\n\t\tassert.strictEqual(result.metadata, original.metadata);\n\t\tassert.strictEqual(result.compression, original.compression);\n\t};\n\n\tconst generateChunkableOp = (contentSizeInBytes: number): BatchMessage => {\n\t\tconst contents = { value: crypto.randomBytes(contentSizeInBytes / 2).toString(\"hex\") };\n\t\treturn {\n\t\t\tlocalOpMetadata: undefined,\n\t\t\ttype: ContainerMessageType.FluidDataStoreOp,\n\t\t\treferenceSequenceNumber: Infinity,\n\t\t\tmetadata: { meta: \"data\" },\n\t\t\tcompression: CompressionAlgorithms.lz4,\n\t\t\tcontents: JSON.stringify(contents),\n\t\t};\n\t};\n\n\tconst wrapChunkedOps = (ops: IChunkedOp[], clientId: string): ISequencedDocumentMessage[] =>\n\t\tops.map((op) => {\n\t\t\tconst result = {\n\t\t\t\tcontents: op,\n\t\t\t\tclientId,\n\t\t\t\ttype: ContainerMessageType.ChunkedOp,\n\t\t\t};\n\n\t\t\treturn result as ISequencedDocumentMessage;\n\t\t});\n});\n"]}