@fluidframework/container-runtime 2.0.0-dev.2.2.0.111723 → 2.0.0-dev.3.1.0.125672

package/dist/opLifecycle/index.d.ts

@@ -0,0 +1,12 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+export { BatchManager } from "./batchManager";
+export { BatchMessage, IBatch, IBatchCheckpoint, IChunkedOp, IMessageProcessingResult, } from "./definitions";
+export { Outbox } from "./outbox";
+export { OpCompressor } from "./opCompressor";
+export { OpDecompressor } from "./opDecompressor";
+export { OpSplitter, splitOp } from "./opSplitter";
+export { RemoteMessageProcessor, unpackRuntimeMessage } from "./remoteMessageProcessor";
+//# sourceMappingURL=index.d.ts.map

package/dist/opLifecycle/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EACN,YAAY,EACZ,MAAM,EACN,gBAAgB,EAChB,UAAU,EACV,wBAAwB,GACxB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC"}

package/dist/opLifecycle/index.js

@@ -0,0 +1,22 @@
+"use strict";
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.unpackRuntimeMessage = exports.RemoteMessageProcessor = exports.splitOp = exports.OpSplitter = exports.OpDecompressor = exports.OpCompressor = exports.Outbox = exports.BatchManager = void 0;
+var batchManager_1 = require("./batchManager");
+Object.defineProperty(exports, "BatchManager", { enumerable: true, get: function () { return batchManager_1.BatchManager; } });
+var outbox_1 = require("./outbox");
+Object.defineProperty(exports, "Outbox", { enumerable: true, get: function () { return outbox_1.Outbox; } });
+var opCompressor_1 = require("./opCompressor");
+Object.defineProperty(exports, "OpCompressor", { enumerable: true, get: function () { return opCompressor_1.OpCompressor; } });
+var opDecompressor_1 = require("./opDecompressor");
+Object.defineProperty(exports, "OpDecompressor", { enumerable: true, get: function () { return opDecompressor_1.OpDecompressor; } });
+var opSplitter_1 = require("./opSplitter");
+Object.defineProperty(exports, "OpSplitter", { enumerable: true, get: function () { return opSplitter_1.OpSplitter; } });
+Object.defineProperty(exports, "splitOp", { enumerable: true, get: function () { return opSplitter_1.splitOp; } });
+var remoteMessageProcessor_1 = require("./remoteMessageProcessor");
+Object.defineProperty(exports, "RemoteMessageProcessor", { enumerable: true, get: function () { return remoteMessageProcessor_1.RemoteMessageProcessor; } });
+Object.defineProperty(exports, "unpackRuntimeMessage", { enumerable: true, get: function () { return remoteMessageProcessor_1.unpackRuntimeMessage; } });
+//# sourceMappingURL=index.js.map

package/dist/opLifecycle/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/opLifecycle/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,+CAA8C;AAArC,4GAAA,YAAY,OAAA;AAQrB,mCAAkC;AAAzB,gGAAA,MAAM,OAAA;AACf,+CAA8C;AAArC,4GAAA,YAAY,OAAA;AACrB,mDAAkD;AAAzC,gHAAA,cAAc,OAAA;AACvB,2CAAmD;AAA1C,wGAAA,UAAU,OAAA;AAAE,qGAAA,OAAO,OAAA;AAC5B,mEAAwF;AAA/E,gIAAA,sBAAsB,OAAA;AAAE,8HAAA,oBAAoB,OAAA","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nexport { BatchManager } from \"./batchManager\";\nexport {\n\tBatchMessage,\n\tIBatch,\n\tIBatchCheckpoint,\n\tIChunkedOp,\n\tIMessageProcessingResult,\n} from \"./definitions\";\nexport { Outbox } from \"./outbox\";\nexport { OpCompressor } from \"./opCompressor\";\nexport { OpDecompressor } from \"./opDecompressor\";\nexport { OpSplitter, splitOp } from \"./opSplitter\";\nexport { RemoteMessageProcessor, unpackRuntimeMessage } from \"./remoteMessageProcessor\";\n"]}

package/dist/{opCompressor.d.ts → opLifecycle/opCompressor.d.ts}

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { BatchMessage } from "./batchManager";
+import { IBatch } from "./definitions";
 /**
  * Compresses batches of ops. It generates a single compressed op that contains
  * the contents of each op in the batch. It then submits empty ops for each original
@@ -11,8 +11,8 @@ import { BatchMessage } from "./batchManager";
  */
 export declare class OpCompressor {
     private readonly logger;
-    private compressedBatchCount;
     constructor(logger: ITelemetryLogger);
-    compressBatch(batch: BatchMessage[], originalLength: number): BatchMessage[];
+    compressBatch(batch: IBatch): IBatch;
+    private serializeBatch;
 }
 //# sourceMappingURL=opCompressor.d.ts.map

package/dist/opLifecycle/opCompressor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"opCompressor.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAMtE,OAAO,EAAE,MAAM,EAAgB,MAAM,eAAe,CAAC;AAErD;;;;GAIG;AACH,qBAAa,YAAY;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAEZ,MAAM,EAAE,gBAAgB;IAI7B,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAkC3C,OAAO,CAAC,cAAc;CAsBtB"}

package/dist/opLifecycle/opCompressor.js

@@ -0,0 +1,67 @@
+"use strict";
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpCompressor = void 0;
+const common_utils_1 = require("@fluidframework/common-utils");
+const container_utils_1 = require("@fluidframework/container-utils");
+const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
+const lz4js_1 = require("lz4js");
+const containerRuntime_1 = require("../containerRuntime");
+/**
+ * Compresses batches of ops. It generates a single compressed op that contains
+ * the contents of each op in the batch. It then submits empty ops for each original
+ * op to reserve sequence numbers.
+ */
+class OpCompressor {
+    constructor(logger) {
+        this.logger = telemetry_utils_1.ChildLogger.create(logger, "OpCompressor");
+    }
+    compressBatch(batch) {
+        const compressionStart = Date.now();
+        const contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));
+        const compressedContents = (0, lz4js_1.compress)(contentsAsBuffer);
+        const compressedContent = common_utils_1.IsoBuffer.from(compressedContents).toString("base64");
+        const duration = Date.now() - compressionStart;
+        if (batch.contentSizeInBytes > 200000) {
+            this.logger.sendPerformanceEvent({
+                eventName: "CompressedBatch",
+                duration,
+                sizeBeforeCompression: batch.contentSizeInBytes,
+                sizeAfterCompression: compressedContent.length,
+            });
+        }
+        const messages = [];
+        messages.push(Object.assign(Object.assign({}, batch.content[0]), { contents: JSON.stringify({ packedContents: compressedContent }), metadata: batch.content[0].metadata, compression: containerRuntime_1.CompressionAlgorithms.lz4 }));
+        for (const message of batch.content.slice(1)) {
+            messages.push(Object.assign(Object.assign({}, message), { contents: undefined }));
+        }
+        return {
+            contentSizeInBytes: compressedContent.length,
+            content: messages,
+        };
+    }
+    serializeBatch(batch) {
+        try {
+            return JSON.stringify(batch.content.map((message) => message.deserializedContent));
+        }
+        catch (e) {
+            if (e.message === "Invalid string length") {
+                // This is how JSON.stringify signals that
+                // the content size exceeds its capacity
+                const error = new container_utils_1.UsageError("Payload too large");
+                this.logger.sendErrorEvent({
+                    eventName: "BatchTooLarge",
+                    size: batch.contentSizeInBytes,
+                    length: batch.content.length,
+                }, error);
+                throw error;
+            }
+            throw e;
+        }
+    }
+}
+exports.OpCompressor = OpCompressor;
+//# sourceMappingURL=opCompressor.js.map

package/dist/opLifecycle/opCompressor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"opCompressor.js","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAGH,+DAAyD;AACzD,qEAA6D;AAC7D,qEAA8D;AAC9D,iCAAiC;AACjC,0DAA4D;AAG5D;;;;GAIG;AACH,MAAa,YAAY;IAGxB,YAAY,MAAwB;QACnC,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC1D,CAAC;IAEM,aAAa,CAAC,KAAa;QACjC,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9E,MAAM,kBAAkB,GAAG,IAAA,gBAAQ,EAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,iBAAiB,GAAG,wBAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;QAE/C,IAAI,KAAK,CAAC,kBAAkB,GAAG,MAAM,EAAE;YACtC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAChC,SAAS,EAAE,iBAAiB;gBAC5B,QAAQ;gBACR,qBAAqB,EAAE,KAAK,CAAC,kBAAkB;gBAC/C,oBAAoB,EAAE,iBAAiB,CAAC,MAAM;aAC9C,CAAC,CAAC;SACH;QAED,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,QAAQ,CAAC,IAAI,iCACT,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KACnB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC,EAC/D,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,EACnC,WAAW,EAAE,wCAAqB,CAAC,GAAG,IACrC,CAAC;QAEH,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC7C,QAAQ,CAAC,IAAI,iCAAM,OAAO,KAAE,QAAQ,EAAE,SAAS,IAAG,CAAC;SACnD;QAED,OAAO;YACN,kBAAkB,EAAE,iBAAiB,CAAC,MAAM;YAC5C,OAAO,EAAE,QAAQ;SACjB,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,KAAa;QACnC,IAAI;YACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC;SACnF;QAAC,OAAO,CAAM,EAAE;YAChB,IAAI,CAAC,CAAC,OAAO,KAAK,uBAAuB,EAAE;gBAC1C,0CAA0C;gBAC1C,wCAAwC;gBACxC,MAAM,KAAK,GAAG,IAAI,4BAAU,CAAC,mBAAmB,CAAC,CAAC;gBAClD,IAAI,CAAC,MAAM,CAAC,cAAc,CACzB;oBACC,SAAS,EAAE,eAAe;oBAC1B,IAAI,EAAE,KAAK,CAAC,kBAAkB;oBAC9B,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;iBAC5B,EACD,KAAK,CACL,CAAC;gBACF,MAAM,KAAK,CAAC;aACZ;YAED,MAAM,CAAC,CAAC;SACR;IACF,CAAC;CACD;AA/DD,oCA+DC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { IsoBuffer } from \"@fluidframework/common-utils\";\nimport { UsageError } from \"@fluidframework/container-utils\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { compress } from \"lz4js\";\nimport { CompressionAlgorithms } from \"../containerRuntime\";\nimport { IBatch, BatchMessage } from \"./definitions\";\n\n/**\n * Compresses batches of ops. It generates a single compressed op that contains\n * the contents of each op in the batch. It then submits empty ops for each original\n * op to reserve sequence numbers.\n */\nexport class OpCompressor {\n\tprivate readonly logger;\n\n\tconstructor(logger: ITelemetryLogger) {\n\t\tthis.logger = ChildLogger.create(logger, \"OpCompressor\");\n\t}\n\n\tpublic compressBatch(batch: IBatch): IBatch {\n\t\tconst compressionStart = Date.now();\n\t\tconst contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));\n\t\tconst compressedContents = compress(contentsAsBuffer);\n\t\tconst compressedContent = IsoBuffer.from(compressedContents).toString(\"base64\");\n\t\tconst duration = Date.now() - compressionStart;\n\n\t\tif (batch.contentSizeInBytes > 200000) {\n\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\teventName: \"CompressedBatch\",\n\t\t\t\tduration,\n\t\t\t\tsizeBeforeCompression: batch.contentSizeInBytes,\n\t\t\t\tsizeAfterCompression: compressedContent.length,\n\t\t\t});\n\t\t}\n\n\t\tconst messages: BatchMessage[] = [];\n\t\tmessages.push({\n\t\t\t...batch.content[0],\n\t\t\tcontents: JSON.stringify({ packedContents: compressedContent }),\n\t\t\tmetadata: batch.content[0].metadata,\n\t\t\tcompression: CompressionAlgorithms.lz4,\n\t\t});\n\n\t\tfor (const message of batch.content.slice(1)) {\n\t\t\tmessages.push({ ...message, contents: undefined });\n\t\t}\n\n\t\treturn {\n\t\t\tcontentSizeInBytes: compressedContent.length,\n\t\t\tcontent: messages,\n\t\t};\n\t}\n\n\tprivate serializeBatch(batch: IBatch): string {\n\t\ttry {\n\t\t\treturn JSON.stringify(batch.content.map((message) => message.deserializedContent));\n\t\t} catch (e: any) {\n\t\t\tif (e.message === \"Invalid string length\") {\n\t\t\t\t// This is how JSON.stringify signals that\n\t\t\t\t// the content size exceeds its capacity\n\t\t\t\tconst error = new UsageError(\"Payload too large\");\n\t\t\t\tthis.logger.sendErrorEvent(\n\t\t\t\t\t{\n\t\t\t\t\t\teventName: \"BatchTooLarge\",\n\t\t\t\t\t\tsize: batch.contentSizeInBytes,\n\t\t\t\t\t\tlength: batch.content.length,\n\t\t\t\t\t},\n\t\t\t\t\terror,\n\t\t\t\t);\n\t\t\t\tthrow error;\n\t\t\t}\n\n\t\t\tthrow e;\n\t\t}\n\t}\n}\n"]}

package/dist/{opDecompressor.d.ts → opLifecycle/opDecompressor.d.ts}

@@ -3,6 +3,7 @@
  * Licensed under the MIT License.
  */
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { IMessageProcessingResult } from "./definitions";
 /**
  * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
  * This class relies on some implicit contracts defined below:
@@ -15,6 +16,6 @@ export declare class OpDecompressor {
     private activeBatch;
     private rootMessageContents;
     private processedCount;
-    processMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage;
+    processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult;
 }
 //# sourceMappingURL=opDecompressor.d.ts.map

package/dist/opLifecycle/opDecompressor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"opDecompressor.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opDecompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAGjF,OAAO,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AAEzD;;;;;;;GAOG;AACH,qBAAa,cAAc;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,mBAAmB,CAAkB;IAC7C,OAAO,CAAC,cAAc,CAAK;IAEpB,cAAc,CAAC,OAAO,EAAE,yBAAyB,GAAG,wBAAwB;CAwFnF"}

package/dist/{opDecompressor.js → opLifecycle/opDecompressor.js}

@@ -7,7 +7,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.OpDecompressor = void 0;
 const lz4js_1 = require("lz4js");
 const common_utils_1 = require("@fluidframework/common-utils");
-const _1 = require(".");
+const containerRuntime_1 = require("../containerRuntime");
 /**
  * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
  * This class relies on some implicit contracts defined below:
@@ -22,18 +22,14 @@ class OpDecompressor {
         this.processedCount = 0;
     }
     processMessage(message) {
-        var _a, _b, _c, _d, _e, _f;
-        // We're checking for compression = true or top level compression property so
-        // that we can enable compression without waiting on all ordering services
-        // to pick up protocol change. Eventually only the top level property should
-        // be used.
-        if (((_a = message.metadata) === null || _a === void 0 ? void 0 : _a.batch) === true
-            && (((_b = message.metadata) === null || _b === void 0 ? void 0 : _b.compressed) || message.compression !== undefined)) {
+        var _a, _b, _c, _d;
+        (0, common_utils_1.assert)(message.compression === undefined || message.compression === containerRuntime_1.CompressionAlgorithms.lz4, 0x511 /* Only lz4 compression is supported */);
+        if (((_a = message.metadata) === null || _a === void 0 ? void 0 : _a.batch) === true && message.compression === containerRuntime_1.CompressionAlgorithms.lz4) {
             // Beginning of a compressed batch
-            (0, common_utils_1.assert)(this.activeBatch === false, "shouldn't have multiple active batches");
+            (0, common_utils_1.assert)(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
             if (message.compression) {
                 // lz4 is the only supported compression algorithm for now
-                (0, common_utils_1.assert)(message.compression === _1.CompressionAlgorithms.lz4, "lz4 is currently the only supported compression algorithm");
+                (0, common_utils_1.assert)(message.compression === containerRuntime_1.CompressionAlgorithms.lz4, 0x4b9 /* lz4 is currently the only supported compression algorithm */);
             }
             this.activeBatch = true;
             const contents = common_utils_1.IsoBuffer.from(message.contents.packedContents, "base64");
@@ -41,32 +37,52 @@ class OpDecompressor {
             const intoString = (0, common_utils_1.Uint8ArrayToString)(decompressedMessage);
             const asObj = JSON.parse(intoString);
             this.rootMessageContents = asObj;
-            return Object.assign(Object.assign({}, message), { contents: this.rootMessageContents[this.processedCount++] });
+            return {
+                message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+                state: "Accepted",
+            };
         }
-        if (this.rootMessageContents !== undefined && ((_c = message.metadata) === null || _c === void 0 ? void 0 : _c.batch) === undefined && this.activeBatch) {
+        if (this.rootMessageContents !== undefined &&
+            ((_b = message.metadata) === null || _b === void 0 ? void 0 : _b.batch) === undefined &&
+            this.activeBatch) {
+            (0, common_utils_1.assert)(message.contents === undefined, 0x512 /* Expecting empty message */);
             // Continuation of compressed batch
-            return Object.assign(Object.assign({}, message), { contents: this.rootMessageContents[this.processedCount++] });
+            return {
+                message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+                state: "Accepted",
+            };
         }
-        if (this.rootMessageContents !== undefined && ((_d = message.metadata) === null || _d === void 0 ? void 0 : _d.batch) === false) {
+        if (this.rootMessageContents !== undefined && ((_c = message.metadata) === null || _c === void 0 ? void 0 : _c.batch) === false) {
             // End of compressed batch
-            const returnMessage = Object.assign(Object.assign({}, message), { contents: this.rootMessageContents[this.processedCount++] });
+            const returnMessage = newMessage(message, this.rootMessageContents[this.processedCount++]);
             this.activeBatch = false;
             this.rootMessageContents = undefined;
             this.processedCount = 0;
-            return returnMessage;
+            return {
+                message: returnMessage,
+                state: "Processed",
+            };
         }
-        if (((_e = message.metadata) === null || _e === void 0 ? void 0 : _e.batch) === undefined &&
-            (((_f = message.metadata) === null || _f === void 0 ? void 0 : _f.compressed) || message.compression === _1.CompressionAlgorithms.lz4)) {
+        if (((_d = message.metadata) === null || _d === void 0 ? void 0 : _d.batch) === undefined &&
+            message.compression === containerRuntime_1.CompressionAlgorithms.lz4) {
             // Single compressed message
-            (0, common_utils_1.assert)(this.activeBatch === false, "shouldn't receive compressed message in middle of a batch");
+            (0, common_utils_1.assert)(this.activeBatch === false, 0x4ba /* shouldn't receive compressed message in middle of a batch */);
             const contents = common_utils_1.IsoBuffer.from(message.contents.packedContents, "base64");
             const decompressedMessage = (0, lz4js_1.decompress)(contents);
             const intoString = new TextDecoder().decode(decompressedMessage);
             const asObj = JSON.parse(intoString);
-            return Object.assign(Object.assign({}, message), { contents: asObj[0] });
+            return {
+                message: newMessage(message, asObj[0]),
+                state: "Processed",
+            };
         }
-        return message;
+        return {
+            message,
+            state: "Skipped",
+        };
     }
 }
 exports.OpDecompressor = OpDecompressor;
+// We should not be mutating the input message nor its metadata
+const newMessage = (originalMessage, contents) => (Object.assign(Object.assign({}, originalMessage), { contents, compression: undefined, metadata: Object.assign({}, originalMessage.metadata) }));
 //# sourceMappingURL=opDecompressor.js.map

package/dist/opLifecycle/opDecompressor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"opDecompressor.js","sourceRoot":"","sources":["../../src/opLifecycle/opDecompressor.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,iCAAmC;AAEnC,+DAAqF;AACrF,0DAA4D;AAG5D;;;;;;;GAOG;AACH,MAAa,cAAc;IAA3B;QACS,gBAAW,GAAG,KAAK,CAAC;QAEpB,mBAAc,GAAG,CAAC,CAAC;IA0F5B,CAAC;IAxFO,cAAc,CAAC,OAAkC;;QACvD,IAAA,qBAAM,EACL,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,EACtF,KAAK,CAAC,uCAAuC,CAC7C,CAAC;QAEF,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,EAAE;YAC1F,kCAAkC;YAClC,IAAA,qBAAM,EAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,KAAK,CAAC,4CAA4C,CAAC,CAAC;YACvF,IAAI,OAAO,CAAC,WAAW,EAAE;gBACxB,0DAA0D;gBAC1D,IAAA,qBAAM,EACL,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,EACjD,KAAK,CAAC,+DAA+D,CACrE,CAAC;aACF;YAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YAExB,MAAM,QAAQ,GAAG,wBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YAC3E,MAAM,mBAAmB,GAAG,IAAA,kBAAU,EAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAA,iCAAkB,EAAC,mBAAmB,CAAC,CAAC;YAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACrC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;YAEjC,OAAO;gBACN,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;gBAC7E,KAAK,EAAE,UAAU;aACjB,CAAC;SACF;QAED,IACC,IAAI,CAAC,mBAAmB,KAAK,SAAS;YACtC,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,SAAS;YACrC,IAAI,CAAC,WAAW,EACf;YACD,IAAA,qBAAM,EAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAE5E,mCAAmC;YACnC,OAAO;gBACN,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;gBAC7E,KAAK,EAAE,UAAU;aACjB,CAAC;SACF;QAED,IAAI,IAAI,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,KAAK,EAAE;YAChF,0BAA0B;YAC1B,MAAM,aAAa,GAAG,UAAU,CAC/B,OAAO,EACP,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAC/C,CAAC;YAEF,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,IAAI,CAAC,mBAAmB,GAAG,SAAS,CAAC;YACrC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;YAExB,OAAO;gBACN,OAAO,EAAE,aAAa;gBACtB,KAAK,EAAE,WAAW;aAClB,CAAC;SACF;QAED,IACC,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,SAAS;YACrC,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,EAChD;YACD,4BAA4B;YAC5B,IAAA,qBAAM,EACL,IAAI,CAAC,WAAW,KAAK,KAAK,EAC1B,KAAK,CAAC,+DAA+D,CACrE,CAAC;YAEF,MAAM,QAAQ,GAAG,wBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YAC3E,MAAM,mBAAmB,GAAG,IAAA,kBAAU,EAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAErC,OAAO;gBACN,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;gBACtC,KAAK,EAAE,WAAW;aAClB,CAAC;SACF;QAED,OAAO;YACN,OAAO;YACP,KAAK,EAAE,SAAS;SAChB,CAAC;IACH,CAAC;CACD;AA7FD,wCA6FC;AAED,+DAA+D;AAC/D,MAAM,UAAU,GAAG,CAClB,eAA0C,EAC1C,QAAa,EACe,EAAE,CAAC,iCAC5B,eAAe,KAClB,QAAQ,EACR,WAAW,EAAE,SAAS,EACtB,QAAQ,oBAAO,eAAe,CAAC,QAAQ,KACtC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { decompress } from \"lz4js\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { assert, IsoBuffer, Uint8ArrayToString } from \"@fluidframework/common-utils\";\nimport { CompressionAlgorithms } from \"../containerRuntime\";\nimport { IMessageProcessingResult } from \"./definitions\";\n\n/**\n * State machine that \"unrolls\" contents of compressed batches of ops after decompressing them.\n * This class relies on some implicit contracts defined below:\n * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true\n * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set\n * 3. The final message of a batch will have batch metadata set to false\n * 4. An individually compressed op will have undefined batch metadata and compression set to true\n */\nexport class OpDecompressor {\n\tprivate activeBatch = false;\n\tprivate rootMessageContents: any | undefined;\n\tprivate processedCount = 0;\n\n\tpublic processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {\n\t\tassert(\n\t\t\tmessage.compression === undefined || message.compression === CompressionAlgorithms.lz4,\n\t\t\t0x511 /* Only lz4 compression is supported */,\n\t\t);\n\n\t\tif (message.metadata?.batch === true && message.compression === CompressionAlgorithms.lz4) {\n\t\t\t// Beginning of a compressed batch\n\t\t\tassert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);\n\t\t\tif (message.compression) {\n\t\t\t\t// lz4 is the only supported compression algorithm for now\n\t\t\t\tassert(\n\t\t\t\t\tmessage.compression === CompressionAlgorithms.lz4,\n\t\t\t\t\t0x4b9 /* lz4 is currently the only supported compression algorithm */,\n\t\t\t\t);\n\t\t\t}\n\n\t\t\tthis.activeBatch = true;\n\n\t\t\tconst contents = IsoBuffer.from(message.contents.packedContents, \"base64\");\n\t\t\tconst decompressedMessage = decompress(contents);\n\t\t\tconst intoString = Uint8ArrayToString(decompressedMessage);\n\t\t\tconst asObj = JSON.parse(intoString);\n\t\t\tthis.rootMessageContents = asObj;\n\n\t\t\treturn {\n\t\t\t\tmessage: newMessage(message, this.rootMessageContents[this.processedCount++]),\n\t\t\t\tstate: \"Accepted\",\n\t\t\t};\n\t\t}\n\n\t\tif (\n\t\t\tthis.rootMessageContents !== undefined &&\n\t\t\tmessage.metadata?.batch === undefined &&\n\t\t\tthis.activeBatch\n\t\t) {\n\t\t\tassert(message.contents === undefined, 0x512 /* Expecting empty message */);\n\n\t\t\t// Continuation of compressed batch\n\t\t\treturn {\n\t\t\t\tmessage: newMessage(message, this.rootMessageContents[this.processedCount++]),\n\t\t\t\tstate: \"Accepted\",\n\t\t\t};\n\t\t}\n\n\t\tif (this.rootMessageContents !== undefined && message.metadata?.batch === false) {\n\t\t\t// End of compressed batch\n\t\t\tconst returnMessage = newMessage(\n\t\t\t\tmessage,\n\t\t\t\tthis.rootMessageContents[this.processedCount++],\n\t\t\t);\n\n\t\t\tthis.activeBatch = false;\n\t\t\tthis.rootMessageContents = undefined;\n\t\t\tthis.processedCount = 0;\n\n\t\t\treturn {\n\t\t\t\tmessage: returnMessage,\n\t\t\t\tstate: \"Processed\",\n\t\t\t};\n\t\t}\n\n\t\tif (\n\t\t\tmessage.metadata?.batch === undefined &&\n\t\t\tmessage.compression === CompressionAlgorithms.lz4\n\t\t) {\n\t\t\t// Single compressed message\n\t\t\tassert(\n\t\t\t\tthis.activeBatch === false,\n\t\t\t\t0x4ba /* shouldn't receive compressed message in middle of a batch */,\n\t\t\t);\n\n\t\t\tconst contents = IsoBuffer.from(message.contents.packedContents, \"base64\");\n\t\t\tconst decompressedMessage = decompress(contents);\n\t\t\tconst intoString = new TextDecoder().decode(decompressedMessage);\n\t\t\tconst asObj = JSON.parse(intoString);\n\n\t\t\treturn {\n\t\t\t\tmessage: newMessage(message, asObj[0]),\n\t\t\t\tstate: \"Processed\",\n\t\t\t};\n\t\t}\n\n\t\treturn {\n\t\t\tmessage,\n\t\t\tstate: \"Skipped\",\n\t\t};\n\t}\n}\n\n// We should not be mutating the input message nor its metadata\nconst newMessage = (\n\toriginalMessage: ISequencedDocumentMessage,\n\tcontents: any,\n): ISequencedDocumentMessage => ({\n\t...originalMessage,\n\tcontents,\n\tcompression: undefined,\n\tmetadata: { ...originalMessage.metadata },\n});\n"]}

package/dist/opLifecycle/opSplitter.d.ts

@@ -0,0 +1,49 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { IBatchMessage } from "@fluidframework/container-definitions";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from "./definitions";
+/**
+ * Responsible for creating and reconstructing chunked messages.
+ */
+export declare class OpSplitter {
+    private readonly submitBatchFn;
+    private readonly chunkSizeInBytes;
+    private readonly maxBatchSizeInBytes;
+    private readonly chunkMap;
+    private readonly logger;
+    constructor(chunks: [string, string[]][], submitBatchFn: ((batch: IBatchMessage[]) => number) | undefined, chunkSizeInBytes: number, maxBatchSizeInBytes: number, logger: ITelemetryLogger);
+    get isBatchChunkingEnabled(): boolean;
+    get chunks(): ReadonlyMap<string, string[]>;
+    processRemoteMessage(message: ISequencedDocumentMessage): IMessageProcessingResult;
+    clearPartialChunks(clientId: string): void;
+    private addChunk;
+    /**
+     * Splits the first op of a compressed batch in chunks, sends the chunks separately and
+     * returns a new batch composed of the last chunk and the rest of the ops in the original batch.
+     *
+     * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops
+     * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original
+     * uncompressed ops at ingestion in the runtime.
+     *
+     * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire
+     * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.
+     *
+     * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch
+     * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch
+     * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed
+     * (as it is the last chunk).
+     *
+     * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
+     * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.
+     *
+     * @param batch - the compressed batch which needs to be processed
+     * @returns A new adjusted batch which can be sent over the wire
+     */
+    splitCompressedBatch(batch: IBatch): IBatch;
+}
+export declare const splitOp: (op: BatchMessage, chunkSizeInBytes: number) => IChunkedOp[];
+//# sourceMappingURL=opSplitter.d.ts.map

package/dist/opLifecycle/opSplitter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"opSplitter.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAEtE,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AAKtE,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAGjF,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AAE3F;;GAEG;AACH,qBAAa,UAAU;IAOrB,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IACjC,OAAO,CAAC,QAAQ,CAAC,mBAAmB;IAPrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAGvB,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,EACX,aAAa,EAAE,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,MAAM,CAAC,GAAG,SAAS,EAC/D,gBAAgB,EAAE,MAAM,EACxB,mBAAmB,EAAE,MAAM,EAC5C,MAAM,EAAE,gBAAgB;IAMzB,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,IAAW,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAEjD;IAEM,oBAAoB,CAAC,OAAO,EAAE,yBAAyB,GAAG,wBAAwB;IAoClF,kBAAkB,CAAC,QAAQ,EAAE,MAAM;IAM1C,OAAO,CAAC,QAAQ;IA0BhB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACI,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;CAoDlD;AAoBD,eAAO,MAAM,OAAO,OAAQ,YAAY,oBAAoB,MAAM,KAAG,UAAU,EA+B9E,CAAC"}

package/dist/opLifecycle/opSplitter.js

@@ -0,0 +1,173 @@
+"use strict";
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.splitOp = exports.OpSplitter = void 0;
+const common_utils_1 = require("@fluidframework/common-utils");
+const container_utils_1 = require("@fluidframework/container-utils");
+const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
+const containerRuntime_1 = require("../containerRuntime");
+/**
+ * Responsible for creating and reconstructing chunked messages.
+ */
+class OpSplitter {
+    constructor(chunks, submitBatchFn, chunkSizeInBytes, maxBatchSizeInBytes, logger) {
+        this.submitBatchFn = submitBatchFn;
+        this.chunkSizeInBytes = chunkSizeInBytes;
+        this.maxBatchSizeInBytes = maxBatchSizeInBytes;
+        this.chunkMap = new Map(chunks);
+        this.logger = telemetry_utils_1.ChildLogger.create(logger, "OpSplitter");
+    }
+    get isBatchChunkingEnabled() {
+        return this.chunkSizeInBytes < Number.POSITIVE_INFINITY && this.submitBatchFn !== undefined;
+    }
+    get chunks() {
+        return this.chunkMap;
+    }
+    processRemoteMessage(message) {
+        if (message.type !== containerRuntime_1.ContainerMessageType.ChunkedOp) {
+            return {
+                message,
+                state: "Skipped",
+            };
+        }
+        const clientId = message.clientId;
+        const chunkedContent = message.contents;
+        this.addChunk(clientId, chunkedContent, message);
+        if (chunkedContent.chunkId < chunkedContent.totalChunks) {
+            // We are processing the op in chunks but haven't reached
+            // the last chunk yet in order to reconstruct the original op
+            return {
+                message,
+                state: "Accepted",
+            };
+        }
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const serializedContent = this.chunkMap.get(clientId).join("");
+        this.clearPartialChunks(clientId);
+        const newMessage = Object.assign({}, message);
+        newMessage.contents = serializedContent === "" ? undefined : JSON.parse(serializedContent);
+        newMessage.type = chunkedContent.originalType;
+        newMessage.metadata = chunkedContent.originalMetadata;
+        newMessage.compression = chunkedContent.originalCompression;
+        return {
+            message: newMessage,
+            state: "Processed",
+        };
+    }
+    clearPartialChunks(clientId) {
+        if (this.chunkMap.has(clientId)) {
+            this.chunkMap.delete(clientId);
+        }
+    }
+    addChunk(clientId, chunkedContent, originalMessage) {
+        let map = this.chunkMap.get(clientId);
+        if (map === undefined) {
+            map = [];
+            this.chunkMap.set(clientId, map);
+        }
+        if (chunkedContent.chunkId !== map.length + 1) {
+            // We are expecting the chunks to be processed sequentially, in the same order as they are sent.
+            // Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)
+            // holding the existing chunks for that particular clientId.
+            throw new container_utils_1.DataCorruptionError("Chunk Id mismatch", Object.assign(Object.assign({}, (0, container_utils_1.extractSafePropertiesFromMessage)(originalMessage)), { chunkMapLength: map.length, chunkId: chunkedContent.chunkId, totalChunks: chunkedContent.totalChunks }));
+        }
+        map.push(chunkedContent.contents);
+    }
+    /**
+     * Splits the first op of a compressed batch in chunks, sends the chunks separately and
+     * returns a new batch composed of the last chunk and the rest of the ops in the original batch.
+     *
+     * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops
+     * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original
+     * uncompressed ops at ingestion in the runtime.
+     *
+     * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire
+     * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.
+     *
+     * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch
+     * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch
+     * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed
+     * (as it is the last chunk).
+     *
+     * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
+     * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.
+     *
+     * @param batch - the compressed batch which needs to be processed
+     * @returns A new adjusted batch which can be sent over the wire
+     */
+    splitCompressedBatch(batch) {
+        var _a, _b, _c, _d, _e, _f;
+        (0, common_utils_1.assert)(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);
+        (0, common_utils_1.assert)(batch.contentSizeInBytes > 0 && batch.content.length > 0, 0x514 /* Batch needs to be non-empty */);
+        (0, common_utils_1.assert)(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);
+        (0, common_utils_1.assert)(this.chunkSizeInBytes < this.maxBatchSizeInBytes, 0x516 /* Chunk size needs to be smaller than the max batch size */);
+        const firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split
+        (0, common_utils_1.assert)(((_a = firstMessage.metadata) === null || _a === void 0 ? void 0 : _a.compressed) === true || firstMessage.compression !== undefined, 0x517 /* Batch needs to be compressed */);
+        (0, common_utils_1.assert)(((_c = (_b = firstMessage.contents) === null || _b === void 0 ? void 0 : _b.length) !== null && _c !== void 0 ? _c : 0) >= this.chunkSizeInBytes, 0x518 /* First message in the batch needs to be chunkable */);
+        const restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers
+        const chunks = (0, exports.splitOp)(firstMessage, this.chunkSizeInBytes);
+        (0, common_utils_1.assert)(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);
+        // Send the first N-1 chunks immediately
+        for (const chunk of chunks.slice(0, -1)) {
+            this.submitBatchFn([chunkToBatchMessage(chunk, firstMessage.referenceSequenceNumber)]);
+        }
+        // The last chunk will be part of the new batch and needs to
+        // preserve the batch metadata of the original batch
+        const lastChunk = chunkToBatchMessage(chunks[chunks.length - 1], firstMessage.referenceSequenceNumber, { batch: (_d = firstMessage.metadata) === null || _d === void 0 ? void 0 : _d.batch });
+        this.logger.sendPerformanceEvent({
+            eventName: "Chunked compressed batch",
+            length: batch.content.length,
+            sizeInBytes: batch.contentSizeInBytes,
+            chunks: chunks.length,
+            chunkSizeInBytes: this.chunkSizeInBytes,
+        });
+        return {
+            content: [lastChunk, ...restOfMessages],
+            contentSizeInBytes: (_f = (_e = lastChunk.contents) === null || _e === void 0 ? void 0 : _e.length) !== null && _f !== void 0 ? _f : 0,
+        };
+    }
+}
+exports.OpSplitter = OpSplitter;
+const chunkToBatchMessage = (chunk, referenceSequenceNumber, metadata = undefined) => {
+    const payload = {
+        type: containerRuntime_1.ContainerMessageType.ChunkedOp,
+        contents: chunk,
+    };
+    return {
+        contents: JSON.stringify(payload),
+        deserializedContent: payload,
+        metadata,
+        localOpMetadata: undefined,
+        referenceSequenceNumber,
+    };
+};
+const splitOp = (op, chunkSizeInBytes) => {
+    const chunks = [];
+    (0, common_utils_1.assert)(op.contents !== undefined && op.contents !== null, 0x51a /* We should have something to chunk */);
+    const contentLength = op.contents.length;
+    const chunkN = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1;
+    let offset = 0;
+    for (let i = 1; i <= chunkN; i++) {
+        const chunk = {
+            chunkId: i,
+            contents: op.contents.substr(offset, chunkSizeInBytes),
+            originalType: op.deserializedContent.type,
+            totalChunks: chunkN,
+        };
+        if (i === chunkN) {
+            // We don't need to port these to all the chunks,
+            // as we rebuild the original op when we process the
+            // last chunk, therefore it is the only one that needs it.
+            chunk.originalMetadata = op.metadata;
+            chunk.originalCompression = op.compression;
+        }
+        chunks.push(chunk);
+        offset += chunkSizeInBytes;
+    }
+    return chunks;
+};
+exports.splitOp = splitOp;
+//# sourceMappingURL=opSplitter.js.map

package/dist/opLifecycle/opSplitter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"opSplitter.js","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAGH,+DAAsD;AAEtD,qEAGyC;AAEzC,qEAA8D;AAC9D,0DAAoF;AAGpF;;GAEG;AACH,MAAa,UAAU;IAKtB,YACC,MAA4B,EACX,aAA+D,EAC/D,gBAAwB,EACxB,mBAA2B,EAC5C,MAAwB;QAHP,kBAAa,GAAb,aAAa,CAAkD;QAC/D,qBAAgB,GAAhB,gBAAgB,CAAQ;QACxB,wBAAmB,GAAnB,mBAAmB,CAAQ;QAG5C,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAmB,MAAM,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACxD,CAAC;IAED,IAAW,sBAAsB;QAChC,OAAO,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS,CAAC;IAC7F,CAAC;IAED,IAAW,MAAM;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAEM,oBAAoB,CAAC,OAAkC;QAC7D,IAAI,OAAO,CAAC,IAAI,KAAK,uCAAoB,CAAC,SAAS,EAAE;YACpD,OAAO;gBACN,OAAO;gBACP,KAAK,EAAE,SAAS;aAChB,CAAC;SACF;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,MAAM,cAAc,GAAG,OAAO,CAAC,QAAsB,CAAC;QACtD,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QAEjD,IAAI,cAAc,CAAC,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE;YACxD,yDAAyD;YACzD,6DAA6D;YAC7D,OAAO;gBACN,OAAO;gBACP,KAAK,EAAE,UAAU;aACjB,CAAC;SACF;QAED,oEAAoE;QACpE,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAElC,MAAM,UAAU,qBAAQ,OAAO,CAAE,CAAC;QAClC,UAAU,CAAC,QAAQ,GAAG,iBAAiB,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAC3F,UAAU,CAAC,IAAI,GAAG,cAAc,CAAC,YAAY,CAAC;QAC9C,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC,gBAAgB,CAAC;QACtD,UAAU,CAAC,WAAW,GAAG,cAAc,CAAC,mBAAmB,CAAC;QAC5D,OAAO;YACN,OAAO,EAAE,UAAU;YACnB,KAAK,EAAE,WAAW;SAClB,CAAC;IACH,CAAC;IAEM,kBAAkB,CAAC,QAAgB;QACzC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAChC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;SAC/B;IACF,CAAC;IAEO,QAAQ,CACf,QAAgB,EAChB,cAA0B,EAC1B,eAA0C;QAE1C,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,GAAG,KAAK,SAAS,EAAE;YACtB,GAAG,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;SACjC;QAED,IAAI,cAAc,CAAC,OAAO,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;YAC9C,gGAAgG;YAChG,sGAAsG;YACtG,4DAA4D;YAC5D,MAAM,IAAI,qCAAmB,CAAC,mBAAmB,kCAC7C,IAAA,kDAAgC,EAAC,eAAe,CAAC,KACpD,cAAc,EAAE,GAAG,CAAC,MAAM,EAC1B,OAAO,EAAE,cAAc,CAAC,OAAO,EAC/B,WAAW,EAAE,cAAc,CAAC,WAAW,IACtC,CAAC;SACH;QAED,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACI,oBAAoB,CAAC,KAAa;;QACxC,IAAA,qBAAM,EAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QAC9E,IAAA,qBAAM,EACL,KAAK,CAAC,kBAAkB,GAAG,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EACxD,KAAK,CAAC,iCAAiC,CACvC,CAAC;QACF,IAAA,qBAAM,EAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACjF,IAAA,qBAAM,EACL,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,mBAAmB,EAChD,KAAK,CAAC,4DAA4D,CAClE,CAAC;QAEF,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,wEAAwE;QAC/G,IAAA,qBAAM,EACL,CAAA,MAAA,YAAY,CAAC,QAAQ,0CAAE,UAAU,MAAK,IAAI,IAAI,YAAY,CAAC,WAAW,KAAK,SAAS,EACpF,KAAK,CAAC,kCAAkC,CACxC,CAAC;QACF,IAAA,qBAAM,EACL,CAAC,MAAA,MAAA,YAAY,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAC7D,KAAK,CAAC,sDAAsD,CAC5D,CAAC;QAEF,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,uEAAuE;QACtH,MAAM,MAAM,GAAG,IAAA,eAAO,EAAC,YAAY,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAE5D,IAAA,qBAAM,EAAC,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACnF,wCAAwC;QACxC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;YACxC,IAAI,CAAC,aAAa,CAAC,CAAC,mBAAmB,CAAC,KAAK,EAAE,YAAY,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC;SACvF;QAED,4DAA4D;QAC5D,oDAAoD;QACpD,MAAM,SAAS,GAAG,mBAAmB,CACpC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EACzB,YAAY,CAAC,uBAAuB,EACpC,EAAE,KAAK,EAAE,MAAA,YAAY,CAAC,QAAQ,0CAAE,KAAK,EAAE,CACvC,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;YAChC,SAAS,EAAE,0BAA0B;YACrC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;YAC5B,WAAW,EAAE,KAAK,CAAC,kBAAkB;YACrC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACvC,CAAC,CAAC;QAEH,OAAO;YACN,OAAO,EAAE,CAAC,SAAS,EAAE,GAAG,cAAc,CAAC;YACvC,kBAAkB,EAAE,MAAA,MAAA,SAAS,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC;SACnD,CAAC;IACH,CAAC;CACD;AAtKD,gCAsKC;AAED,MAAM,mBAAmB,GAAG,CAC3B,KAAiB,EACjB,uBAA+B,EAC/B,WAAgD,SAAS,EAC1C,EAAE;IACjB,MAAM,OAAO,GAA4B;QACxC,IAAI,EAAE,uCAAoB,CAAC,SAAS;QACpC,QAAQ,EAAE,KAAK;KACf,CAAC;IACF,OAAO;QACN,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QACjC,mBAAmB,EAAE,OAAO;QAC5B,QAAQ;QACR,eAAe,EAAE,SAAS;QAC1B,uBAAuB;KACvB,CAAC;AACH,CAAC,CAAC;AAEK,MAAM,OAAO,GAAG,CAAC,EAAgB,EAAE,gBAAwB,EAAgB,EAAE;IACnF,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAA,qBAAM,EACL,EAAE,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,CAAC,QAAQ,KAAK,IAAI,EACjD,KAAK,CAAC,uCAAuC,CAC7C,CAAC;IAEF,MAAM,aAAa,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACtE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC,EAAE,EAAE;QACjC,MAAM,KAAK,GAAe;YACzB,OAAO,EAAE,CAAC;YACV,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC;YACtD,YAAY,EAAE,EAAE,CAAC,mBAAmB,CAAC,IAAI;YACzC,WAAW,EAAE,MAAM;SACnB,CAAC;QAEF,IAAI,CAAC,KAAK,MAAM,EAAE;YACjB,iDAAiD;YACjD,oDAAoD;YACpD,0DAA0D;YAC1D,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,QAAQ,CAAC;YACrC,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,WAAW,CAAC;SAC3C;QAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,IAAI,gBAAgB,CAAC;KAC3B;IAED,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AA/BW,QAAA,OAAO,WA+BlB","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert } from \"@fluidframework/common-utils\";\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport {\n\tDataCorruptionError,\n\textractSafePropertiesFromMessage,\n} from \"@fluidframework/container-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { ContainerMessageType, ContainerRuntimeMessage } from \"../containerRuntime\";\nimport { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from \"./definitions\";\n\n/**\n * Responsible for creating and reconstructing chunked messages.\n */\nexport class OpSplitter {\n\t// Local copy of incomplete received chunks.\n\tprivate readonly chunkMap: Map<string, string[]>;\n\tprivate readonly logger;\n\n\tconstructor(\n\t\tchunks: [string, string[]][],\n\t\tprivate readonly submitBatchFn: ((batch: IBatchMessage[]) => number) | undefined,\n\t\tprivate readonly chunkSizeInBytes: number,\n\t\tprivate readonly maxBatchSizeInBytes: number,\n\t\tlogger: ITelemetryLogger,\n\t) {\n\t\tthis.chunkMap = new Map<string, string[]>(chunks);\n\t\tthis.logger = ChildLogger.create(logger, \"OpSplitter\");\n\t}\n\n\tpublic get isBatchChunkingEnabled(): boolean {\n\t\treturn this.chunkSizeInBytes < Number.POSITIVE_INFINITY && this.submitBatchFn !== undefined;\n\t}\n\n\tpublic get chunks(): ReadonlyMap<string, string[]> {\n\t\treturn this.chunkMap;\n\t}\n\n\tpublic processRemoteMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {\n\t\tif (message.type !== ContainerMessageType.ChunkedOp) {\n\t\t\treturn {\n\t\t\t\tmessage,\n\t\t\t\tstate: \"Skipped\",\n\t\t\t};\n\t\t}\n\n\t\tconst clientId = message.clientId;\n\t\tconst chunkedContent = message.contents as IChunkedOp;\n\t\tthis.addChunk(clientId, chunkedContent, message);\n\n\t\tif (chunkedContent.chunkId < chunkedContent.totalChunks) {\n\t\t\t// We are processing the op in chunks but haven't reached\n\t\t\t// the last chunk yet in order to reconstruct the original op\n\t\t\treturn {\n\t\t\t\tmessage,\n\t\t\t\tstate: \"Accepted\",\n\t\t\t};\n\t\t}\n\n\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\tconst serializedContent = this.chunkMap.get(clientId)!.join(\"\");\n\t\tthis.clearPartialChunks(clientId);\n\n\t\tconst newMessage = { ...message };\n\t\tnewMessage.contents = serializedContent === \"\" ? undefined : JSON.parse(serializedContent);\n\t\tnewMessage.type = chunkedContent.originalType;\n\t\tnewMessage.metadata = chunkedContent.originalMetadata;\n\t\tnewMessage.compression = chunkedContent.originalCompression;\n\t\treturn {\n\t\t\tmessage: newMessage,\n\t\t\tstate: \"Processed\",\n\t\t};\n\t}\n\n\tpublic clearPartialChunks(clientId: string) {\n\t\tif (this.chunkMap.has(clientId)) {\n\t\t\tthis.chunkMap.delete(clientId);\n\t\t}\n\t}\n\n\tprivate addChunk(\n\t\tclientId: string,\n\t\tchunkedContent: IChunkedOp,\n\t\toriginalMessage: ISequencedDocumentMessage,\n\t) {\n\t\tlet map = this.chunkMap.get(clientId);\n\t\tif (map === undefined) {\n\t\t\tmap = [];\n\t\t\tthis.chunkMap.set(clientId, map);\n\t\t}\n\n\t\tif (chunkedContent.chunkId !== map.length + 1) {\n\t\t\t// We are expecting the chunks to be processed sequentially, in the same order as they are sent.\n\t\t\t// Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)\n\t\t\t// holding the existing chunks for that particular clientId.\n\t\t\tthrow new DataCorruptionError(\"Chunk Id mismatch\", {\n\t\t\t\t...extractSafePropertiesFromMessage(originalMessage),\n\t\t\t\tchunkMapLength: map.length,\n\t\t\t\tchunkId: chunkedContent.chunkId,\n\t\t\t\ttotalChunks: chunkedContent.totalChunks,\n\t\t\t});\n\t\t}\n\n\t\tmap.push(chunkedContent.contents);\n\t}\n\n\t/**\n\t * Splits the first op of a compressed batch in chunks, sends the chunks separately and\n\t * returns a new batch composed of the last chunk and the rest of the ops in the original batch.\n\t *\n\t * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops\n\t * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original\n\t * uncompressed ops at ingestion in the runtime.\n\t *\n\t * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire\n\t * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.\n\t *\n\t * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch\n\t * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch\n\t * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed\n\t * (as it is the last chunk).\n\t *\n\t * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.\n\t * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.\n\t *\n\t * @param batch - the compressed batch which needs to be processed\n\t * @returns A new adjusted batch which can be sent over the wire\n\t */\n\tpublic splitCompressedBatch(batch: IBatch): IBatch {\n\t\tassert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);\n\t\tassert(\n\t\t\tbatch.contentSizeInBytes > 0 && batch.content.length > 0,\n\t\t\t0x514 /* Batch needs to be non-empty */,\n\t\t);\n\t\tassert(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);\n\t\tassert(\n\t\t\tthis.chunkSizeInBytes < this.maxBatchSizeInBytes,\n\t\t\t0x516 /* Chunk size needs to be smaller than the max batch size */,\n\t\t);\n\n\t\tconst firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split\n\t\tassert(\n\t\t\tfirstMessage.metadata?.compressed === true || firstMessage.compression !== undefined,\n\t\t\t0x517 /* Batch needs to be compressed */,\n\t\t);\n\t\tassert(\n\t\t\t(firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,\n\t\t\t0x518 /* First message in the batch needs to be chunkable */,\n\t\t);\n\n\t\tconst restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers\n\t\tconst chunks = splitOp(firstMessage, this.chunkSizeInBytes);\n\n\t\tassert(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);\n\t\t// Send the first N-1 chunks immediately\n\t\tfor (const chunk of chunks.slice(0, -1)) {\n\t\t\tthis.submitBatchFn([chunkToBatchMessage(chunk, firstMessage.referenceSequenceNumber)]);\n\t\t}\n\n\t\t// The last chunk will be part of the new batch and needs to\n\t\t// preserve the batch metadata of the original batch\n\t\tconst lastChunk = chunkToBatchMessage(\n\t\t\tchunks[chunks.length - 1],\n\t\t\tfirstMessage.referenceSequenceNumber,\n\t\t\t{ batch: firstMessage.metadata?.batch },\n\t\t);\n\n\t\tthis.logger.sendPerformanceEvent({\n\t\t\teventName: \"Chunked compressed batch\",\n\t\t\tlength: batch.content.length,\n\t\t\tsizeInBytes: batch.contentSizeInBytes,\n\t\t\tchunks: chunks.length,\n\t\t\tchunkSizeInBytes: this.chunkSizeInBytes,\n\t\t});\n\n\t\treturn {\n\t\t\tcontent: [lastChunk, ...restOfMessages],\n\t\t\tcontentSizeInBytes: lastChunk.contents?.length ?? 0,\n\t\t};\n\t}\n}\n\nconst chunkToBatchMessage = (\n\tchunk: IChunkedOp,\n\treferenceSequenceNumber: number,\n\tmetadata: Record<string, unknown> | undefined = undefined,\n): BatchMessage => {\n\tconst payload: ContainerRuntimeMessage = {\n\t\ttype: ContainerMessageType.ChunkedOp,\n\t\tcontents: chunk,\n\t};\n\treturn {\n\t\tcontents: JSON.stringify(payload),\n\t\tdeserializedContent: payload,\n\t\tmetadata,\n\t\tlocalOpMetadata: undefined,\n\t\treferenceSequenceNumber,\n\t};\n};\n\nexport const splitOp = (op: BatchMessage, chunkSizeInBytes: number): IChunkedOp[] => {\n\tconst chunks: IChunkedOp[] = [];\n\tassert(\n\t\top.contents !== undefined && op.contents !== null,\n\t\t0x51a /* We should have something to chunk */,\n\t);\n\n\tconst contentLength = op.contents.length;\n\tconst chunkN = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1;\n\tlet offset = 0;\n\tfor (let i = 1; i <= chunkN; i++) {\n\t\tconst chunk: IChunkedOp = {\n\t\t\tchunkId: i,\n\t\t\tcontents: op.contents.substr(offset, chunkSizeInBytes),\n\t\t\toriginalType: op.deserializedContent.type,\n\t\t\ttotalChunks: chunkN,\n\t\t};\n\n\t\tif (i === chunkN) {\n\t\t\t// We don't need to port these to all the chunks,\n\t\t\t// as we rebuild the original op when we process the\n\t\t\t// last chunk, therefore it is the only one that needs it.\n\t\t\tchunk.originalMetadata = op.metadata;\n\t\t\tchunk.originalCompression = op.compression;\n\t\t}\n\n\t\tchunks.push(chunk);\n\t\toffset += chunkSizeInBytes;\n\t}\n\n\treturn chunks;\n};\n"]}

package/dist/opLifecycle/outbox.d.ts

@@ -0,0 +1,52 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { IContainerContext } from "@fluidframework/container-definitions";
+import { ICompressionRuntimeOptions } from "../containerRuntime";
+import { PendingStateManager } from "../pendingStateManager";
+import { BatchMessage } from "./definitions";
+import { OpCompressor } from "./opCompressor";
+import { OpSplitter } from "./opSplitter";
+export interface IOutboxConfig {
+    readonly compressionOptions: ICompressionRuntimeOptions;
+    readonly maxBatchSizeInBytes: number;
+    readonly enableOpReentryCheck?: boolean;
+}
+export interface IOutboxParameters {
+    readonly shouldSend: () => boolean;
+    readonly pendingStateManager: PendingStateManager;
+    readonly containerContext: IContainerContext;
+    readonly config: IOutboxConfig;
+    readonly compressor: OpCompressor;
+    readonly splitter: OpSplitter;
+    readonly logger: ITelemetryLogger;
+}
+export declare class Outbox {
+    private readonly params;
+    private readonly attachFlowBatch;
+    private readonly mainBatch;
+    private readonly defaultAttachFlowSoftLimitInBytes;
+    constructor(params: IOutboxParameters);
+    get isEmpty(): boolean;
+    submit(message: BatchMessage): void;
+    submitAttach(message: BatchMessage): void;
+    flush(): void;
+    private flushInternal;
+    private compressBatch;
+    /**
+     * Sends the batch object to the container context to be sent over the wire.
+     *
+     * @param batch - batch to be sent
+     * @returns the client sequence number of the last batched op which was sent and
+     * -1 if there are no ops or the container cannot send ops.
+     */
+    private sendBatch;
+    private persistBatch;
+    checkpoint(): {
+        mainBatch: import("./definitions").IBatchCheckpoint;
+        attachFlowBatch: import("./definitions").IBatchCheckpoint;
+    };
+}
+//# sourceMappingURL=outbox.d.ts.map

package/dist/opLifecycle/outbox.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"outbox.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/outbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAEtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,uCAAuC,CAAC;AAG1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,OAAO,EAAE,YAAY,EAAU,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,0BAA0B,CAAC;IAExD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,OAAO,CAAC;CACxC;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,OAAO,CAAC;IACnC,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;IAClD,QAAQ,CAAC,gBAAgB,EAAE,iBAAiB,CAAC;IAC7C,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,YAAY,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;CAClC;AAED,qBAAa,MAAM;IAKN,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJnC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAa;gBAElC,MAAM,EAAE,iBAAiB;IAyBtD,IAAW,OAAO,IAAI,OAAO,CAE5B;IAEM,MAAM,CAAC,OAAO,EAAE,YAAY;IAW5B,YAAY,CAAC,OAAO,EAAE,YAAY;IA6BlC,KAAK;IAKZ,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,aAAa;IA+BrB;;;;;;OAMG;IACH,OAAO,CAAC,SAAS;IA6CjB,OAAO,CAAC,YAAY;IAkBb,UAAU;;;;CAMjB"}