@fluidframework/container-runtime 1.4.0-121020 → 2.0.0-dev-rc.1.0.0.225277

package/lib/opLifecycle/batchManager.d.mts

@@ -0,0 +1,48 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { ICompressionRuntimeOptions } from "../containerRuntime.mjs";
+import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions.mjs";
+export interface IBatchManagerOptions {
+    readonly hardLimit: number;
+    readonly softLimit?: number;
+    readonly compressionOptions?: ICompressionRuntimeOptions;
+}
+export interface BatchSequenceNumbers {
+    referenceSequenceNumber?: number;
+    clientSequenceNumber?: number;
+}
+/**
+ * Helper class that manages partial batch & rollback.
+ */
+export declare class BatchManager {
+    readonly options: IBatchManagerOptions;
+    private pendingBatch;
+    private batchContentSize;
+    private hasReentrantOps;
+    get length(): number;
+    get contentSizeInBytes(): number;
+    get sequenceNumbers(): BatchSequenceNumbers;
+    private get referenceSequenceNumber();
+    private clientSequenceNumber;
+    constructor(options: IBatchManagerOptions);
+    push(message: BatchMessage, reentrant: boolean, currentClientSequenceNumber?: number): boolean;
+    get empty(): boolean;
+    popBatch(): IBatch;
+    /**
+     * Capture the pending state at this point
+     */
+    checkpoint(): IBatchCheckpoint;
+}
+/**
+ * Estimates the real size in bytes on the socket for a given batch. It assumes that
+ * the envelope size (and the size of an empty op) is 200 bytes, taking into account
+ * extra overhead from stringification.
+ *
+ * @param batch - the batch to inspect
+ * @returns An estimate of the payload size in bytes which will be produced when the batch is sent over the wire
+ */
+export declare const estimateSocketSize: (batch: IBatch) => number;
+export declare const sequenceNumbersMatch: (seqNums: BatchSequenceNumbers, otherSeqNums: BatchSequenceNumbers) => boolean;
+//# sourceMappingURL=batchManager.d.mts.map

package/lib/opLifecycle/batchManager.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batchManager.d.mts","sourceRoot":"","sources":["../../src/opLifecycle/batchManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;OAEI,EAAE,0BAA0B,EAAE;OAC9B,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE;AAEjD,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,0BAA0B,CAAC;CACzD;AAED,MAAM,WAAW,oBAAoB;IACpC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAQD;;GAEG;AACH,qBAAa,YAAY;aA2BI,OAAO,EAAE,oBAAoB;IA1BzD,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,eAAe,CAAS;IAEhC,IAAW,MAAM,WAEhB;IACD,IAAW,kBAAkB,WAE5B;IAED,IAAW,eAAe,IAAI,oBAAoB,CAKjD;IAED,OAAO,KAAK,uBAAuB,GAIlC;IAED,OAAO,CAAC,oBAAoB,CAAqB;gBAErB,OAAO,EAAE,oBAAoB;IAElD,IAAI,CACV,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,OAAO,EAClB,2BAA2B,CAAC,EAAE,MAAM,GAClC,OAAO;IAsCV,IAAW,KAAK,YAEf;IAEM,QAAQ,IAAI,MAAM;IAgBzB;;OAEG;IACI,UAAU,IAAI,gBAAgB;CAerC;AAiBD;;;;;;;GAOG;AACH,eAAO,MAAM,kBAAkB,UAAW,MAAM,KAAG,MAElD,CAAC;AAEF,eAAO,MAAM,oBAAoB,YACvB,oBAAoB,gBACf,oBAAoB,KAChC,OASF,CAAC"}

package/lib/opLifecycle/batchManager.mjs

@@ -0,0 +1,133 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+/**
+ * Estimated size of the stringification overhead for an op accumulated
+ * from runtime to loader to the service.
+ */
+const opOverhead = 200;
+/**
+ * Helper class that manages partial batch & rollback.
+ */
+export class BatchManager {
+    get length() {
+        return this.pendingBatch.length;
+    }
+    get contentSizeInBytes() {
+        return this.batchContentSize;
+    }
+    get sequenceNumbers() {
+        return {
+            referenceSequenceNumber: this.referenceSequenceNumber,
+            clientSequenceNumber: this.clientSequenceNumber,
+        };
+    }
+    get referenceSequenceNumber() {
+        return this.pendingBatch.length === 0
+            ? undefined
+            : this.pendingBatch[this.pendingBatch.length - 1].referenceSequenceNumber;
+    }
+    constructor(options) {
+        this.options = options;
+        this.pendingBatch = [];
+        this.batchContentSize = 0;
+        this.hasReentrantOps = false;
+    }
+    push(message, reentrant, currentClientSequenceNumber) {
+        const contentSize = this.batchContentSize + (message.contents?.length ?? 0);
+        const opCount = this.pendingBatch.length;
+        this.hasReentrantOps = this.hasReentrantOps || reentrant;
+        // Attempt to estimate batch size, aka socket message size.
+        // Each op has pretty large envelope, estimating to be 200 bytes.
+        // Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.
+        // Not taking it into account, as compression work should help there - compressed payload will be
+        // initially stored as base64, and that requires only 2 extra escape characters.
+        const socketMessageSize = contentSize + opOverhead * opCount;
+        // If we were provided soft limit, check for exceeding it.
+        // But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)
+        // and start over. That's not an option if we have no ops.
+        // If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.
+        // Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.
+        if (this.options.softLimit !== undefined &&
+            this.length > 0 &&
+            socketMessageSize >= this.options.softLimit) {
+            return false;
+        }
+        if (socketMessageSize >= this.options.hardLimit) {
+            return false;
+        }
+        if (this.pendingBatch.length === 0) {
+            this.clientSequenceNumber = currentClientSequenceNumber;
+        }
+        this.batchContentSize = contentSize;
+        this.pendingBatch.push(message);
+        return true;
+    }
+    get empty() {
+        return this.pendingBatch.length === 0;
+    }
+    popBatch() {
+        const batch = {
+            content: this.pendingBatch,
+            contentSizeInBytes: this.batchContentSize,
+            referenceSequenceNumber: this.referenceSequenceNumber,
+            hasReentrantOps: this.hasReentrantOps,
+        };
+        this.pendingBatch = [];
+        this.batchContentSize = 0;
+        this.clientSequenceNumber = undefined;
+        this.hasReentrantOps = false;
+        return addBatchMetadata(batch);
+    }
+    /**
+     * Capture the pending state at this point
+     */
+    checkpoint() {
+        const startPoint = this.pendingBatch.length;
+        return {
+            rollback: (process) => {
+                for (let i = this.pendingBatch.length; i > startPoint;) {
+                    i--;
+                    const message = this.pendingBatch[i];
+                    this.batchContentSize -= message.contents?.length ?? 0;
+                    process(message);
+                }
+                this.pendingBatch.length = startPoint;
+            },
+        };
+    }
+}
+const addBatchMetadata = (batch) => {
+    if (batch.content.length > 1) {
+        batch.content[0].metadata = {
+            ...batch.content[0].metadata,
+            batch: true,
+        };
+        batch.content[batch.content.length - 1].metadata = {
+            ...batch.content[batch.content.length - 1].metadata,
+            batch: false,
+        };
+    }
+    return batch;
+};
+/**
+ * Estimates the real size in bytes on the socket for a given batch. It assumes that
+ * the envelope size (and the size of an empty op) is 200 bytes, taking into account
+ * extra overhead from stringification.
+ *
+ * @param batch - the batch to inspect
+ * @returns An estimate of the payload size in bytes which will be produced when the batch is sent over the wire
+ */
+export const estimateSocketSize = (batch) => {
+    return batch.contentSizeInBytes + opOverhead * batch.content.length;
+};
+export const sequenceNumbersMatch = (seqNums, otherSeqNums) => {
+    return ((seqNums.referenceSequenceNumber === undefined ||
+        otherSeqNums.referenceSequenceNumber === undefined ||
+        seqNums.referenceSequenceNumber === otherSeqNums.referenceSequenceNumber) &&
+        (seqNums.clientSequenceNumber === undefined ||
+            otherSeqNums.clientSequenceNumber === undefined ||
+            seqNums.clientSequenceNumber === otherSeqNums.clientSequenceNumber));
+};
+//# sourceMappingURL=batchManager.mjs.map

package/lib/opLifecycle/batchManager.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"batchManager.mjs","sourceRoot":"","sources":["../../src/opLifecycle/batchManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAgBH;;;GAGG;AACH,MAAM,UAAU,GAAG,GAAG,CAAC;AAEvB;;GAEG;AACH,MAAM,OAAO,YAAY;IAKxB,IAAW,MAAM;QAChB,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;IACjC,CAAC;IACD,IAAW,kBAAkB;QAC5B,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC9B,CAAC;IAED,IAAW,eAAe;QACzB,OAAO;YACN,uBAAuB,EAAE,IAAI,CAAC,uBAAuB;YACrD,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;SAC/C,CAAC;IACH,CAAC;IAED,IAAY,uBAAuB;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YACpC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,uBAAuB,CAAC;IAC5E,CAAC;IAID,YAA4B,OAA6B;QAA7B,YAAO,GAAP,OAAO,CAAsB;QA1BjD,iBAAY,GAAmB,EAAE,CAAC;QAClC,qBAAgB,GAAG,CAAC,CAAC;QACrB,oBAAe,GAAG,KAAK,CAAC;IAwB4B,CAAC;IAEtD,IAAI,CACV,OAAqB,EACrB,SAAkB,EAClB,2BAAoC;QAEpC,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;QAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QACzC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,SAAS,CAAC;QAEzD,2DAA2D;QAC3D,iEAAiE;QACjE,sGAAsG;QACtG,iGAAiG;QACjG,gFAAgF;QAChF,MAAM,iBAAiB,GAAG,WAAW,GAAG,UAAU,GAAG,OAAO,CAAC;QAE7D,0DAA0D;QAC1D,wGAAwG;QACxG,0DAA0D;QAC1D,yGAAyG;QACzG,mGAAmG;QACnG,IACC,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS;YACpC,IAAI,CAAC,MAAM,GAAG,CAAC;YACf,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAC1C;YACD,OAAO,KAAK,CAAC;SACb;QAED,IAAI,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAChD,OAAO,KAAK,CAAC;SACb;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;YACnC,IAAI,CAAC,oBAAoB,GAAG,2BAA2B,CAAC;SACxD;QAED,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC;QACpC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAW,KAAK;QACf,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC;IACvC,CAAC;IAEM,QAAQ;QACd,MAAM,KAAK,GAAW;YACrB,OAAO,EAAE,IAAI,CAAC,YAAY;YAC1B,kBAAkB,EAAE,IAAI,CAAC,gBAAgB;YACzC,uBAAuB,EAAE,IAAI,CAAC,uBAAuB;YACrD,eAAe,EAAE,IAAI,CAAC,eAAe;SACrC,CAAC;QAEF,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAC1B,IAAI,CAAC,oBAAoB,GAAG,SAAS,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAE7B,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACI,UAAU;QAChB,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5C,OAAO;YACN,QAAQ,EAAE,CAAC,OAAwC,EAAE,EAAE;gBACtD,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,GAAG,UAAU,GAAI;oBACxD,CAAC,EAAE,CAAC;oBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBACrC,IAAI,CAAC,gBAAgB,IAAI,OAAO,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC;oBACvD,OAAO,CAAC,OAAO,CAAC,CAAC;iBACjB;gBAED,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU,CAAC;YACvC,CAAC;SACD,CAAC;IACH,CAAC;CACD;AAED,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAU,EAAE;IAClD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG;YAC3B,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ;YAC5B,KAAK,EAAE,IAAI;SACX,CAAC;QACF,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,GAAG;YAClD,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ;YACnD,KAAK,EAAE,KAAK;SACZ,CAAC;KACF;IAED,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,KAAa,EAAU,EAAE;IAC3D,OAAO,KAAK,CAAC,kBAAkB,GAAG,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;AACrE,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CACnC,OAA6B,EAC7B,YAAkC,EACxB,EAAE;IACZ,OAAO,CACN,CAAC,OAAO,CAAC,uBAAuB,KAAK,SAAS;QAC7C,YAAY,CAAC,uBAAuB,KAAK,SAAS;QAClD,OAAO,CAAC,uBAAuB,KAAK,YAAY,CAAC,uBAAuB,CAAC;QAC1E,CAAC,OAAO,CAAC,oBAAoB,KAAK,SAAS;YAC1C,YAAY,CAAC,oBAAoB,KAAK,SAAS;YAC/C,OAAO,CAAC,oBAAoB,KAAK,YAAY,CAAC,oBAAoB,CAAC,CACpE,CAAC;AACH,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ICompressionRuntimeOptions } from \"../containerRuntime\";\nimport { BatchMessage, IBatch, IBatchCheckpoint } from \"./definitions\";\n\nexport interface IBatchManagerOptions {\n\treadonly hardLimit: number;\n\treadonly softLimit?: number;\n\treadonly compressionOptions?: ICompressionRuntimeOptions;\n}\n\nexport interface BatchSequenceNumbers {\n\treferenceSequenceNumber?: number;\n\tclientSequenceNumber?: number;\n}\n\n/**\n * Estimated size of the stringification overhead for an op accumulated\n * from runtime to loader to the service.\n */\nconst opOverhead = 200;\n\n/**\n * Helper class that manages partial batch & rollback.\n */\nexport class BatchManager {\n\tprivate pendingBatch: BatchMessage[] = [];\n\tprivate batchContentSize = 0;\n\tprivate hasReentrantOps = false;\n\n\tpublic get length() {\n\t\treturn this.pendingBatch.length;\n\t}\n\tpublic get contentSizeInBytes() {\n\t\treturn this.batchContentSize;\n\t}\n\n\tpublic get sequenceNumbers(): BatchSequenceNumbers {\n\t\treturn {\n\t\t\treferenceSequenceNumber: this.referenceSequenceNumber,\n\t\t\tclientSequenceNumber: this.clientSequenceNumber,\n\t\t};\n\t}\n\n\tprivate get referenceSequenceNumber(): number | undefined {\n\t\treturn this.pendingBatch.length === 0\n\t\t\t? undefined\n\t\t\t: this.pendingBatch[this.pendingBatch.length - 1].referenceSequenceNumber;\n\t}\n\n\tprivate clientSequenceNumber: number | undefined;\n\n\tconstructor(public readonly options: IBatchManagerOptions) {}\n\n\tpublic push(\n\t\tmessage: BatchMessage,\n\t\treentrant: boolean,\n\t\tcurrentClientSequenceNumber?: number,\n\t): boolean {\n\t\tconst contentSize = this.batchContentSize + (message.contents?.length ?? 0);\n\t\tconst opCount = this.pendingBatch.length;\n\t\tthis.hasReentrantOps = this.hasReentrantOps || reentrant;\n\n\t\t// Attempt to estimate batch size, aka socket message size.\n\t\t// Each op has pretty large envelope, estimating to be 200 bytes.\n\t\t// Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.\n\t\t// Not taking it into account, as compression work should help there - compressed payload will be\n\t\t// initially stored as base64, and that requires only 2 extra escape characters.\n\t\tconst socketMessageSize = contentSize + opOverhead * opCount;\n\n\t\t// If we were provided soft limit, check for exceeding it.\n\t\t// But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)\n\t\t// and start over. That's not an option if we have no ops.\n\t\t// If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.\n\t\t// Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.\n\t\tif (\n\t\t\tthis.options.softLimit !== undefined &&\n\t\t\tthis.length > 0 &&\n\t\t\tsocketMessageSize >= this.options.softLimit\n\t\t) {\n\t\t\treturn false;\n\t\t}\n\n\t\tif (socketMessageSize >= this.options.hardLimit) {\n\t\t\treturn false;\n\t\t}\n\n\t\tif (this.pendingBatch.length === 0) {\n\t\t\tthis.clientSequenceNumber = currentClientSequenceNumber;\n\t\t}\n\n\t\tthis.batchContentSize = contentSize;\n\t\tthis.pendingBatch.push(message);\n\t\treturn true;\n\t}\n\n\tpublic get empty() {\n\t\treturn this.pendingBatch.length === 0;\n\t}\n\n\tpublic popBatch(): IBatch {\n\t\tconst batch: IBatch = {\n\t\t\tcontent: this.pendingBatch,\n\t\t\tcontentSizeInBytes: this.batchContentSize,\n\t\t\treferenceSequenceNumber: this.referenceSequenceNumber,\n\t\t\thasReentrantOps: this.hasReentrantOps,\n\t\t};\n\n\t\tthis.pendingBatch = [];\n\t\tthis.batchContentSize = 0;\n\t\tthis.clientSequenceNumber = undefined;\n\t\tthis.hasReentrantOps = false;\n\n\t\treturn addBatchMetadata(batch);\n\t}\n\n\t/**\n\t * Capture the pending state at this point\n\t */\n\tpublic checkpoint(): IBatchCheckpoint {\n\t\tconst startPoint = this.pendingBatch.length;\n\t\treturn {\n\t\t\trollback: (process: (message: BatchMessage) => void) => {\n\t\t\t\tfor (let i = this.pendingBatch.length; i > startPoint; ) {\n\t\t\t\t\ti--;\n\t\t\t\t\tconst message = this.pendingBatch[i];\n\t\t\t\t\tthis.batchContentSize -= message.contents?.length ?? 0;\n\t\t\t\t\tprocess(message);\n\t\t\t\t}\n\n\t\t\t\tthis.pendingBatch.length = startPoint;\n\t\t\t},\n\t\t};\n\t}\n}\n\nconst addBatchMetadata = (batch: IBatch): IBatch => {\n\tif (batch.content.length > 1) {\n\t\tbatch.content[0].metadata = {\n\t\t\t...batch.content[0].metadata,\n\t\t\tbatch: true,\n\t\t};\n\t\tbatch.content[batch.content.length - 1].metadata = {\n\t\t\t...batch.content[batch.content.length - 1].metadata,\n\t\t\tbatch: false,\n\t\t};\n\t}\n\n\treturn batch;\n};\n\n/**\n * Estimates the real size in bytes on the socket for a given batch. It assumes that\n * the envelope size (and the size of an empty op) is 200 bytes, taking into account\n * extra overhead from stringification.\n *\n * @param batch - the batch to inspect\n * @returns An estimate of the payload size in bytes which will be produced when the batch is sent over the wire\n */\nexport const estimateSocketSize = (batch: IBatch): number => {\n\treturn batch.contentSizeInBytes + opOverhead * batch.content.length;\n};\n\nexport const sequenceNumbersMatch = (\n\tseqNums: BatchSequenceNumbers,\n\totherSeqNums: BatchSequenceNumbers,\n): boolean => {\n\treturn (\n\t\t(seqNums.referenceSequenceNumber === undefined ||\n\t\t\totherSeqNums.referenceSequenceNumber === undefined ||\n\t\t\tseqNums.referenceSequenceNumber === otherSeqNums.referenceSequenceNumber) &&\n\t\t(seqNums.clientSequenceNumber === undefined ||\n\t\t\totherSeqNums.clientSequenceNumber === undefined ||\n\t\t\tseqNums.clientSequenceNumber === otherSeqNums.clientSequenceNumber)\n\t);\n};\n"]}

package/lib/opLifecycle/definitions.d.mts

@@ -0,0 +1,83 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { IBatchMessage } from "@fluidframework/container-definitions";
+import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
+import { CompressionAlgorithms } from "../containerRuntime.mjs";
+import { ContainerMessageType } from "../messageTypes.mjs";
+/**
+ * Batch message type used internally by the runtime
+ */
+export type BatchMessage = IBatchMessage & {
+    localOpMetadata: unknown;
+    type: ContainerMessageType;
+    referenceSequenceNumber: number;
+    compression?: CompressionAlgorithms;
+};
+/**
+ * Batch interface used internally by the runtime.
+ */
+export interface IBatch {
+    /**
+     * Sum of the in-memory content sizes of all messages in the batch.
+     * If the batch is compressed, this number reflects the post-compression size.
+     */
+    readonly contentSizeInBytes: number;
+    /**
+     * All the messages in the batch
+     */
+    readonly content: BatchMessage[];
+    /**
+     * The reference sequence number for the batch
+     */
+    readonly referenceSequenceNumber: number | undefined;
+    /**
+     * Wether or not the batch contains at least one op which was produced as the result
+     * of processing another op. This means that the batch must be rebased before
+     * submitted, to ensure that all ops have the same reference sequence numbers and a
+     * consistent view of the data model. This happens when the op is created within a
+     * 'changed' event handler of a DDS and will have a different reference sequence number
+     * than the rest of the ops in the batch, meaning that it has a different view of the
+     * state of the data model, therefore all ops must be resubmitted and rebased to the current
+     * reference sequence number to be in agreement about the data model state.
+     */
+    readonly hasReentrantOps?: boolean;
+}
+export interface IBatchCheckpoint {
+    rollback: (action: (message: BatchMessage) => void) => void;
+}
+/**
+ * @internal
+ */
+export interface IChunkedOp {
+    chunkId: number;
+    totalChunks: number;
+    contents: string;
+    originalType: MessageType | ContainerMessageType;
+    originalMetadata?: Record<string, unknown>;
+    originalCompression?: string;
+}
+/**
+ * The state of remote message processing:
+ * `Processed` - the message can be considered processed
+ * `Skipped` - the message was ignored by the processor
+ * `Accepted` - the message was processed partially. Eventually, a message
+ * will make the processor return `Processed`.
+ */
+export type ProcessingState = "Processed" | "Skipped" | "Accepted";
+/**
+ * Return type for functions which process remote messages
+ */
+export interface IMessageProcessingResult {
+    /**
+     * A shallow copy of the input message if processing happened, or
+     * the original message otherwise
+     */
+    readonly message: ISequencedDocumentMessage;
+    /**
+     * Processing result of the input message.
+     */
+    readonly state: ProcessingState;
+}
+//# sourceMappingURL=definitions.d.mts.map

package/lib/opLifecycle/definitions.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"definitions.d.mts","sourceRoot":"","sources":["../../src/opLifecycle/definitions.ts"],"names":[],"mappings":"AAAA;;;GAGG;OAEI,EAAE,aAAa,EAAE,MAAM,uCAAuC;OAC9D,EAAE,yBAAyB,EAAE,WAAW,EAAE,MAAM,sCAAsC;OACtF,EAAE,qBAAqB,EAAE;OACzB,EAAE,oBAAoB,EAAE;AAE/B;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,aAAa,GAAG;IAC1C,eAAe,EAAE,OAAO,CAAC;IACzB,IAAI,EAAE,oBAAoB,CAAC;IAC3B,uBAAuB,EAAE,MAAM,CAAC;IAChC,WAAW,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,MAAM;IACtB;;;OAGG;IACH,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC;IACjC;;OAEG;IACH,QAAQ,CAAC,uBAAuB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrD;;;;;;;;;OASG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,KAAK,IAAI,CAAC;CAC5D;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,WAAW,GAAG,oBAAoB,CAAC;IACjD,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3C,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG,WAAW,GAAG,SAAS,GAAG,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,yBAAyB,CAAC;IAC5C;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;CAChC"}

package/lib/opLifecycle/definitions.mjs

@@ -0,0 +1,6 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+export {};
+//# sourceMappingURL=definitions.mjs.map

package/lib/opLifecycle/definitions.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"definitions.mjs","sourceRoot":"","sources":["../../src/opLifecycle/definitions.ts"],"names":[],"mappings":"AAAA;;;GAGG","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport { ISequencedDocumentMessage, MessageType } from \"@fluidframework/protocol-definitions\";\nimport { CompressionAlgorithms } from \"../containerRuntime\";\nimport { ContainerMessageType } from \"../messageTypes\";\n\n/**\n * Batch message type used internally by the runtime\n */\nexport type BatchMessage = IBatchMessage & {\n\tlocalOpMetadata: unknown;\n\ttype: ContainerMessageType;\n\treferenceSequenceNumber: number;\n\tcompression?: CompressionAlgorithms;\n};\n\n/**\n * Batch interface used internally by the runtime.\n */\nexport interface IBatch {\n\t/**\n\t * Sum of the in-memory content sizes of all messages in the batch.\n\t * If the batch is compressed, this number reflects the post-compression size.\n\t */\n\treadonly contentSizeInBytes: number;\n\t/**\n\t * All the messages in the batch\n\t */\n\treadonly content: BatchMessage[];\n\t/**\n\t * The reference sequence number for the batch\n\t */\n\treadonly referenceSequenceNumber: number | undefined;\n\t/**\n\t * Wether or not the batch contains at least one op which was produced as the result\n\t * of processing another op. This means that the batch must be rebased before\n\t * submitted, to ensure that all ops have the same reference sequence numbers and a\n\t * consistent view of the data model. This happens when the op is created within a\n\t * 'changed' event handler of a DDS and will have a different reference sequence number\n\t * than the rest of the ops in the batch, meaning that it has a different view of the\n\t * state of the data model, therefore all ops must be resubmitted and rebased to the current\n\t * reference sequence number to be in agreement about the data model state.\n\t */\n\treadonly hasReentrantOps?: boolean;\n}\n\nexport interface IBatchCheckpoint {\n\trollback: (action: (message: BatchMessage) => void) => void;\n}\n\n/**\n * @internal\n */\nexport interface IChunkedOp {\n\tchunkId: number;\n\ttotalChunks: number;\n\tcontents: string;\n\toriginalType: MessageType | ContainerMessageType;\n\toriginalMetadata?: Record<string, unknown>;\n\toriginalCompression?: string;\n}\n\n/**\n * The state of remote message processing:\n * `Processed` - the message can be considered processed\n * `Skipped` - the message was ignored by the processor\n * `Accepted` - the message was processed partially. Eventually, a message\n * will make the processor return `Processed`.\n */\nexport type ProcessingState = \"Processed\" | \"Skipped\" | \"Accepted\";\n\n/**\n * Return type for functions which process remote messages\n */\nexport interface IMessageProcessingResult {\n\t/**\n\t * A shallow copy of the input message if processing happened, or\n\t * the original message otherwise\n\t */\n\treadonly message: ISequencedDocumentMessage;\n\t/**\n\t * Processing result of the input message.\n\t */\n\treadonly state: ProcessingState;\n}\n"]}

package/lib/opLifecycle/index.d.mts

@@ -0,0 +1,13 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+export { BatchManager, estimateSocketSize, BatchSequenceNumbers } from "./batchManager.mjs";
+export { BatchMessage, IBatch, IBatchCheckpoint, IChunkedOp, IMessageProcessingResult, } from "./definitions.mjs";
+export { Outbox, getLongStack } from "./outbox.mjs";
+export { OpCompressor } from "./opCompressor.mjs";
+export { OpDecompressor } from "./opDecompressor.mjs";
+export { OpSplitter, splitOp } from "./opSplitter.mjs";
+export { RemoteMessageProcessor, unpackRuntimeMessage } from "./remoteMessageProcessor.mjs";
+export { OpGroupingManager } from "./opGroupingManager.mjs";
+//# sourceMappingURL=index.d.mts.map

package/lib/opLifecycle/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../../src/opLifecycle/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;OAEI,EAAE,YAAY,EAAE,kBAAkB,EAAE,oBAAoB,EAAE;OAC1D,EACN,YAAY,EACZ,MAAM,EACN,gBAAgB,EAChB,UAAU,EACV,wBAAwB,GACxB;OACM,EAAE,MAAM,EAAE,YAAY,EAAE;OACxB,EAAE,YAAY,EAAE;OAChB,EAAE,cAAc,EAAE;OAClB,EAAE,UAAU,EAAE,OAAO,EAAE;OACvB,EAAE,sBAAsB,EAAE,oBAAoB,EAAE;OAChD,EAAE,iBAAiB,EAAE"}

package/lib/opLifecycle/index.mjs

@@ -0,0 +1,12 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+export { BatchManager, estimateSocketSize } from "./batchManager.mjs";
+export { Outbox, getLongStack } from "./outbox.mjs";
+export { OpCompressor } from "./opCompressor.mjs";
+export { OpDecompressor } from "./opDecompressor.mjs";
+export { OpSplitter, splitOp } from "./opSplitter.mjs";
+export { RemoteMessageProcessor, unpackRuntimeMessage } from "./remoteMessageProcessor.mjs";
+export { OpGroupingManager } from "./opGroupingManager.mjs";
+//# sourceMappingURL=index.mjs.map

package/lib/opLifecycle/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../../src/opLifecycle/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;OAEI,EAAE,YAAY,EAAE,kBAAkB,EAAwB;OAQ1D,EAAE,MAAM,EAAE,YAAY,EAAE;OACxB,EAAE,YAAY,EAAE;OAChB,EAAE,cAAc,EAAE;OAClB,EAAE,UAAU,EAAE,OAAO,EAAE;OACvB,EAAE,sBAAsB,EAAE,oBAAoB,EAAE;OAChD,EAAE,iBAAiB,EAAE","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nexport { BatchManager, estimateSocketSize, BatchSequenceNumbers } from \"./batchManager\";\nexport {\n\tBatchMessage,\n\tIBatch,\n\tIBatchCheckpoint,\n\tIChunkedOp,\n\tIMessageProcessingResult,\n} from \"./definitions\";\nexport { Outbox, getLongStack } from \"./outbox\";\nexport { OpCompressor } from \"./opCompressor\";\nexport { OpDecompressor } from \"./opDecompressor\";\nexport { OpSplitter, splitOp } from \"./opSplitter\";\nexport { RemoteMessageProcessor, unpackRuntimeMessage } from \"./remoteMessageProcessor\";\nexport { OpGroupingManager } from \"./opGroupingManager\";\n"]}

package/lib/opLifecycle/opCompressor.d.mts

@@ -0,0 +1,18 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { IBatch } from "./definitions.mjs";
+/**
+ * Compresses batches of ops. It generates a single compressed op that contains
+ * the contents of each op in the batch. It then submits empty ops for each original
+ * op to reserve sequence numbers.
+ */
+export declare class OpCompressor {
+    private readonly logger;
+    constructor(logger: ITelemetryBaseLogger);
+    compressBatch(batch: IBatch): IBatch;
+    private serializeBatch;
+}
+//# sourceMappingURL=opCompressor.d.mts.map

package/lib/opLifecycle/opCompressor.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"opCompressor.d.mts","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;OAMI,EAAE,oBAAoB,EAAE,MAAM,iCAAiC;OAG/D,EAAE,MAAM,EAAgB;AAE/B;;;;GAIG;AACH,qBAAa,YAAY;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAEZ,MAAM,EAAE,oBAAoB;IAIjC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAkD3C,OAAO,CAAC,cAAc;CAsBtB"}

package/lib/opLifecycle/opCompressor.mjs

@@ -0,0 +1,80 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { createChildLogger, UsageError } from "@fluidframework/telemetry-utils";
+import { assert } from "@fluidframework/core-utils";
+import { IsoBuffer } from "@fluid-internal/client-utils";
+import { compress } from "lz4js";
+import { CompressionAlgorithms } from "../containerRuntime.mjs";
+import { estimateSocketSize } from "./batchManager.mjs";
+/**
+ * Compresses batches of ops. It generates a single compressed op that contains
+ * the contents of each op in the batch. It then submits empty ops for each original
+ * op to reserve sequence numbers.
+ */
+export class OpCompressor {
+    constructor(logger) {
+        this.logger = createChildLogger({ logger, namespace: "OpCompressor" });
+    }
+    compressBatch(batch) {
+        assert(batch.contentSizeInBytes > 0 && batch.content.length > 0, 0x5a4 /* Batch should not be empty */);
+        const compressionStart = Date.now();
+        const contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));
+        const compressedContents = compress(contentsAsBuffer);
+        const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
+        const duration = Date.now() - compressionStart;
+        const messages = [];
+        messages.push({
+            ...batch.content[0],
+            contents: JSON.stringify({ packedContents: compressedContent }),
+            metadata: batch.content[0].metadata,
+            compression: CompressionAlgorithms.lz4,
+        });
+        // Add empty placeholder messages to reserve the sequence numbers
+        for (const message of batch.content.slice(1)) {
+            messages.push({
+                type: message.type,
+                localOpMetadata: message.localOpMetadata,
+                metadata: message.metadata,
+                referenceSequenceNumber: message.referenceSequenceNumber,
+            });
+        }
+        const compressedBatch = {
+            contentSizeInBytes: compressedContent.length,
+            content: messages,
+            referenceSequenceNumber: batch.referenceSequenceNumber,
+        };
+        if (batch.contentSizeInBytes > 200000) {
+            this.logger.sendPerformanceEvent({
+                eventName: "CompressedBatch",
+                duration,
+                sizeBeforeCompression: batch.contentSizeInBytes,
+                sizeAfterCompression: compressedBatch.contentSizeInBytes,
+                opCount: compressedBatch.content.length,
+                socketSize: estimateSocketSize(compressedBatch),
+            });
+        }
+        return compressedBatch;
+    }
+    serializeBatch(batch) {
+        try {
+            return `[${batch.content.map((message) => message.contents).join(",")}]`;
+        }
+        catch (e) {
+            if (e.message === "Invalid string length") {
+                // This is how JSON.stringify signals that
+                // the content size exceeds its capacity
+                const error = new UsageError("Payload too large");
+                this.logger.sendErrorEvent({
+                    eventName: "BatchTooLarge",
+                    size: batch.contentSizeInBytes,
+                    length: batch.content.length,
+                }, error);
+                throw error;
+            }
+            throw e;
+        }
+    }
+}
+//# sourceMappingURL=opCompressor.mjs.map

package/lib/opLifecycle/opCompressor.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"opCompressor.mjs","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;OAEI,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,iCAAiC;OACxE,EAAE,MAAM,EAAE,MAAM,4BAA4B;OAC5C,EAAE,SAAS,EAAE,MAAM,8BAA8B;OACjD,EAAE,QAAQ,EAAE,MAAM,OAAO;OAEzB,EAAE,qBAAqB,EAAE;OACzB,EAAE,kBAAkB,EAAE;AAG7B;;;;GAIG;AACH,MAAM,OAAO,YAAY;IAGxB,YAAY,MAA4B;QACvC,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,CAAC;IACxE,CAAC;IAEM,aAAa,CAAC,KAAa;QACjC,MAAM,CACL,KAAK,CAAC,kBAAkB,GAAG,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EACxD,KAAK,CAAC,+BAA+B,CACrC,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9E,MAAM,kBAAkB,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;QAE/C,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,QAAQ,CAAC,IAAI,CAAC;YACb,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YACnB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC;YAC/D,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ;YACnC,WAAW,EAAE,qBAAqB,CAAC,GAAG;SACtC,CAAC,CAAC;QAEH,iEAAiE;QACjE,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACb,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,eAAe,EAAE,OAAO,CAAC,eAAe;gBACxC,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;aACxD,CAAC,CAAC;SACH;QAED,MAAM,eAAe,GAAW;YAC/B,kBAAkB,EAAE,iBAAiB,CAAC,MAAM;YAC5C,OAAO,EAAE,QAAQ;YACjB,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;SACtD,CAAC;QAEF,IAAI,KAAK,CAAC,kBAAkB,GAAG,MAAM,EAAE;YACtC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAChC,SAAS,EAAE,iBAAiB;gBAC5B,QAAQ;gBACR,qBAAqB,EAAE,KAAK,CAAC,kBAAkB;gBAC/C,oBAAoB,EAAE,eAAe,CAAC,kBAAkB;gBACxD,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,MAAM;gBACvC,UAAU,EAAE,kBAAkB,CAAC,eAAe,CAAC;aAC/C,CAAC,CAAC;SACH;QAED,OAAO,eAAe,CAAC;IACxB,CAAC;IAEO,cAAc,CAAC,KAAa;QACnC,IAAI;YACH,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;SACzE;QAAC,OAAO,CAAM,EAAE;YAChB,IAAI,CAAC,CAAC,OAAO,KAAK,uBAAuB,EAAE;gBAC1C,0CAA0C;gBAC1C,wCAAwC;gBACxC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,mBAAmB,CAAC,CAAC;gBAClD,IAAI,CAAC,MAAM,CAAC,cAAc,CACzB;oBACC,SAAS,EAAE,eAAe;oBAC1B,IAAI,EAAE,KAAK,CAAC,kBAAkB;oBAC9B,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;iBAC5B,EACD,KAAK,CACL,CAAC;gBACF,MAAM,KAAK,CAAC;aACZ;YAED,MAAM,CAAC,CAAC;SACR;IACF,CAAC;CACD","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { createChildLogger, UsageError } from \"@fluidframework/telemetry-utils\";\nimport { assert } from \"@fluidframework/core-utils\";\nimport { IsoBuffer } from \"@fluid-internal/client-utils\";\nimport { compress } from \"lz4js\";\nimport { ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport { CompressionAlgorithms } from \"../containerRuntime\";\nimport { estimateSocketSize } from \"./batchManager\";\nimport { IBatch, BatchMessage } from \"./definitions\";\n\n/**\n * Compresses batches of ops. It generates a single compressed op that contains\n * the contents of each op in the batch. It then submits empty ops for each original\n * op to reserve sequence numbers.\n */\nexport class OpCompressor {\n\tprivate readonly logger;\n\n\tconstructor(logger: ITelemetryBaseLogger) {\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpCompressor\" });\n\t}\n\n\tpublic compressBatch(batch: IBatch): IBatch {\n\t\tassert(\n\t\t\tbatch.contentSizeInBytes > 0 && batch.content.length > 0,\n\t\t\t0x5a4 /* Batch should not be empty */,\n\t\t);\n\n\t\tconst compressionStart = Date.now();\n\t\tconst contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));\n\t\tconst compressedContents = compress(contentsAsBuffer);\n\t\tconst compressedContent = IsoBuffer.from(compressedContents).toString(\"base64\");\n\t\tconst duration = Date.now() - compressionStart;\n\n\t\tconst messages: BatchMessage[] = [];\n\t\tmessages.push({\n\t\t\t...batch.content[0],\n\t\t\tcontents: JSON.stringify({ packedContents: compressedContent }),\n\t\t\tmetadata: batch.content[0].metadata,\n\t\t\tcompression: CompressionAlgorithms.lz4,\n\t\t});\n\n\t\t// Add empty placeholder messages to reserve the sequence numbers\n\t\tfor (const message of batch.content.slice(1)) {\n\t\t\tmessages.push({\n\t\t\t\ttype: message.type,\n\t\t\t\tlocalOpMetadata: message.localOpMetadata,\n\t\t\t\tmetadata: message.metadata,\n\t\t\t\treferenceSequenceNumber: message.referenceSequenceNumber,\n\t\t\t});\n\t\t}\n\n\t\tconst compressedBatch: IBatch = {\n\t\t\tcontentSizeInBytes: compressedContent.length,\n\t\t\tcontent: messages,\n\t\t\treferenceSequenceNumber: batch.referenceSequenceNumber,\n\t\t};\n\n\t\tif (batch.contentSizeInBytes > 200000) {\n\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\teventName: \"CompressedBatch\",\n\t\t\t\tduration,\n\t\t\t\tsizeBeforeCompression: batch.contentSizeInBytes,\n\t\t\t\tsizeAfterCompression: compressedBatch.contentSizeInBytes,\n\t\t\t\topCount: compressedBatch.content.length,\n\t\t\t\tsocketSize: estimateSocketSize(compressedBatch),\n\t\t\t});\n\t\t}\n\n\t\treturn compressedBatch;\n\t}\n\n\tprivate serializeBatch(batch: IBatch): string {\n\t\ttry {\n\t\t\treturn `[${batch.content.map((message) => message.contents).join(\",\")}]`;\n\t\t} catch (e: any) {\n\t\t\tif (e.message === \"Invalid string length\") {\n\t\t\t\t// This is how JSON.stringify signals that\n\t\t\t\t// the content size exceeds its capacity\n\t\t\t\tconst error = new UsageError(\"Payload too large\");\n\t\t\t\tthis.logger.sendErrorEvent(\n\t\t\t\t\t{\n\t\t\t\t\t\teventName: \"BatchTooLarge\",\n\t\t\t\t\t\tsize: batch.contentSizeInBytes,\n\t\t\t\t\t\tlength: batch.content.length,\n\t\t\t\t\t},\n\t\t\t\t\terror,\n\t\t\t\t);\n\t\t\t\tthrow error;\n\t\t\t}\n\n\t\t\tthrow e;\n\t\t}\n\t}\n}\n"]}

package/lib/opLifecycle/opDecompressor.d.mts

@@ -0,0 +1,25 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
+import { IMessageProcessingResult } from "./definitions.mjs";
+/**
+ * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
+ * This class relies on some implicit contracts defined below:
+ * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true
+ * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
+ * 3. The final message of a batch will have batch metadata set to false
+ * 4. An individually compressed op will have undefined batch metadata and compression set to true
+ */
+export declare class OpDecompressor {
+    private activeBatch;
+    private rootMessageContents;
+    private processedCount;
+    private readonly logger;
+    constructor(logger: ITelemetryBaseLogger);
+    processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult;
+    private isCompressed;
+}
+//# sourceMappingURL=opDecompressor.d.mts.map

package/lib/opLifecycle/opDecompressor.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"opDecompressor.d.mts","sourceRoot":"","sources":["../../src/opLifecycle/opDecompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;OAGI,EAAE,yBAAyB,EAAE,MAAM,sCAAsC;OAIzE,EAAE,oBAAoB,EAAE,MAAM,iCAAiC;OAG/D,EAAE,wBAAwB,EAAE;AASnC;;;;;;;GAOG;AACH,qBAAa,cAAc;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,mBAAmB,CAAkB;IAC7C,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAEZ,MAAM,EAAE,oBAAoB;IAIjC,cAAc,CAAC,OAAO,EAAE,yBAAyB,GAAG,wBAAwB;IA6FnF,OAAO,CAAC,YAAY;CA2CpB"}

package/lib/opLifecycle/opDecompressor.mjs

@@ -0,0 +1,128 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { decompress } from "lz4js";
+import { assert } from "@fluidframework/core-utils";
+import { IsoBuffer, Uint8ArrayToString } from "@fluid-internal/client-utils";
+import { createChildLogger } from "@fluidframework/telemetry-utils";
+import { CompressionAlgorithms } from "../containerRuntime.mjs";
+/**
+ * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
+ * This class relies on some implicit contracts defined below:
+ * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true
+ * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
+ * 3. The final message of a batch will have batch metadata set to false
+ * 4. An individually compressed op will have undefined batch metadata and compression set to true
+ */
+export class OpDecompressor {
+    constructor(logger) {
+        this.activeBatch = false;
+        this.processedCount = 0;
+        this.logger = createChildLogger({ logger, namespace: "OpDecompressor" });
+    }
+    processMessage(message) {
+        assert(message.compression === undefined || message.compression === CompressionAlgorithms.lz4, 0x511 /* Only lz4 compression is supported */);
+        if (message.metadata?.batch === true &&
+            this.isCompressed(message)) {
+            // Beginning of a compressed batch
+            assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
+            this.activeBatch = true;
+            const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+            const decompressedMessage = decompress(contents);
+            const intoString = Uint8ArrayToString(decompressedMessage);
+            const asObj = JSON.parse(intoString);
+            this.rootMessageContents = asObj;
+            return {
+                message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+                state: "Accepted",
+            };
+        }
+        if (this.rootMessageContents !== undefined &&
+            message.metadata?.batch === undefined &&
+            this.activeBatch) {
+            assert(message.contents === undefined, 0x512 /* Expecting empty message */);
+            // Continuation of compressed batch
+            return {
+                message: newMessage(message, this.rootMessageContents[this.processedCount++]),
+                state: "Accepted",
+            };
+        }
+        if (this.rootMessageContents !== undefined &&
+            message.metadata?.batch === false) {
+            // End of compressed batch
+            const returnMessage = newMessage(message, this.rootMessageContents[this.processedCount++]);
+            this.activeBatch = false;
+            this.rootMessageContents = undefined;
+            this.processedCount = 0;
+            return {
+                message: returnMessage,
+                state: "Processed",
+            };
+        }
+        if (message.metadata?.batch === undefined &&
+            this.isCompressed(message)) {
+            // Single compressed message
+            assert(this.activeBatch === false, 0x4ba /* shouldn't receive compressed message in middle of a batch */);
+            const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+            const decompressedMessage = decompress(contents);
+            const intoString = new TextDecoder().decode(decompressedMessage);
+            const asObj = JSON.parse(intoString);
+            return {
+                message: newMessage(message, asObj[0]),
+                state: "Processed",
+            };
+        }
+        return {
+            message,
+            state: "Skipped",
+        };
+    }
+    isCompressed(message) {
+        if (message.compression === CompressionAlgorithms.lz4) {
+            return true;
+        }
+        /**
+         * Back-compat self healing mechanism for ADO:3538, as loaders from
+         * version client_v2.0.0-internal.1.2.0 to client_v2.0.0-internal.2.2.0 do not
+         * support adding the proper compression metadata to compressed messages submitted
+         * by the runtime. Should be removed after the loader reaches sufficient saturation
+         * for a version greater or equal than client_v2.0.0-internal.2.2.0.
+         *
+         * The condition holds true for compressed messages, regardless of metadata. We are ultimately
+         * looking for a message with a single property `packedContents` inside `contents`, of type 'string'
+         * with a base64 encoded value.
+         */
+        try {
+            if (message.contents !== null &&
+                typeof message.contents === "object" &&
+                Object.keys(message.contents).length === 1 &&
+                typeof message.contents.packedContents ===
+                    "string" &&
+                message.contents.packedContents.length > 0 &&
+                IsoBuffer.from(message.contents.packedContents, "base64").toString("base64") ===
+                    message.contents.packedContents) {
+                this.logger.sendTelemetryEvent({
+                    eventName: "LegacyCompression",
+                    type: message.type,
+                    batch: message.metadata?.batch,
+                });
+                return true;
+            }
+        }
+        catch (err) {
+            return false;
+        }
+        return false;
+    }
+}
+// We should not be mutating the input message nor its metadata
+const newMessage = (originalMessage, contents) => ({
+    ...originalMessage,
+    contents,
+    compression: undefined,
+    // TODO: It should already be the case that we're not modifying any metadata, not clear if/why this shallow clone should be required.
+    // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+    metadata: { ...originalMessage.metadata },
+});
+//# sourceMappingURL=opDecompressor.mjs.map