@fluidframework/container-runtime 2.0.0-dev.2.2.0.111723 → 2.0.0-dev.3.1.0.125672

package/lib/batchManager.js

@@ -1,79 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-import { OpCompressor } from "./opCompressor";
-/**
- * Helper class that manages partial batch & rollback.
- */
-export class BatchManager {
-    constructor(logger, options) {
-        this.logger = logger;
-        this.options = options;
-        this.pendingBatch = [];
-        this.batchContentSize = 0;
-        this.opCompressor = new OpCompressor(logger);
-    }
-    get length() { return this.pendingBatch.length; }
-    push(message) {
-        var _a, _b, _c, _d, _e, _f;
-        const contentSize = this.batchContentSize + ((_b = (_a = message.contents) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0);
-        const opCount = this.pendingBatch.length;
-        // Attempt to estimate batch size, aka socket message size.
-        // Each op has pretty large envelope, estimating to be 200 bytes.
-        // Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.
-        // Not taking it into account, as compression work should help there - compressed payload will be
-        // initially stored as base64, and that requires only 2 extra escape characters.
-        const socketMessageSize = contentSize + 200 * opCount;
-        // If we were provided soft limit, check for exceeding it.
-        // But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)
-        // and start over. That's not an option if we have no ops.
-        // If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.
-        // Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.
-        if (this.options.softLimit !== undefined
-            && this.length > 0
-            && socketMessageSize >= this.options.softLimit
-            && Infinity === ((_d = (_c = this.options.compressionOptions) === null || _c === void 0 ? void 0 : _c.minimumBatchSizeInBytes) !== null && _d !== void 0 ? _d : Infinity)) {
-            return false;
-        }
-        if (socketMessageSize >= this.options.hardLimit
-            && Infinity === ((_f = (_e = this.options.compressionOptions) === null || _e === void 0 ? void 0 : _e.minimumBatchSizeInBytes) !== null && _f !== void 0 ? _f : Infinity)) {
-            return false;
-        }
-        this.batchContentSize = contentSize;
-        this.pendingBatch.push(message);
-        return true;
-    }
-    get empty() { return this.pendingBatch.length === 0; }
-    popBatch() {
-        const batch = this.pendingBatch;
-        const size = this.batchContentSize;
-        this.pendingBatch = [];
-        this.batchContentSize = 0;
-        if (batch.length > 0
-            && this.options.compressionOptions !== undefined
-            && this.options.compressionOptions.minimumBatchSizeInBytes < size) {
-            return this.opCompressor.compressBatch(batch, size);
-        }
-        return batch;
-    }
-    /**
-     * Capture the pending state at this point
-     */
-    checkpoint() {
-        const startPoint = this.pendingBatch.length;
-        return {
-            rollback: (process) => {
-                var _a, _b;
-                for (let i = this.pendingBatch.length; i > startPoint;) {
-                    i--;
-                    const message = this.pendingBatch[i];
-                    this.batchContentSize -= (_b = (_a = message.contents) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0;
-                    process(message);
-                }
-                this.pendingBatch.length = startPoint;
-            },
-        };
-    }
-}
-//# sourceMappingURL=batchManager.js.map

package/lib/batchManager.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"batchManager.js","sourceRoot":"","sources":["../src/batchManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAiB9C;;GAEG;AACH,MAAM,OAAO,YAAY;IAOrB,YAA4B,MAAwB,EAAkB,OAA6B;QAAvE,WAAM,GAAN,MAAM,CAAkB;QAAkB,YAAO,GAAP,OAAO,CAAsB;QAL3F,iBAAY,GAAoB,EAAE,CAAC;QACnC,qBAAgB,GAAG,CAAC,CAAC;QAKzB,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IAJD,IAAW,MAAM,KAAK,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IAMjD,IAAI,CAAC,OAAqB;;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,GAAG,CAAC,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC,CAAC;QAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAEzC,2DAA2D;QAC3D,iEAAiE;QACjE,sGAAsG;QACtG,iGAAiG;QACjG,gFAAgF;QAChF,MAAM,iBAAiB,GAAG,WAAW,GAAG,GAAG,GAAG,OAAO,CAAC;QAEtD,0DAA0D;QAC1D,wGAAwG;QACxG,0DAA0D;QAC1D,yGAAyG;QACzG,mGAAmG;QACnG,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS;eACjC,IAAI,CAAC,MAAM,GAAG,CAAC;eACf,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS;eAC3C,QAAQ,KAAK,CAAC,MAAA,MAAA,IAAI,CAAC,OAAO,CAAC,kBAAkB,0CAAE,uBAAuB,mCAAI,QAAQ,CAAC,EAAE;YACxF,OAAO,KAAK,CAAC;SAChB;QAED,IAAI,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS;eACxC,QAAQ,KAAK,CAAC,MAAA,MAAA,IAAI,CAAC,OAAO,CAAC,kBAAkB,0CAAE,uBAAuB,mCAAI,QAAQ,CAAC,EAAE;YACxF,OAAO,KAAK,CAAC;SAChB;QAED,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC;QACpC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,IAAW,KAAK,KAAK,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC;IAEtD,QAAQ;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAE1B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;eACb,IAAI,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS;eAC7C,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,uBAAuB,GAAG,IAAI,EAAE;YACnE,OAAO,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;SACvD;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACI,UAAU;QACb,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5C,OAAO;YACH,QAAQ,EAAE,CAAC,OAAwC,EAAE,EAAE;;gBACnD,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,GAAG,UAAU,GAAG;oBACpD,CAAC,EAAE,CAAC;oBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBACrC,IAAI,CAAC,gBAAgB,IAAI,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC;oBACvD,OAAO,CAAC,OAAO,CAAC,CAAC;iBACpB;gBAED,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU,CAAC;YAC1C,CAAC;SACJ,CAAC;IACN,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport { ContainerRuntimeMessage, ICompressionRuntimeOptions } from \"./containerRuntime\";\nimport { OpCompressor } from \"./opCompressor\";\n\n/**\n * Message type used by BatchManager\n */\nexport type BatchMessage = IBatchMessage & {\n    localOpMetadata: unknown;\n    deserializedContent: ContainerRuntimeMessage;\n    referenceSequenceNumber: number;\n};\n\nexport interface IBatchManagerOptions {\n    readonly hardLimit: number;\n    readonly softLimit?: number;\n    readonly compressionOptions?: ICompressionRuntimeOptions;\n}\n\n/**\n * Helper class that manages partial batch & rollback.\n */\nexport class BatchManager {\n    private readonly opCompressor: OpCompressor;\n    private pendingBatch: BatchMessage [] = [];\n    private batchContentSize = 0;\n\n    public get length() { return this.pendingBatch.length; }\n\n    constructor(public readonly logger: ITelemetryLogger, public readonly options: IBatchManagerOptions) {\n        this.opCompressor = new OpCompressor(logger);\n    }\n\n    public push(message: BatchMessage): boolean {\n        const contentSize = this.batchContentSize + (message.contents?.length ?? 0);\n        const opCount = this.pendingBatch.length;\n\n        // Attempt to estimate batch size, aka socket message size.\n        // Each op has pretty large envelope, estimating to be 200 bytes.\n        // Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.\n        // Not taking it into account, as compression work should help there - compressed payload will be\n        // initially stored as base64, and that requires only 2 extra escape characters.\n        const socketMessageSize = contentSize + 200 * opCount;\n\n        // If we were provided soft limit, check for exceeding it.\n        // But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)\n        // and start over. That's not an option if we have no ops.\n        // If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.\n        // Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.\n        if (this.options.softLimit !== undefined\n            && this.length > 0\n            && socketMessageSize >= this.options.softLimit\n            && Infinity === (this.options.compressionOptions?.minimumBatchSizeInBytes ?? Infinity)) {\n            return false;\n        }\n\n        if (socketMessageSize >= this.options.hardLimit\n            && Infinity === (this.options.compressionOptions?.minimumBatchSizeInBytes ?? Infinity)) {\n            return false;\n        }\n\n        this.batchContentSize = contentSize;\n        this.pendingBatch.push(message);\n        return true;\n    }\n\n    public get empty() { return this.pendingBatch.length === 0; }\n\n    public popBatch() {\n        const batch = this.pendingBatch;\n        const size = this.batchContentSize;\n        this.pendingBatch = [];\n        this.batchContentSize = 0;\n\n        if (batch.length > 0\n            && this.options.compressionOptions !== undefined\n            && this.options.compressionOptions.minimumBatchSizeInBytes < size) {\n            return this.opCompressor.compressBatch(batch, size);\n        }\n\n        return batch;\n    }\n\n    /**\n     * Capture the pending state at this point\n     */\n    public checkpoint() {\n        const startPoint = this.pendingBatch.length;\n        return {\n            rollback: (process: (message: BatchMessage) => void) => {\n                for (let i = this.pendingBatch.length; i > startPoint;) {\n                    i--;\n                    const message = this.pendingBatch[i];\n                    this.batchContentSize -= message.contents?.length ?? 0;\n                    process(message);\n                }\n\n                this.pendingBatch.length = startPoint;\n            },\n        };\n    }\n}\n"]}

package/lib/opCompressor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"opCompressor.d.ts","sourceRoot":"","sources":["../src/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAItE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C;;;;GAIG;AACH,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IACxB,OAAO,CAAC,oBAAoB,CAAK;gBAErB,MAAM,EAAE,gBAAgB;IAI7B,aAAa,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,EAAE,MAAM,GAAG,YAAY,EAAE;CAiCtF"}

package/lib/opCompressor.js

@@ -1,46 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-import { IsoBuffer } from "@fluidframework/common-utils";
-import { ChildLogger } from "@fluidframework/telemetry-utils";
-import { compress } from "lz4js";
-import { CompressionAlgorithms } from "./containerRuntime";
-/**
- * Compresses batches of ops. It generates a single compressed op that contains
- * the contents of each op in the batch. It then submits empty ops for each original
- * op to reserve sequence numbers.
- */
-export class OpCompressor {
-    constructor(logger) {
-        this.compressedBatchCount = 0;
-        this.logger = ChildLogger.create(logger, "OpCompressor");
-    }
-    compressBatch(batch, originalLength) {
-        const batchToSend = [];
-        this.compressedBatchCount++;
-        const batchedContents = [];
-        for (const message of batch) {
-            batchedContents.push(message.deserializedContent);
-        }
-        const compressionStart = Date.now();
-        const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(batchedContents));
-        const compressedContents = compress(contentsAsBuffer);
-        const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
-        const duration = Date.now() - compressionStart;
-        if (originalLength > 200000 || this.compressedBatchCount % 100) {
-            this.logger.sendPerformanceEvent({
-                eventName: "CompressedBatch",
-                duration,
-                sizeBeforeCompression: originalLength,
-                sizeAfterCompression: compressedContent.length,
-            });
-        }
-        batchToSend.push(Object.assign(Object.assign({}, batch[0]), { contents: JSON.stringify({ packedContents: compressedContent }), metadata: Object.assign(Object.assign({}, batch[0].metadata), { compressed: true }), compression: CompressionAlgorithms.lz4 }));
-        for (const message of batch.slice(1)) {
-            batchToSend.push(Object.assign(Object.assign({}, message), { contents: undefined }));
-        }
-        return batchToSend;
-    }
-}
-//# sourceMappingURL=opCompressor.js.map

package/lib/opCompressor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"opCompressor.js","sourceRoot":"","sources":["../src/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AAEjC,OAAO,EAAE,qBAAqB,EAA2B,MAAM,oBAAoB,CAAC;AAEpF;;;;GAIG;AACH,MAAM,OAAO,YAAY;IAIrB,YAAY,MAAwB;QAF5B,yBAAoB,GAAG,CAAC,CAAC;QAG7B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC7D,CAAC;IAEM,aAAa,CAAC,KAAqB,EAAE,cAAsB;QAC9D,MAAM,WAAW,GAAmB,EAAE,CAAC;QACvC,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5B,MAAM,eAAe,GAA8B,EAAE,CAAC;QACtD,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE;YACzB,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;SACrD;QAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC;QACnF,MAAM,kBAAkB,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;QAE/C,IAAI,cAAc,GAAG,MAAM,IAAI,IAAI,CAAC,oBAAoB,GAAG,GAAG,EAAE;YAC5D,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAC7B,SAAS,EAAE,iBAAiB;gBAC5B,QAAQ;gBACR,qBAAqB,EAAE,cAAc;gBACrC,oBAAoB,EAAE,iBAAiB,CAAC,MAAM;aACjD,CAAC,CAAC;SACN;QAED,WAAW,CAAC,IAAI,iCAAM,KAAK,CAAC,CAAC,CAAC,KAAE,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC,EAC5E,QAAQ,kCAAO,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAE,UAAU,EAAE,IAAI,KAClD,WAAW,EAAE,qBAAqB,CAAC,GAAG,IAAG,CAAC;QAE7D,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAClC,WAAW,CAAC,IAAI,iCAAM,OAAO,KAAE,QAAQ,EAAE,SAAS,IAAG,CAAC;SACzD;QAED,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { IsoBuffer } from \"@fluidframework/common-utils\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { compress } from \"lz4js\";\nimport { BatchMessage } from \"./batchManager\";\nimport { CompressionAlgorithms, ContainerRuntimeMessage } from \"./containerRuntime\";\n\n/**\n * Compresses batches of ops. It generates a single compressed op that contains\n * the contents of each op in the batch. It then submits empty ops for each original\n * op to reserve sequence numbers.\n */\nexport class OpCompressor {\n    private readonly logger;\n    private compressedBatchCount = 0;\n\n    constructor(logger: ITelemetryLogger) {\n        this.logger = ChildLogger.create(logger, \"OpCompressor\");\n    }\n\n    public compressBatch(batch: BatchMessage[], originalLength: number): BatchMessage[] {\n        const batchToSend: BatchMessage[] = [];\n        this.compressedBatchCount++;\n        const batchedContents: ContainerRuntimeMessage[] = [];\n        for (const message of batch) {\n            batchedContents.push(message.deserializedContent);\n        }\n\n        const compressionStart = Date.now();\n        const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(batchedContents));\n        const compressedContents = compress(contentsAsBuffer);\n        const compressedContent = IsoBuffer.from(compressedContents).toString(\"base64\");\n        const duration = Date.now() - compressionStart;\n\n        if (originalLength > 200000 || this.compressedBatchCount % 100) {\n            this.logger.sendPerformanceEvent({\n                eventName: \"CompressedBatch\",\n                duration,\n                sizeBeforeCompression: originalLength,\n                sizeAfterCompression: compressedContent.length,\n            });\n        }\n\n        batchToSend.push({ ...batch[0], contents: JSON.stringify({ packedContents: compressedContent }),\n                           metadata: { ...batch[0].metadata, compressed: true },\n                           compression: CompressionAlgorithms.lz4 });\n\n        for (const message of batch.slice(1)) {\n            batchToSend.push({ ...message, contents: undefined });\n        }\n\n        return batchToSend;\n    }\n}\n"]}

package/lib/opDecompressor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"opDecompressor.d.ts","sourceRoot":"","sources":["../src/opDecompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAIjF;;;;;;;GAOG;AACH,qBAAa,cAAc;IACvB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,mBAAmB,CAAkB;IAC7C,OAAO,CAAC,cAAc,CAAK;IAEpB,cAAc,CAAC,OAAO,EAAE,yBAAyB,GAAG,yBAAyB;CA0DvF"}

package/lib/opDecompressor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"opDecompressor.js","sourceRoot":"","sources":["../src/opDecompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AAEnC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AACrF,OAAO,EAAE,qBAAqB,EAAE,MAAM,GAAG,CAAC;AAE1C;;;;;;;GAOG;AACH,MAAM,OAAO,cAAc;IAA3B;QACY,gBAAW,GAAG,KAAK,CAAC;QAEpB,mBAAc,GAAG,CAAC,CAAC;IA4D/B,CAAC;IA1DU,cAAc,CAAC,OAAkC;;QACpD,6EAA6E;QAC7E,0EAA0E;QAC1E,4EAA4E;QAC5E,WAAW;QACX,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,IAAI;eAC7B,CAAC,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,UAAU,KAAI,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,EAAE;YACxE,kCAAkC;YAClC,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,wCAAwC,CAAC,CAAC;YAC7E,IAAI,OAAO,CAAC,WAAW,EAAE;gBACrB,0DAA0D;gBAC1D,MAAM,CAAC,OAAO,CAAC,WAAW,KAAK,qBAAqB,CAAC,GAAG,EAChD,2DAA2D,CAAC,CAAC;aACxE;YAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YAExB,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YAC3E,MAAM,mBAAmB,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;YAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACrC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;YAEjC,uCAAY,OAAO,KAAE,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,IAAG;SACpF;QAED,IAAI,IAAI,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,SAAS,IAAI,IAAI,CAAC,WAAW,EAAE;YACrG,mCAAmC;YACnC,uCAAY,OAAO,KAAE,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,IAAG;SACpF;QAED,IAAI,IAAI,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,KAAK,EAAE;YAC7E,0BAA0B;YAC1B,MAAM,aAAa,mCAAQ,OAAO,KACV,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,GAAE,CAAC;YAEpF,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,IAAI,CAAC,mBAAmB,GAAG,SAAS,CAAC;YACrC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;YAExB,OAAO,aAAa,CAAC;SACxB;QAED,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,SAAS;YACrC,CAAC,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,UAAU,KAAI,OAAO,CAAC,WAAW,KAAK,qBAAqB,CAAC,GAAG,CAAC,EAAE;YACrF,4BAA4B;YAC5B,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,2DAA2D,CAAC,CAAC;YAEhG,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YAC3E,MAAM,mBAAmB,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAErC,uCAAY,OAAO,KAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,IAAG;SAC7C;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { decompress } from \"lz4js\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { assert, IsoBuffer, Uint8ArrayToString } from \"@fluidframework/common-utils\";\nimport { CompressionAlgorithms } from \".\";\n\n/**\n * State machine that \"unrolls\" contents of compressed batches of ops after decompressing them.\n * This class relies on some implicit contracts defined below:\n * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true\n * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set\n * 3. The final message of a batch will have batch metadata set to false\n * 4. An individually compressed op will have undefined batch metadata and compression set to true\n */\nexport class OpDecompressor {\n    private activeBatch = false;\n    private rootMessageContents: any | undefined;\n    private processedCount = 0;\n\n    public processMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {\n        // We're checking for compression = true or top level compression property so\n        // that we can enable compression without waiting on all ordering services\n        // to pick up protocol change. Eventually only the top level property should\n        // be used.\n        if (message.metadata?.batch === true\n            && (message.metadata?.compressed || message.compression !== undefined)) {\n            // Beginning of a compressed batch\n            assert(this.activeBatch === false, \"shouldn't have multiple active batches\");\n            if (message.compression) {\n                // lz4 is the only supported compression algorithm for now\n                assert(message.compression === CompressionAlgorithms.lz4,\n                        \"lz4 is currently the only supported compression algorithm\");\n            }\n\n            this.activeBatch = true;\n\n            const contents = IsoBuffer.from(message.contents.packedContents, \"base64\");\n            const decompressedMessage = decompress(contents);\n            const intoString = Uint8ArrayToString(decompressedMessage);\n            const asObj = JSON.parse(intoString);\n            this.rootMessageContents = asObj;\n\n            return { ...message, contents: this.rootMessageContents[this.processedCount++] };\n        }\n\n        if (this.rootMessageContents !== undefined && message.metadata?.batch === undefined && this.activeBatch) {\n            // Continuation of compressed batch\n            return { ...message, contents: this.rootMessageContents[this.processedCount++] };\n        }\n\n        if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {\n            // End of compressed batch\n            const returnMessage = { ...message,\n                                    contents: this.rootMessageContents[this.processedCount++] };\n\n            this.activeBatch = false;\n            this.rootMessageContents = undefined;\n            this.processedCount = 0;\n\n            return returnMessage;\n        }\n\n        if (message.metadata?.batch === undefined &&\n            (message.metadata?.compressed || message.compression === CompressionAlgorithms.lz4)) {\n            // Single compressed message\n            assert(this.activeBatch === false, \"shouldn't receive compressed message in middle of a batch\");\n\n            const contents = IsoBuffer.from(message.contents.packedContents, \"base64\");\n            const decompressedMessage = decompress(contents);\n            const intoString = new TextDecoder().decode(decompressedMessage);\n            const asObj = JSON.parse(intoString);\n\n            return { ...message, contents: asObj[0] };\n        }\n\n        return message;\n    }\n}\n"]}

package/src/batchManager.ts

@@ -1,108 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { IBatchMessage } from "@fluidframework/container-definitions";
-import { ContainerRuntimeMessage, ICompressionRuntimeOptions } from "./containerRuntime";
-import { OpCompressor } from "./opCompressor";
-
-/**
- * Message type used by BatchManager
- */
-export type BatchMessage = IBatchMessage & {
-    localOpMetadata: unknown;
-    deserializedContent: ContainerRuntimeMessage;
-    referenceSequenceNumber: number;
-};
-
-export interface IBatchManagerOptions {
-    readonly hardLimit: number;
-    readonly softLimit?: number;
-    readonly compressionOptions?: ICompressionRuntimeOptions;
-}
-
-/**
- * Helper class that manages partial batch & rollback.
- */
-export class BatchManager {
-    private readonly opCompressor: OpCompressor;
-    private pendingBatch: BatchMessage [] = [];
-    private batchContentSize = 0;
-
-    public get length() { return this.pendingBatch.length; }
-
-    constructor(public readonly logger: ITelemetryLogger, public readonly options: IBatchManagerOptions) {
-        this.opCompressor = new OpCompressor(logger);
-    }
-
-    public push(message: BatchMessage): boolean {
-        const contentSize = this.batchContentSize + (message.contents?.length ?? 0);
-        const opCount = this.pendingBatch.length;
-
-        // Attempt to estimate batch size, aka socket message size.
-        // Each op has pretty large envelope, estimating to be 200 bytes.
-        // Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.
-        // Not taking it into account, as compression work should help there - compressed payload will be
-        // initially stored as base64, and that requires only 2 extra escape characters.
-        const socketMessageSize = contentSize + 200 * opCount;
-
-        // If we were provided soft limit, check for exceeding it.
-        // But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)
-        // and start over. That's not an option if we have no ops.
-        // If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.
-        // Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.
-        if (this.options.softLimit !== undefined
-            && this.length > 0
-            && socketMessageSize >= this.options.softLimit
-            && Infinity === (this.options.compressionOptions?.minimumBatchSizeInBytes ?? Infinity)) {
-            return false;
-        }
-
-        if (socketMessageSize >= this.options.hardLimit
-            && Infinity === (this.options.compressionOptions?.minimumBatchSizeInBytes ?? Infinity)) {
-            return false;
-        }
-
-        this.batchContentSize = contentSize;
-        this.pendingBatch.push(message);
-        return true;
-    }
-
-    public get empty() { return this.pendingBatch.length === 0; }
-
-    public popBatch() {
-        const batch = this.pendingBatch;
-        const size = this.batchContentSize;
-        this.pendingBatch = [];
-        this.batchContentSize = 0;
-
-        if (batch.length > 0
-            && this.options.compressionOptions !== undefined
-            && this.options.compressionOptions.minimumBatchSizeInBytes < size) {
-            return this.opCompressor.compressBatch(batch, size);
-        }
-
-        return batch;
-    }
-
-    /**
-     * Capture the pending state at this point
-     */
-    public checkpoint() {
-        const startPoint = this.pendingBatch.length;
-        return {
-            rollback: (process: (message: BatchMessage) => void) => {
-                for (let i = this.pendingBatch.length; i > startPoint;) {
-                    i--;
-                    const message = this.pendingBatch[i];
-                    this.batchContentSize -= message.contents?.length ?? 0;
-                    process(message);
-                }
-
-                this.pendingBatch.length = startPoint;
-            },
-        };
-    }
-}

package/src/opCompressor.ts

@@ -1,59 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { IsoBuffer } from "@fluidframework/common-utils";
-import { ChildLogger } from "@fluidframework/telemetry-utils";
-import { compress } from "lz4js";
-import { BatchMessage } from "./batchManager";
-import { CompressionAlgorithms, ContainerRuntimeMessage } from "./containerRuntime";
-
-/**
- * Compresses batches of ops. It generates a single compressed op that contains
- * the contents of each op in the batch. It then submits empty ops for each original
- * op to reserve sequence numbers.
- */
-export class OpCompressor {
-    private readonly logger;
-    private compressedBatchCount = 0;
-
-    constructor(logger: ITelemetryLogger) {
-        this.logger = ChildLogger.create(logger, "OpCompressor");
-    }
-
-    public compressBatch(batch: BatchMessage[], originalLength: number): BatchMessage[] {
-        const batchToSend: BatchMessage[] = [];
-        this.compressedBatchCount++;
-        const batchedContents: ContainerRuntimeMessage[] = [];
-        for (const message of batch) {
-            batchedContents.push(message.deserializedContent);
-        }
-
-        const compressionStart = Date.now();
-        const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(batchedContents));
-        const compressedContents = compress(contentsAsBuffer);
-        const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
-        const duration = Date.now() - compressionStart;
-
-        if (originalLength > 200000 || this.compressedBatchCount % 100) {
-            this.logger.sendPerformanceEvent({
-                eventName: "CompressedBatch",
-                duration,
-                sizeBeforeCompression: originalLength,
-                sizeAfterCompression: compressedContent.length,
-            });
-        }
-
-        batchToSend.push({ ...batch[0], contents: JSON.stringify({ packedContents: compressedContent }),
-                           metadata: { ...batch[0].metadata, compressed: true },
-                           compression: CompressionAlgorithms.lz4 });
-
-        for (const message of batch.slice(1)) {
-            batchToSend.push({ ...message, contents: undefined });
-        }
-
-        return batchToSend;
-    }
-}

package/src/opDecompressor.ts

@@ -1,82 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-
-import { decompress } from "lz4js";
-import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { assert, IsoBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
-import { CompressionAlgorithms } from ".";
-
-/**
- * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
- * This class relies on some implicit contracts defined below:
- * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true
- * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
- * 3. The final message of a batch will have batch metadata set to false
- * 4. An individually compressed op will have undefined batch metadata and compression set to true
- */
-export class OpDecompressor {
-    private activeBatch = false;
-    private rootMessageContents: any | undefined;
-    private processedCount = 0;
-
-    public processMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {
-        // We're checking for compression = true or top level compression property so
-        // that we can enable compression without waiting on all ordering services
-        // to pick up protocol change. Eventually only the top level property should
-        // be used.
-        if (message.metadata?.batch === true
-            && (message.metadata?.compressed || message.compression !== undefined)) {
-            // Beginning of a compressed batch
-            assert(this.activeBatch === false, "shouldn't have multiple active batches");
-            if (message.compression) {
-                // lz4 is the only supported compression algorithm for now
-                assert(message.compression === CompressionAlgorithms.lz4,
-                        "lz4 is currently the only supported compression algorithm");
-            }
-
-            this.activeBatch = true;
-
-            const contents = IsoBuffer.from(message.contents.packedContents, "base64");
-            const decompressedMessage = decompress(contents);
-            const intoString = Uint8ArrayToString(decompressedMessage);
-            const asObj = JSON.parse(intoString);
-            this.rootMessageContents = asObj;
-
-            return { ...message, contents: this.rootMessageContents[this.processedCount++] };
-        }
-
-        if (this.rootMessageContents !== undefined && message.metadata?.batch === undefined && this.activeBatch) {
-            // Continuation of compressed batch
-            return { ...message, contents: this.rootMessageContents[this.processedCount++] };
-        }
-
-        if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {
-            // End of compressed batch
-            const returnMessage = { ...message,
-                                    contents: this.rootMessageContents[this.processedCount++] };
-
-            this.activeBatch = false;
-            this.rootMessageContents = undefined;
-            this.processedCount = 0;
-
-            return returnMessage;
-        }
-
-        if (message.metadata?.batch === undefined &&
-            (message.metadata?.compressed || message.compression === CompressionAlgorithms.lz4)) {
-            // Single compressed message
-            assert(this.activeBatch === false, "shouldn't receive compressed message in middle of a batch");
-
-            const contents = IsoBuffer.from(message.contents.packedContents, "base64");
-            const decompressedMessage = decompress(contents);
-            const intoString = new TextDecoder().decode(decompressedMessage);
-            const asObj = JSON.parse(intoString);
-
-            return { ...message, contents: asObj[0] };
-        }
-
-        return message;
-    }
-}