@fluidframework/container-runtime 2.20.0 → 2.21.0

package/lib/opLifecycle/opDecompressor.js

@@ -14,6 +14,9 @@ import { CompressionAlgorithms } from "../containerRuntime.js";
  * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
  * 3. The final message of a batch will have batch metadata set to false
  * 4. An individually compressed op will have undefined batch metadata and compression set to true
+ *
+ * Compressed batches from current code are always a single message but this class needs to handle a legacy compressed batch with multiple messages
+ * because we need that functionality for back compat.
  */
 export class OpDecompressor {
     constructor(logger) {
@@ -56,7 +59,7 @@ export class OpDecompressor {
                 return true;
             }
         }
-        catch (err) {
+        catch {
             return false;
         }
         return false;

package/lib/opLifecycle/opDecompressor.js.map

@@ -1 +1 @@
-{"version":3,"file":"opDecompressor.js","sourceRoot":"","sources":["../../src/opLifecycle/opDecompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAE7E,OAAO,EAAE,MAAM,EAAE,MAAM,qCAAqC,CAAC;AAE7D,OAAO,EACN,iBAAiB,GAEjB,MAAM,0CAA0C,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AAEnC,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAU/D;;;;;;;GAOG;AACH,MAAM,OAAO,cAAc;IAQ1B,YAAY,MAA4B;QAPhC,gBAAW,GAAG,KAAK,CAAC;QAIpB,mBAAc,GAAG,CAAC,CAAC;QAsD3B;;WAEG;QACK,yBAAoB,GAAG,KAAK,CAAC;QArDpC,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAC1E,CAAC;IAEM,mBAAmB,CAAC,OAAkC;QAC5D,IAAI,OAAO,CAAC,WAAW,KAAK,qBAAqB,CAAC,GAAG,EAAE,CAAC;YACvD,OAAO,IAAI,CAAC;QACb,CAAC;QAED;;;;;;;;;;WAUG;QACH,IAAI,CAAC;YACJ,IACC,OAAO,CAAC,QAAQ,KAAK,IAAI;gBACzB,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,KAAK,CAAC;gBAC1C,OAAQ,OAAO,CAAC,QAAyC,CAAC,cAAc;oBACvE,QAAQ;gBACR,OAAO,CAAC,QAAoC,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC;gBACvE,SAAS,CAAC,IAAI,CACZ,OAAO,CAAC,QAAoC,CAAC,cAAc,EAC5D,QAAQ,CACR,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAM,OAAO,CAAC,QAAoC,CAAC,cAAc,EACpF,CAAC;gBACF,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;oBAC9B,SAAS,EAAE,mBAAmB;oBAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,KAAK,EAAG,OAAO,CAAC,QAAuC,EAAE,KAAK;iBAC9D,CAAC,CAAC;gBACH,OAAO,IAAI,CAAC;YACb,CAAC;QACF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACd,CAAC;IAED,IAAW,kBAAkB;QAC5B,OAAO,IAAI,CAAC,WAAW,CAAC;IACzB,CAAC;IAOD;;;OAGG;IACI,kBAAkB,CAAC,OAAkC;QAC3D,MAAM,CACL,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,CAAC,WAAW,KAAK,qBAAqB,CAAC,GAAG,EACtF,KAAK,CAAC,uCAAuC,CAC7C,CAAC;QACF,MAAM,CACL,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,EACjC,KAAK,CAAC,2CAA2C,CACjD,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,KAAK,CAAC,4CAA4C,CAAC,CAAC;QACvF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,MAAM,aAAa,GAAI,OAAO,CAAC,QAAuC,EAAE,KAAK,CAAC;QAC9E,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;QAClC,CAAC;aAAM,CAAC;YACP,MAAM,CAAC,aAAa,KAAK,IAAI,EAAE,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAC7B,OAAO,CAAC,QAAoC,CAAC,cAAc,EAC5D,QAAQ,CACR,CAAC;QACF,MAAM,mBAAmB,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;QAC3D,mEAAmE;QACnE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACnD,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,OAAkC;QAC/C,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,mBAAmB,KAAK,SAAS,EAAE,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACxF,MAAM;QACL,sEAAsE;QACtE,IAAI,CAAC,mBAAmB,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EACrD,KAAK,CAAC,+BAA+B,CACrC,CAAC;QAEF,MAAM,aAAa,GAAI,OAAO,CAAC,QAAuC,EAAE,KAAK,CAAC;QAE9E,IAAI,aAAa,KAAK,KAAK,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC1D,0BAA0B;YAC1B,sEAAsE;YACtE,MAAM,aAAa,GAAG,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;YAEzF,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC;YAClC,IAAI,CAAC,mBAAmB,GAAG,SAAS,CAAC;YACrC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;YAExB,OAAO,aAAa,CAAC;QACtB,CAAC;aAAM,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;YACnC,4BAA4B;YAC5B,sEAAsE;YACtE,OAAO,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACxE,MAAM,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAE5E,mCAAmC;QACnC,sEAAsE;QACtE,OAAO,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IAC7E,CAAC;CACD;AAED,+DAA+D;AAC/D,MAAM,UAAU,GAAG,CAClB,eAA0C,EAC1C,QAAiB,EACW,EAAE,CAAC,CAAC;IAChC,GAAG,eAAe;IAClB,QAAQ;IACR,WAAW,EAAE,SAAS;IACtB,qIAAqI;IAErI,QAAQ,EACP,eAAe,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,eAAe,CAAC,QAAQ,EAAE;CACrF,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IsoBuffer, Uint8ArrayToString } from \"@fluid-internal/client-utils\";\nimport { ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport { assert } from \"@fluidframework/core-utils/internal\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/driver-definitions/internal\";\nimport {\n\tcreateChildLogger,\n\ttype ITelemetryLoggerExt,\n} from \"@fluidframework/telemetry-utils/internal\";\nimport { decompress } from \"lz4js\";\n\nimport { CompressionAlgorithms } from \"../containerRuntime.js\";\nimport { IBatchMetadata } from \"../metadata.js\";\n\n/**\n * Compression makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.\n */\ninterface IPackedContentsContents {\n\tpackedContents: string;\n}\n\n/**\n * State machine that \"unrolls\" contents of compressed batches of ops after decompressing them.\n * This class relies on some implicit contracts defined below:\n * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true\n * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set\n * 3. The final message of a batch will have batch metadata set to false\n * 4. An individually compressed op will have undefined batch metadata and compression set to true\n */\nexport class OpDecompressor {\n\tprivate activeBatch = false;\n\t// TODO: better typing\n\t// eslint-disable-next-line @typescript-eslint/no-explicit-any\n\tprivate rootMessageContents: any | undefined;\n\tprivate processedCount = 0;\n\tprivate readonly logger: ITelemetryLoggerExt;\n\n\tconstructor(logger: ITelemetryBaseLogger) {\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpDecompressor\" });\n\t}\n\n\tpublic isCompressedMessage(message: ISequencedDocumentMessage): boolean {\n\t\tif (message.compression === CompressionAlgorithms.lz4) {\n\t\t\treturn true;\n\t\t}\n\n\t\t/**\n\t\t * Back-compat self healing mechanism for ADO:3538, as loaders from\n\t\t * version client_v2.0.0-internal.1.2.0 to client_v2.0.0-internal.2.2.0 do not\n\t\t * support adding the proper compression metadata to compressed messages submitted\n\t\t * by the runtime. Should be removed after the loader reaches sufficient saturation\n\t\t * for a version greater or equal than client_v2.0.0-internal.2.2.0.\n\t\t *\n\t\t * The condition holds true for compressed messages, regardless of metadata. We are ultimately\n\t\t * looking for a message with a single property `packedContents` inside `contents`, of type 'string'\n\t\t * with a base64 encoded value.\n\t\t */\n\t\ttry {\n\t\t\tif (\n\t\t\t\tmessage.contents !== null &&\n\t\t\t\ttypeof message.contents === \"object\" &&\n\t\t\t\tObject.keys(message.contents).length === 1 &&\n\t\t\t\ttypeof (message.contents as { packedContents?: unknown }).packedContents ===\n\t\t\t\t\t\"string\" &&\n\t\t\t\t(message.contents as IPackedContentsContents).packedContents.length > 0 &&\n\t\t\t\tIsoBuffer.from(\n\t\t\t\t\t(message.contents as IPackedContentsContents).packedContents,\n\t\t\t\t\t\"base64\",\n\t\t\t\t).toString(\"base64\") === (message.contents as IPackedContentsContents).packedContents\n\t\t\t) {\n\t\t\t\tthis.logger.sendTelemetryEvent({\n\t\t\t\t\teventName: \"LegacyCompression\",\n\t\t\t\t\ttype: message.type,\n\t\t\t\t\tbatch: (message.metadata as IBatchMetadata | undefined)?.batch,\n\t\t\t\t});\n\t\t\t\treturn true;\n\t\t\t}\n\t\t} catch (err) {\n\t\t\treturn false;\n\t\t}\n\n\t\treturn false;\n\t}\n\n\tpublic get currentlyUnrolling(): boolean {\n\t\treturn this.activeBatch;\n\t}\n\n\t/**\n\t * Is the decompressed and stored batch only comprised of a single message\n\t */\n\tprivate isSingleMessageBatch = false;\n\n\t/**\n\t * Decompress the given compressed message and store it to be subsequently unrolled.\n\t * The stored message will be of type `any[]` where each element represents a message's `contents`\n\t */\n\tpublic decompressAndStore(message: ISequencedDocumentMessage): void {\n\t\tassert(\n\t\t\tmessage.compression === undefined || message.compression === CompressionAlgorithms.lz4,\n\t\t\t0x511 /* Only lz4 compression is supported */,\n\t\t);\n\t\tassert(\n\t\t\tthis.isCompressedMessage(message),\n\t\t\t0x940 /* provided message should be compressed */,\n\t\t);\n\n\t\tassert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);\n\t\tthis.activeBatch = true;\n\n\t\tconst batchMetadata = (message.metadata as IBatchMetadata | undefined)?.batch;\n\t\tif (batchMetadata === undefined) {\n\t\t\tthis.isSingleMessageBatch = true;\n\t\t} else {\n\t\t\tassert(batchMetadata === true, 0x941 /* invalid batch metadata */);\n\t\t}\n\n\t\tconst contents = IsoBuffer.from(\n\t\t\t(message.contents as IPackedContentsContents).packedContents,\n\t\t\t\"base64\",\n\t\t);\n\t\tconst decompressedMessage = decompress(contents);\n\t\tconst intoString = Uint8ArrayToString(decompressedMessage);\n\t\t// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment\n\t\tthis.rootMessageContents = JSON.parse(intoString);\n\t}\n\n\t/**\n\t * Unroll the next message from the decompressed content provided to {@link decompressAndStore}\n\t * @returns the unrolled `ISequencedDocumentMessage`\n\t */\n\tpublic unroll(message: ISequencedDocumentMessage): ISequencedDocumentMessage {\n\t\tassert(this.currentlyUnrolling, 0x942 /* not currently unrolling */);\n\t\tassert(this.rootMessageContents !== undefined, 0x943 /* missing rootMessageContents */);\n\t\tassert(\n\t\t\t// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access\n\t\t\tthis.rootMessageContents.length > this.processedCount,\n\t\t\t0x944 /* no more content to unroll */,\n\t\t);\n\n\t\tconst batchMetadata = (message.metadata as IBatchMetadata | undefined)?.batch;\n\n\t\tif (batchMetadata === false || this.isSingleMessageBatch) {\n\t\t\t// End of compressed batch\n\t\t\t// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access\n\t\t\tconst returnMessage = newMessage(message, this.rootMessageContents[this.processedCount]);\n\n\t\t\tthis.activeBatch = false;\n\t\t\tthis.isSingleMessageBatch = false;\n\t\t\tthis.rootMessageContents = undefined;\n\t\t\tthis.processedCount = 0;\n\n\t\t\treturn returnMessage;\n\t\t} else if (batchMetadata === true) {\n\t\t\t// Start of compressed batch\n\t\t\t// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access\n\t\t\treturn newMessage(message, this.rootMessageContents[this.processedCount++]);\n\t\t}\n\n\t\tassert(batchMetadata === undefined, 0x945 /* invalid batch metadata */);\n\t\tassert(message.contents === undefined, 0x512 /* Expecting empty message */);\n\n\t\t// Continuation of compressed batch\n\t\t// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access\n\t\treturn newMessage(message, this.rootMessageContents[this.processedCount++]);\n\t}\n}\n\n// We should not be mutating the input message nor its metadata\nconst newMessage = (\n\toriginalMessage: ISequencedDocumentMessage,\n\tcontents: unknown,\n): ISequencedDocumentMessage => ({\n\t...originalMessage,\n\tcontents,\n\tcompression: undefined,\n\t// TODO: It should already be the case that we're not modifying any metadata, not clear if/why this shallow clone should be required.\n\n\tmetadata:\n\t\toriginalMessage.metadata === undefined ? undefined : { ...originalMessage.metadata },\n});\n"]}
+{"version":3,"file":"opDecompressor.js","sourceRoot":"","sources":["../../src/opLifecycle/opDecompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAE7E,OAAO,EAAE,MAAM,EAAE,MAAM,qCAAqC,CAAC;AAE7D,OAAO,EACN,iBAAiB,GAEjB,MAAM,0CAA0C,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AAEnC,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAU/D;;;;;;;;;;GAUG;AACH,MAAM,OAAO,cAAc;IAQ1B,YAAY,MAA4B;QAPhC,gBAAW,GAAG,KAAK,CAAC;QAIpB,mBAAc,GAAG,CAAC,CAAC;QAsD3B;;WAEG;QACK,yBAAoB,GAAG,KAAK,CAAC;QArDpC,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAC1E,CAAC;IAEM,mBAAmB,CAAC,OAAkC;QAC5D,IAAI,OAAO,CAAC,WAAW,KAAK,qBAAqB,CAAC,GAAG,EAAE,CAAC;YACvD,OAAO,IAAI,CAAC;QACb,CAAC;QAED;;;;;;;;;;WAUG;QACH,IAAI,CAAC;YACJ,IACC,OAAO,CAAC,QAAQ,KAAK,IAAI;gBACzB,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,KAAK,CAAC;gBAC1C,OAAQ,OAAO,CAAC,QAAyC,CAAC,cAAc;oBACvE,QAAQ;gBACR,OAAO,CAAC,QAAoC,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC;gBACvE,SAAS,CAAC,IAAI,CACZ,OAAO,CAAC,QAAoC,CAAC,cAAc,EAC5D,QAAQ,CACR,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAM,OAAO,CAAC,QAAoC,CAAC,cAAc,EACpF,CAAC;gBACF,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;oBAC9B,SAAS,EAAE,mBAAmB;oBAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,KAAK,EAAG,OAAO,CAAC,QAAuC,EAAE,KAAK;iBAC9D,CAAC,CAAC;gBACH,OAAO,IAAI,CAAC;YACb,CAAC;QACF,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACd,CAAC;IAED,IAAW,kBAAkB;QAC5B,OAAO,IAAI,CAAC,WAAW,CAAC;IACzB,CAAC;IAOD;;;OAGG;IACI,kBAAkB,CAAC,OAAkC;QAC3D,MAAM,CACL,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,CAAC,WAAW,KAAK,qBAAqB,CAAC,GAAG,EACtF,KAAK,CAAC,uCAAuC,CAC7C,CAAC;QACF,MAAM,CACL,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,EACjC,KAAK,CAAC,2CAA2C,CACjD,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,KAAK,CAAC,4CAA4C,CAAC,CAAC;QACvF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,MAAM,aAAa,GAAI,OAAO,CAAC,QAAuC,EAAE,KAAK,CAAC;QAC9E,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;QAClC,CAAC;aAAM,CAAC;YACP,MAAM,CAAC,aAAa,KAAK,IAAI,EAAE,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAC7B,OAAO,CAAC,QAAoC,CAAC,cAAc,EAC5D,QAAQ,CACR,CAAC;QACF,MAAM,mBAAmB,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;QAC3D,mEAAmE;QACnE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACnD,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,OAAkC;QAC/C,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,mBAAmB,KAAK,SAAS,EAAE,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACxF,MAAM;QACL,sEAAsE;QACtE,IAAI,CAAC,mBAAmB,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EACrD,KAAK,CAAC,+BAA+B,CACrC,CAAC;QAEF,MAAM,aAAa,GAAI,OAAO,CAAC,QAAuC,EAAE,KAAK,CAAC;QAE9E,IAAI,aAAa,KAAK,KAAK,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC1D,0BAA0B;YAC1B,sEAAsE;YACtE,MAAM,aAAa,GAAG,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;YAEzF,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,IAAI,CAAC,oBAAoB,GAAG,KAAK,CAAC;YAClC,IAAI,CAAC,mBAAmB,GAAG,SAAS,CAAC;YACrC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;YAExB,OAAO,aAAa,CAAC;QACtB,CAAC;aAAM,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;YACnC,4BAA4B;YAC5B,sEAAsE;YACtE,OAAO,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACxE,MAAM,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAE5E,mCAAmC;QACnC,sEAAsE;QACtE,OAAO,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IAC7E,CAAC;CACD;AAED,+DAA+D;AAC/D,MAAM,UAAU,GAAG,CAClB,eAA0C,EAC1C,QAAiB,EACW,EAAE,CAAC,CAAC;IAChC,GAAG,eAAe;IAClB,QAAQ;IACR,WAAW,EAAE,SAAS;IACtB,qIAAqI;IAErI,QAAQ,EACP,eAAe,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,eAAe,CAAC,QAAQ,EAAE;CACrF,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IsoBuffer, Uint8ArrayToString } from \"@fluid-internal/client-utils\";\nimport { ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport { assert } from \"@fluidframework/core-utils/internal\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/driver-definitions/internal\";\nimport {\n\tcreateChildLogger,\n\ttype ITelemetryLoggerExt,\n} from \"@fluidframework/telemetry-utils/internal\";\nimport { decompress } from \"lz4js\";\n\nimport { CompressionAlgorithms } from \"../containerRuntime.js\";\nimport { IBatchMetadata } from \"../metadata.js\";\n\n/**\n * Compression makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.\n */\ninterface IPackedContentsContents {\n\tpackedContents: string;\n}\n\n/**\n * State machine that \"unrolls\" contents of compressed batches of ops after decompressing them.\n * This class relies on some implicit contracts defined below:\n * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true\n * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set\n * 3. The final message of a batch will have batch metadata set to false\n * 4. An individually compressed op will have undefined batch metadata and compression set to true\n *\n * Compressed batches from current code are always a single message but this class needs to handle a legacy compressed batch with multiple messages\n * because we need that functionality for back compat.\n */\nexport class OpDecompressor {\n\tprivate activeBatch = false;\n\t// TODO: better typing\n\t// eslint-disable-next-line @typescript-eslint/no-explicit-any\n\tprivate rootMessageContents: any | undefined;\n\tprivate processedCount = 0;\n\tprivate readonly logger: ITelemetryLoggerExt;\n\n\tconstructor(logger: ITelemetryBaseLogger) {\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpDecompressor\" });\n\t}\n\n\tpublic isCompressedMessage(message: ISequencedDocumentMessage): boolean {\n\t\tif (message.compression === CompressionAlgorithms.lz4) {\n\t\t\treturn true;\n\t\t}\n\n\t\t/**\n\t\t * Back-compat self healing mechanism for ADO:3538, as loaders from\n\t\t * version client_v2.0.0-internal.1.2.0 to client_v2.0.0-internal.2.2.0 do not\n\t\t * support adding the proper compression metadata to compressed messages submitted\n\t\t * by the runtime. Should be removed after the loader reaches sufficient saturation\n\t\t * for a version greater or equal than client_v2.0.0-internal.2.2.0.\n\t\t *\n\t\t * The condition holds true for compressed messages, regardless of metadata. We are ultimately\n\t\t * looking for a message with a single property `packedContents` inside `contents`, of type 'string'\n\t\t * with a base64 encoded value.\n\t\t */\n\t\ttry {\n\t\t\tif (\n\t\t\t\tmessage.contents !== null &&\n\t\t\t\ttypeof message.contents === \"object\" &&\n\t\t\t\tObject.keys(message.contents).length === 1 &&\n\t\t\t\ttypeof (message.contents as { packedContents?: unknown }).packedContents ===\n\t\t\t\t\t\"string\" &&\n\t\t\t\t(message.contents as IPackedContentsContents).packedContents.length > 0 &&\n\t\t\t\tIsoBuffer.from(\n\t\t\t\t\t(message.contents as IPackedContentsContents).packedContents,\n\t\t\t\t\t\"base64\",\n\t\t\t\t).toString(\"base64\") === (message.contents as IPackedContentsContents).packedContents\n\t\t\t) {\n\t\t\t\tthis.logger.sendTelemetryEvent({\n\t\t\t\t\teventName: \"LegacyCompression\",\n\t\t\t\t\ttype: message.type,\n\t\t\t\t\tbatch: (message.metadata as IBatchMetadata | undefined)?.batch,\n\t\t\t\t});\n\t\t\t\treturn true;\n\t\t\t}\n\t\t} catch {\n\t\t\treturn false;\n\t\t}\n\n\t\treturn false;\n\t}\n\n\tpublic get currentlyUnrolling(): boolean {\n\t\treturn this.activeBatch;\n\t}\n\n\t/**\n\t * Is the decompressed and stored batch only comprised of a single message\n\t */\n\tprivate isSingleMessageBatch = false;\n\n\t/**\n\t * Decompress the given compressed message and store it to be subsequently unrolled.\n\t * The stored message will be of type `any[]` where each element represents a message's `contents`\n\t */\n\tpublic decompressAndStore(message: ISequencedDocumentMessage): void {\n\t\tassert(\n\t\t\tmessage.compression === undefined || message.compression === CompressionAlgorithms.lz4,\n\t\t\t0x511 /* Only lz4 compression is supported */,\n\t\t);\n\t\tassert(\n\t\t\tthis.isCompressedMessage(message),\n\t\t\t0x940 /* provided message should be compressed */,\n\t\t);\n\n\t\tassert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);\n\t\tthis.activeBatch = true;\n\n\t\tconst batchMetadata = (message.metadata as IBatchMetadata | undefined)?.batch;\n\t\tif (batchMetadata === undefined) {\n\t\t\tthis.isSingleMessageBatch = true;\n\t\t} else {\n\t\t\tassert(batchMetadata === true, 0x941 /* invalid batch metadata */);\n\t\t}\n\n\t\tconst contents = IsoBuffer.from(\n\t\t\t(message.contents as IPackedContentsContents).packedContents,\n\t\t\t\"base64\",\n\t\t);\n\t\tconst decompressedMessage = decompress(contents);\n\t\tconst intoString = Uint8ArrayToString(decompressedMessage);\n\t\t// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment\n\t\tthis.rootMessageContents = JSON.parse(intoString);\n\t}\n\n\t/**\n\t * Unroll the next message from the decompressed content provided to {@link decompressAndStore}\n\t * @returns the unrolled `ISequencedDocumentMessage`\n\t */\n\tpublic unroll(message: ISequencedDocumentMessage): ISequencedDocumentMessage {\n\t\tassert(this.currentlyUnrolling, 0x942 /* not currently unrolling */);\n\t\tassert(this.rootMessageContents !== undefined, 0x943 /* missing rootMessageContents */);\n\t\tassert(\n\t\t\t// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access\n\t\t\tthis.rootMessageContents.length > this.processedCount,\n\t\t\t0x944 /* no more content to unroll */,\n\t\t);\n\n\t\tconst batchMetadata = (message.metadata as IBatchMetadata | undefined)?.batch;\n\n\t\tif (batchMetadata === false || this.isSingleMessageBatch) {\n\t\t\t// End of compressed batch\n\t\t\t// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access\n\t\t\tconst returnMessage = newMessage(message, this.rootMessageContents[this.processedCount]);\n\n\t\t\tthis.activeBatch = false;\n\t\t\tthis.isSingleMessageBatch = false;\n\t\t\tthis.rootMessageContents = undefined;\n\t\t\tthis.processedCount = 0;\n\n\t\t\treturn returnMessage;\n\t\t} else if (batchMetadata === true) {\n\t\t\t// Start of compressed batch\n\t\t\t// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access\n\t\t\treturn newMessage(message, this.rootMessageContents[this.processedCount++]);\n\t\t}\n\n\t\tassert(batchMetadata === undefined, 0x945 /* invalid batch metadata */);\n\t\tassert(message.contents === undefined, 0x512 /* Expecting empty message */);\n\n\t\t// Continuation of compressed batch\n\t\t// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access\n\t\treturn newMessage(message, this.rootMessageContents[this.processedCount++]);\n\t}\n}\n\n// We should not be mutating the input message nor its metadata\nconst newMessage = (\n\toriginalMessage: ISequencedDocumentMessage,\n\tcontents: unknown,\n): ISequencedDocumentMessage => ({\n\t...originalMessage,\n\tcontents,\n\tcompression: undefined,\n\t// TODO: It should already be the case that we're not modifying any metadata, not clear if/why this shallow clone should be required.\n\n\tmetadata:\n\t\toriginalMessage.metadata === undefined ? undefined : { ...originalMessage.metadata },\n});\n"]}

package/lib/opLifecycle/opGroupingManager.d.ts

@@ -8,7 +8,6 @@ import { IBatch, type BatchMessage } from "./definitions.js";
 export declare function isGroupedBatch(op: ISequencedDocumentMessage): boolean;
 export interface OpGroupingManagerConfig {
     readonly groupedBatchingEnabled: boolean;
-    readonly opCountThreshold: number;
 }
 export declare class OpGroupingManager {
     private readonly config;
@@ -33,5 +32,6 @@ export declare class OpGroupingManager {
     groupBatch(batch: IBatch): IBatch<[BatchMessage]>;
     ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[];
     shouldGroup(batch: IBatch): boolean;
+    groupedBatchingEnabled(): boolean;
 }
 //# sourceMappingURL=opGroupingManager.d.ts.map

package/lib/opLifecycle/opGroupingManager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"opGroupingManager.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opGroupingManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AAEvE,OAAO,EAAE,yBAAyB,EAAE,MAAM,6CAA6C,CAAC;AAMxF,OAAO,EAAE,MAAM,EAAE,KAAK,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAuB7D,wBAAgB,cAAc,CAAC,EAAE,EAAE,yBAAyB,GAAG,OAAO,CAErE;AAED,MAAM,WAAW,uBAAuB;IACvC,QAAQ,CAAC,sBAAsB,EAAE,OAAO,CAAC;IACzC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;CAClC;AAED,qBAAa,iBAAiB;IAK5B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJxB,MAAM,CAAC,QAAQ,CAAC,cAAc,kBAAkB;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;gBAG3B,MAAM,EAAE,uBAAuB,EAChD,MAAM,EAAE,oBAAoB;IAK7B;;;;;;OAMG;IACI,uBAAuB,CAC7B,mBAAmB,EAAE,MAAM,EAC3B,uBAAuB,EAAE,MAAM,GAC7B,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAwBzB;;;;;;OAMG;IACI,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IA8CjD,SAAS,CAAC,EAAE,EAAE,yBAAyB,GAAG,yBAAyB,EAAE;IAcrE,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;CAU1C"}
+{"version":3,"file":"opGroupingManager.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opGroupingManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AAEvE,OAAO,EAAE,yBAAyB,EAAE,MAAM,6CAA6C,CAAC;AAMxF,OAAO,EAAE,MAAM,EAAE,KAAK,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAuB7D,wBAAgB,cAAc,CAAC,EAAE,EAAE,yBAAyB,GAAG,OAAO,CAErE;AAED,MAAM,WAAW,uBAAuB;IACvC,QAAQ,CAAC,sBAAsB,EAAE,OAAO,CAAC;CACzC;AAED,qBAAa,iBAAiB;IAK5B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJxB,MAAM,CAAC,QAAQ,CAAC,cAAc,kBAAkB;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;gBAG3B,MAAM,EAAE,uBAAuB,EAChD,MAAM,EAAE,oBAAoB;IAK7B;;;;;;OAMG;IACI,uBAAuB,CAC7B,mBAAmB,EAAE,MAAM,EAC3B,uBAAuB,EAAE,MAAM,GAC7B,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAwBzB;;;;;;OAMG;IACI,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IA6CjD,SAAS,CAAC,EAAE,EAAE,yBAAyB,GAAG,yBAAyB,EAAE;IAcrE,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAUnC,sBAAsB,IAAI,OAAO;CAGxC"}

package/lib/opLifecycle/opGroupingManager.js

@@ -55,7 +55,6 @@ export class OpGroupingManager {
             this.logger.sendTelemetryEvent({
                 eventName: "GroupLargeBatch",
                 length: batch.messages.length,
-                threshold: this.config.opCountThreshold,
                 reentrant: batch.hasReentrantOps,
                 referenceSequenceNumber: batch.messages[0].referenceSequenceNumber,
             });
@@ -107,12 +106,15 @@ export class OpGroupingManager {
         return (
         // Grouped batching must be enabled
         this.config.groupedBatchingEnabled &&
-            // The number of ops in the batch must surpass the configured threshold
+            // The number of ops in the batch must be 2 or more
             // or be empty (to allow for empty batches to be grouped)
-            (batch.messages.length === 0 || batch.messages.length >= this.config.opCountThreshold)
+            batch.messages.length !== 1
         // Support for reentrant batches will be on by default
         );
     }
+    groupedBatchingEnabled() {
+        return this.config.groupedBatchingEnabled;
+    }
 }
 OpGroupingManager.groupedBatchOp = "groupedBatch";
 //# sourceMappingURL=opGroupingManager.js.map

package/lib/opLifecycle/opGroupingManager.js.map

@@ -1 +1 @@
-{"version":3,"file":"opGroupingManager.js","sourceRoot":"","sources":["../../src/opLifecycle/opGroupingManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,MAAM,EAAE,MAAM,qCAAqC,CAAC;AAE7D,OAAO,EACN,iBAAiB,GAEjB,MAAM,0CAA0C,CAAC;AAkBlD,SAAS,eAAe,CAAC,UAAmB;IAC3C,OAAO,CACL,UAAoD,EAAE,IAAI;QAC3D,iBAAiB,CAAC,cAAc,CAChC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,EAA6B;IAC3D,OAAO,eAAe,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAOD,MAAM,OAAO,iBAAiB;IAI7B,YACkB,MAA+B,EAChD,MAA4B;QADX,WAAM,GAAN,MAAM,CAAyB;QAGhD,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED;;;;;;OAMG;IACI,uBAAuB,CAC7B,mBAA2B,EAC3B,uBAA+B;QAE/B,MAAM,CACL,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAClC,KAAK,CAAC,yEAAyE,CAC/E,CAAC;QACF,MAAM,iBAAiB,GAAG,IAAI,CAAC,SAAS,CAAC;YACxC,IAAI,EAAE,iBAAiB,CAAC,cAAc;YACtC,QAAQ,EAAE,EAAE;SACZ,CAAC,CAAC;QAEH,OAAO;YACN,kBAAkB,EAAE,CAAC;YACrB,QAAQ,EAAE;gBACT;oBACC,QAAQ,EAAE,EAAE,OAAO,EAAE,mBAAmB,EAAE;oBAC1C,eAAe,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;oBACrC,uBAAuB;oBACvB,QAAQ,EAAE,iBAAiB;iBAC3B;aACD;YACD,uBAAuB;SACvB,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACI,UAAU,CAAC,KAAa;QAC9B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAE7E,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;gBAC9B,SAAS,EAAE,iBAAiB;gBAC5B,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM;gBAC7B,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB;gBACvC,SAAS,EAAE,KAAK,CAAC,eAAe;gBAChC,uBAAuB,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,uBAAuB;aAClE,CAAC,CAAC;QACJ,CAAC;QACD,kEAAkE;QAClE,IAAI,cAAc,CAAC;QACnB,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACtC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACtB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC;gBAC7D,IAAI,OAAO,EAAE,CAAC;oBACb,cAAc,GAAG,OAAO,CAAC;gBAC1B,CAAC;gBACD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,KAAK,CAAC,oCAAoC,CAAC,CAAC;YACpF,CAAC;QACF,CAAC;QAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,SAAS,CAAC;YACxC,IAAI,EAAE,iBAAiB,CAAC,cAAc;YACtC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAkB,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;gBACnF,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;aAChC,CAAC,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,YAAY,GAA2B;YAC5C,GAAG,KAAK;YACR,QAAQ,EAAE;gBACT;oBACC,QAAQ,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE;oBACrC,uBAAuB,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,uBAAuB;oBAClE,QAAQ,EAAE,iBAAiB;iBAC3B;aACD;SACD,CAAC;QACF,OAAO,YAAY,CAAC;IACrB,CAAC;IAEM,SAAS,CAAC,EAA6B;QAC7C,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,sCAAsC,CAAC,CAAC;QACnF,MAAM,QAAQ,GAAiC,EAAE,CAAC,QAAQ,CAAC;QAE3D,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,OAAO,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAC7C,GAAG,EAAE;YACL,oBAAoB,EAAE,OAAO,EAAE;YAC/B,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,WAAW,EAAE,UAAU,CAAC,WAAW;SACnC,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,WAAW,CAAC,KAAa;QAC/B,OAAO;QACN,mCAAmC;QACnC,IAAI,CAAC,MAAM,CAAC,sBAAsB;YAClC,uEAAuE;YACvE,yDAAyD;YACzD,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC;QACtF,sDAAsD;SACtD,CAAC;IACH,CAAC;;AAxHe,gCAAc,GAAG,cAAc,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport { assert } from \"@fluidframework/core-utils/internal\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/driver-definitions/internal\";\nimport {\n\tcreateChildLogger,\n\ttype ITelemetryLoggerExt,\n} from \"@fluidframework/telemetry-utils/internal\";\n\nimport { IBatch, type BatchMessage } from \"./definitions.js\";\n\n/**\n * Grouping makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.\n */\ninterface IGroupedBatchMessageContents {\n\ttype: typeof OpGroupingManager.groupedBatchOp;\n\tcontents: IGroupedMessage[];\n}\n\ninterface IGroupedMessage {\n\tcontents?: unknown;\n\tmetadata?: Record<string, unknown>;\n\tcompression?: string;\n}\n\nfunction isGroupContents(opContents: unknown): opContents is IGroupedBatchMessageContents {\n\treturn (\n\t\t(opContents as Partial<IGroupedBatchMessageContents>)?.type ===\n\t\tOpGroupingManager.groupedBatchOp\n\t);\n}\n\nexport function isGroupedBatch(op: ISequencedDocumentMessage): boolean {\n\treturn isGroupContents(op.contents);\n}\n\nexport interface OpGroupingManagerConfig {\n\treadonly groupedBatchingEnabled: boolean;\n\treadonly opCountThreshold: number;\n}\n\nexport class OpGroupingManager {\n\tstatic readonly groupedBatchOp = \"groupedBatch\";\n\tprivate readonly logger: ITelemetryLoggerExt;\n\n\tconstructor(\n\t\tprivate readonly config: OpGroupingManagerConfig,\n\t\tlogger: ITelemetryBaseLogger,\n\t) {\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpGroupingManager\" });\n\t}\n\n\t/**\n\t * Creates a new batch with a single message of type \"groupedBatch\" and empty contents.\n\t * This is needed as a placeholder if a batch becomes empty on resubmit, but we are tracking batch IDs.\n\t * @param resubmittingBatchId - batch ID of the resubmitting batch\n\t * @param referenceSequenceNumber - reference sequence number\n\t * @returns - IBatch containing a single empty Grouped Batch op\n\t */\n\tpublic createEmptyGroupedBatch(\n\t\tresubmittingBatchId: string,\n\t\treferenceSequenceNumber: number,\n\t): IBatch<[BatchMessage]> {\n\t\tassert(\n\t\t\tthis.config.groupedBatchingEnabled,\n\t\t\t0xa00 /* cannot create empty grouped batch when grouped batching is disabled */,\n\t\t);\n\t\tconst serializedContent = JSON.stringify({\n\t\t\ttype: OpGroupingManager.groupedBatchOp,\n\t\t\tcontents: [],\n\t\t});\n\n\t\treturn {\n\t\t\tcontentSizeInBytes: 0,\n\t\t\tmessages: [\n\t\t\t\t{\n\t\t\t\t\tmetadata: { batchId: resubmittingBatchId },\n\t\t\t\t\tlocalOpMetadata: { emptyBatch: true },\n\t\t\t\t\treferenceSequenceNumber,\n\t\t\t\t\tcontents: serializedContent,\n\t\t\t\t},\n\t\t\t],\n\t\t\treferenceSequenceNumber,\n\t\t};\n\t}\n\n\t/**\n\t * Converts the given batch into a \"grouped batch\" - a batch with a single message of type \"groupedBatch\",\n\t * with contents being an array of the original batch's messages.\n\t *\n\t * @remarks - Remember that a BatchMessage has its content JSON serialized, so the incoming batch message contents\n\t * must be parsed first, and then the type and contents mentioned above are hidden in that JSON serialization.\n\t */\n\tpublic groupBatch(batch: IBatch): IBatch<[BatchMessage]> {\n\t\tassert(this.shouldGroup(batch), 0x946 /* cannot group the provided batch */);\n\n\t\tif (batch.messages.length >= 1000) {\n\t\t\tthis.logger.sendTelemetryEvent({\n\t\t\t\teventName: \"GroupLargeBatch\",\n\t\t\t\tlength: batch.messages.length,\n\t\t\t\tthreshold: this.config.opCountThreshold,\n\t\t\t\treentrant: batch.hasReentrantOps,\n\t\t\t\treferenceSequenceNumber: batch.messages[0].referenceSequenceNumber,\n\t\t\t});\n\t\t}\n\t\t// We expect this will be on the first message, if present at all.\n\t\tlet groupedBatchId;\n\t\tfor (const message of batch.messages) {\n\t\t\tif (message.metadata) {\n\t\t\t\tconst { batch: _batch, batchId, ...rest } = message.metadata;\n\t\t\t\tif (batchId) {\n\t\t\t\t\tgroupedBatchId = batchId;\n\t\t\t\t}\n\t\t\t\tassert(Object.keys(rest).length === 0, 0x5dd /* cannot group ops with metadata */);\n\t\t\t}\n\t\t}\n\n\t\tconst serializedContent = JSON.stringify({\n\t\t\ttype: OpGroupingManager.groupedBatchOp,\n\t\t\tcontents: batch.messages.map<IGroupedMessage>((message) => ({\n\t\t\t\tcontents: message.contents === undefined ? undefined : JSON.parse(message.contents),\n\t\t\t\tmetadata: message.metadata,\n\t\t\t\tcompression: message.compression,\n\t\t\t})),\n\t\t});\n\n\t\tconst groupedBatch: IBatch<[BatchMessage]> = {\n\t\t\t...batch,\n\t\t\tmessages: [\n\t\t\t\t{\n\t\t\t\t\tmetadata: { batchId: groupedBatchId },\n\t\t\t\t\treferenceSequenceNumber: batch.messages[0].referenceSequenceNumber,\n\t\t\t\t\tcontents: serializedContent,\n\t\t\t\t},\n\t\t\t],\n\t\t};\n\t\treturn groupedBatch;\n\t}\n\n\tpublic ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[] {\n\t\tassert(isGroupContents(op.contents), 0x947 /* can only ungroup a grouped batch */);\n\t\tconst contents: IGroupedBatchMessageContents = op.contents;\n\n\t\tlet fakeCsn = 1;\n\t\treturn contents.contents.map((subMessage) => ({\n\t\t\t...op,\n\t\t\tclientSequenceNumber: fakeCsn++,\n\t\t\tcontents: subMessage.contents,\n\t\t\tmetadata: subMessage.metadata,\n\t\t\tcompression: subMessage.compression,\n\t\t}));\n\t}\n\n\tpublic shouldGroup(batch: IBatch): boolean {\n\t\treturn (\n\t\t\t// Grouped batching must be enabled\n\t\t\tthis.config.groupedBatchingEnabled &&\n\t\t\t// The number of ops in the batch must surpass the configured threshold\n\t\t\t// or be empty (to allow for empty batches to be grouped)\n\t\t\t(batch.messages.length === 0 || batch.messages.length >= this.config.opCountThreshold)\n\t\t\t// Support for reentrant batches will be on by default\n\t\t);\n\t}\n}\n"]}
+{"version":3,"file":"opGroupingManager.js","sourceRoot":"","sources":["../../src/opLifecycle/opGroupingManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,MAAM,EAAE,MAAM,qCAAqC,CAAC;AAE7D,OAAO,EACN,iBAAiB,GAEjB,MAAM,0CAA0C,CAAC;AAkBlD,SAAS,eAAe,CAAC,UAAmB;IAC3C,OAAO,CACL,UAAoD,EAAE,IAAI;QAC3D,iBAAiB,CAAC,cAAc,CAChC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,EAA6B;IAC3D,OAAO,eAAe,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAMD,MAAM,OAAO,iBAAiB;IAI7B,YACkB,MAA+B,EAChD,MAA4B;QADX,WAAM,GAAN,MAAM,CAAyB;QAGhD,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED;;;;;;OAMG;IACI,uBAAuB,CAC7B,mBAA2B,EAC3B,uBAA+B;QAE/B,MAAM,CACL,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAClC,KAAK,CAAC,yEAAyE,CAC/E,CAAC;QACF,MAAM,iBAAiB,GAAG,IAAI,CAAC,SAAS,CAAC;YACxC,IAAI,EAAE,iBAAiB,CAAC,cAAc;YACtC,QAAQ,EAAE,EAAE;SACZ,CAAC,CAAC;QAEH,OAAO;YACN,kBAAkB,EAAE,CAAC;YACrB,QAAQ,EAAE;gBACT;oBACC,QAAQ,EAAE,EAAE,OAAO,EAAE,mBAAmB,EAAE;oBAC1C,eAAe,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;oBACrC,uBAAuB;oBACvB,QAAQ,EAAE,iBAAiB;iBAC3B;aACD;YACD,uBAAuB;SACvB,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACI,UAAU,CAAC,KAAa;QAC9B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAE7E,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;gBAC9B,SAAS,EAAE,iBAAiB;gBAC5B,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM;gBAC7B,SAAS,EAAE,KAAK,CAAC,eAAe;gBAChC,uBAAuB,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,uBAAuB;aAClE,CAAC,CAAC;QACJ,CAAC;QACD,kEAAkE;QAClE,IAAI,cAAc,CAAC;QACnB,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACtC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACtB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC;gBAC7D,IAAI,OAAO,EAAE,CAAC;oBACb,cAAc,GAAG,OAAO,CAAC;gBAC1B,CAAC;gBACD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,KAAK,CAAC,oCAAoC,CAAC,CAAC;YACpF,CAAC;QACF,CAAC;QAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,SAAS,CAAC;YACxC,IAAI,EAAE,iBAAiB,CAAC,cAAc;YACtC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAkB,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;gBACnF,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;aAChC,CAAC,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,YAAY,GAA2B;YAC5C,GAAG,KAAK;YACR,QAAQ,EAAE;gBACT;oBACC,QAAQ,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE;oBACrC,uBAAuB,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,uBAAuB;oBAClE,QAAQ,EAAE,iBAAiB;iBAC3B;aACD;SACD,CAAC;QACF,OAAO,YAAY,CAAC;IACrB,CAAC;IAEM,SAAS,CAAC,EAA6B;QAC7C,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,sCAAsC,CAAC,CAAC;QACnF,MAAM,QAAQ,GAAiC,EAAE,CAAC,QAAQ,CAAC;QAE3D,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,OAAO,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAC7C,GAAG,EAAE;YACL,oBAAoB,EAAE,OAAO,EAAE;YAC/B,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,WAAW,EAAE,UAAU,CAAC,WAAW;SACnC,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,WAAW,CAAC,KAAa;QAC/B,OAAO;QACN,mCAAmC;QACnC,IAAI,CAAC,MAAM,CAAC,sBAAsB;YAClC,mDAAmD;YACnD,yDAAyD;YACzD,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAC3B,sDAAsD;SACtD,CAAC;IACH,CAAC;IACM,sBAAsB;QAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC;IAC3C,CAAC;;AA1He,gCAAc,GAAG,cAAc,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport { assert } from \"@fluidframework/core-utils/internal\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/driver-definitions/internal\";\nimport {\n\tcreateChildLogger,\n\ttype ITelemetryLoggerExt,\n} from \"@fluidframework/telemetry-utils/internal\";\n\nimport { IBatch, type BatchMessage } from \"./definitions.js\";\n\n/**\n * Grouping makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.\n */\ninterface IGroupedBatchMessageContents {\n\ttype: typeof OpGroupingManager.groupedBatchOp;\n\tcontents: IGroupedMessage[];\n}\n\ninterface IGroupedMessage {\n\tcontents?: unknown;\n\tmetadata?: Record<string, unknown>;\n\tcompression?: string;\n}\n\nfunction isGroupContents(opContents: unknown): opContents is IGroupedBatchMessageContents {\n\treturn (\n\t\t(opContents as Partial<IGroupedBatchMessageContents>)?.type ===\n\t\tOpGroupingManager.groupedBatchOp\n\t);\n}\n\nexport function isGroupedBatch(op: ISequencedDocumentMessage): boolean {\n\treturn isGroupContents(op.contents);\n}\n\nexport interface OpGroupingManagerConfig {\n\treadonly groupedBatchingEnabled: boolean;\n}\n\nexport class OpGroupingManager {\n\tstatic readonly groupedBatchOp = \"groupedBatch\";\n\tprivate readonly logger: ITelemetryLoggerExt;\n\n\tconstructor(\n\t\tprivate readonly config: OpGroupingManagerConfig,\n\t\tlogger: ITelemetryBaseLogger,\n\t) {\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpGroupingManager\" });\n\t}\n\n\t/**\n\t * Creates a new batch with a single message of type \"groupedBatch\" and empty contents.\n\t * This is needed as a placeholder if a batch becomes empty on resubmit, but we are tracking batch IDs.\n\t * @param resubmittingBatchId - batch ID of the resubmitting batch\n\t * @param referenceSequenceNumber - reference sequence number\n\t * @returns - IBatch containing a single empty Grouped Batch op\n\t */\n\tpublic createEmptyGroupedBatch(\n\t\tresubmittingBatchId: string,\n\t\treferenceSequenceNumber: number,\n\t): IBatch<[BatchMessage]> {\n\t\tassert(\n\t\t\tthis.config.groupedBatchingEnabled,\n\t\t\t0xa00 /* cannot create empty grouped batch when grouped batching is disabled */,\n\t\t);\n\t\tconst serializedContent = JSON.stringify({\n\t\t\ttype: OpGroupingManager.groupedBatchOp,\n\t\t\tcontents: [],\n\t\t});\n\n\t\treturn {\n\t\t\tcontentSizeInBytes: 0,\n\t\t\tmessages: [\n\t\t\t\t{\n\t\t\t\t\tmetadata: { batchId: resubmittingBatchId },\n\t\t\t\t\tlocalOpMetadata: { emptyBatch: true },\n\t\t\t\t\treferenceSequenceNumber,\n\t\t\t\t\tcontents: serializedContent,\n\t\t\t\t},\n\t\t\t],\n\t\t\treferenceSequenceNumber,\n\t\t};\n\t}\n\n\t/**\n\t * Converts the given batch into a \"grouped batch\" - a batch with a single message of type \"groupedBatch\",\n\t * with contents being an array of the original batch's messages.\n\t *\n\t * @remarks - Remember that a BatchMessage has its content JSON serialized, so the incoming batch message contents\n\t * must be parsed first, and then the type and contents mentioned above are hidden in that JSON serialization.\n\t */\n\tpublic groupBatch(batch: IBatch): IBatch<[BatchMessage]> {\n\t\tassert(this.shouldGroup(batch), 0x946 /* cannot group the provided batch */);\n\n\t\tif (batch.messages.length >= 1000) {\n\t\t\tthis.logger.sendTelemetryEvent({\n\t\t\t\teventName: \"GroupLargeBatch\",\n\t\t\t\tlength: batch.messages.length,\n\t\t\t\treentrant: batch.hasReentrantOps,\n\t\t\t\treferenceSequenceNumber: batch.messages[0].referenceSequenceNumber,\n\t\t\t});\n\t\t}\n\t\t// We expect this will be on the first message, if present at all.\n\t\tlet groupedBatchId;\n\t\tfor (const message of batch.messages) {\n\t\t\tif (message.metadata) {\n\t\t\t\tconst { batch: _batch, batchId, ...rest } = message.metadata;\n\t\t\t\tif (batchId) {\n\t\t\t\t\tgroupedBatchId = batchId;\n\t\t\t\t}\n\t\t\t\tassert(Object.keys(rest).length === 0, 0x5dd /* cannot group ops with metadata */);\n\t\t\t}\n\t\t}\n\n\t\tconst serializedContent = JSON.stringify({\n\t\t\ttype: OpGroupingManager.groupedBatchOp,\n\t\t\tcontents: batch.messages.map<IGroupedMessage>((message) => ({\n\t\t\t\tcontents: message.contents === undefined ? undefined : JSON.parse(message.contents),\n\t\t\t\tmetadata: message.metadata,\n\t\t\t\tcompression: message.compression,\n\t\t\t})),\n\t\t});\n\n\t\tconst groupedBatch: IBatch<[BatchMessage]> = {\n\t\t\t...batch,\n\t\t\tmessages: [\n\t\t\t\t{\n\t\t\t\t\tmetadata: { batchId: groupedBatchId },\n\t\t\t\t\treferenceSequenceNumber: batch.messages[0].referenceSequenceNumber,\n\t\t\t\t\tcontents: serializedContent,\n\t\t\t\t},\n\t\t\t],\n\t\t};\n\t\treturn groupedBatch;\n\t}\n\n\tpublic ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[] {\n\t\tassert(isGroupContents(op.contents), 0x947 /* can only ungroup a grouped batch */);\n\t\tconst contents: IGroupedBatchMessageContents = op.contents;\n\n\t\tlet fakeCsn = 1;\n\t\treturn contents.contents.map((subMessage) => ({\n\t\t\t...op,\n\t\t\tclientSequenceNumber: fakeCsn++,\n\t\t\tcontents: subMessage.contents,\n\t\t\tmetadata: subMessage.metadata,\n\t\t\tcompression: subMessage.compression,\n\t\t}));\n\t}\n\n\tpublic shouldGroup(batch: IBatch): boolean {\n\t\treturn (\n\t\t\t// Grouped batching must be enabled\n\t\t\tthis.config.groupedBatchingEnabled &&\n\t\t\t// The number of ops in the batch must be 2 or more\n\t\t\t// or be empty (to allow for empty batches to be grouped)\n\t\t\tbatch.messages.length !== 1\n\t\t\t// Support for reentrant batches will be on by default\n\t\t);\n\t}\n\tpublic groupedBatchingEnabled(): boolean {\n\t\treturn this.config.groupedBatchingEnabled;\n\t}\n}\n"]}

package/lib/opLifecycle/opSplitter.d.ts

@@ -25,25 +25,28 @@ export declare class OpSplitter {
      * Splits the first op of a compressed batch in chunks, sends the chunks separately and
      * returns a new batch composed of the last chunk and the rest of the ops in the original batch.
      *
-     * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops
-     * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original
-     * uncompressed ops at ingestion in the runtime.
+     * A compressed batch is formed by one large op at the first position.
      *
-     * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire
+     * If the op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire
      * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.
      *
-     * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch
-     * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch
-     * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed
-     * (as it is the last chunk).
+     * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch.
+     * This will ensure that the batch semantics of the original (non-compressed) batch are preserved, as the original chunked op
+     * will be unrolled by the runtime when the first message in the batch is processed (as it is the last chunk).
      *
-     * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
+     * To handle legacy compressed batches with empty placeholders this method can attach the empty placeholder ops at the end
+     * of the result batch, ensuring that the batch semantics are preserved.
+     *
+     * To illustrate the current functionality, if the input is `[largeOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
+     * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4]` will be returned.
+     *
+     * With the legacy code, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
      * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.
      *
      * @remarks - A side effect here is that 1 or more chunks are queued immediately for sending in next JS turn.
      *
      * @param batch - the compressed batch which needs to be processed
-     * @returns A new adjusted batch (last chunk + empty placeholders) which can be sent over the wire
+     * @returns A batch with the last chunk of the original message
      */
     splitFirstBatchMessage(batch: IBatch): IBatch;
     processChunk(message: ISequencedDocumentMessage): ProcessChunkResult;

package/lib/opLifecycle/opSplitter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"opSplitter.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAC;AAC/E,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AAEvE,OAAO,EAAE,yBAAyB,EAAE,MAAM,6CAA6C,CAAC;AAWxF,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEpE,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAE5E;AAWD;;GAEG;AACH,qBAAa,UAAU;IAOrB,OAAO,CAAC,QAAQ,CAAC,aAAa;aAGd,gBAAgB,EAAE,MAAM;IACxC,OAAO,CAAC,QAAQ,CAAC,mBAAmB;IATrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;gBAG5C,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,EACX,aAAa,EAC3B,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,EAAE,uBAAuB,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC,GACtE,SAAS,EACI,gBAAgB,EAAE,MAAM,EACvB,mBAAmB,EAAE,MAAM,EAC5C,MAAM,EAAE,oBAAoB;IAM7B,IAAW,sBAAsB,IAAI,OAAO,CAI3C;IAED,IAAW,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAEjD;IAEM,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAMjD,OAAO,CAAC,QAAQ;IA0BhB;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACI,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAmE7C,YAAY,CAAC,OAAO,EAAE,yBAAyB,GAAG,kBAAkB;CAuC3E;AAED,KAAK,kBAAkB,GACpB;IACA,QAAQ,CAAC,YAAY,EAAE,KAAK,CAAC;CAC5B,GACD;IACA,QAAQ,CAAC,YAAY,EAAE,IAAI,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,yBAAyB,CAAC;CAC3C,CAAC;AAkBL;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,OAAO,OACf,YAAY,oBACE,MAAM,YACf,OAAO,KACd,UAAU,EAkDZ,CAAC"}
+{"version":3,"file":"opSplitter.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAC;AAC/E,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AAEvE,OAAO,EAAE,yBAAyB,EAAE,MAAM,6CAA6C,CAAC;AAWxF,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEpE,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAE5E;AAWD;;GAEG;AACH,qBAAa,UAAU;IAOrB,OAAO,CAAC,QAAQ,CAAC,aAAa;aAGd,gBAAgB,EAAE,MAAM;IACxC,OAAO,CAAC,QAAQ,CAAC,mBAAmB;IATrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;gBAG5C,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,EACX,aAAa,EAC3B,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,EAAE,uBAAuB,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC,GACtE,SAAS,EACI,gBAAgB,EAAE,MAAM,EACvB,mBAAmB,EAAE,MAAM,EAC5C,MAAM,EAAE,oBAAoB;IAM7B,IAAW,sBAAsB,IAAI,OAAO,CAI3C;IAED,IAAW,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAEjD;IAEM,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAMjD,OAAO,CAAC,QAAQ;IA0BhB;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACI,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAmE7C,YAAY,CAAC,OAAO,EAAE,yBAAyB,GAAG,kBAAkB;CAuC3E;AAED,KAAK,kBAAkB,GACpB;IACA,QAAQ,CAAC,YAAY,EAAE,KAAK,CAAC;CAC5B,GACD;IACA,QAAQ,CAAC,YAAY,EAAE,IAAI,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,yBAAyB,CAAC;CAC3C,CAAC;AAkBL;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,OAAO,OACf,YAAY,oBACE,MAAM,YACf,OAAO,KACd,UAAU,EAkDZ,CAAC"}

package/lib/opLifecycle/opSplitter.js

@@ -57,25 +57,28 @@ export class OpSplitter {
      * Splits the first op of a compressed batch in chunks, sends the chunks separately and
      * returns a new batch composed of the last chunk and the rest of the ops in the original batch.
      *
-     * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops
-     * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original
-     * uncompressed ops at ingestion in the runtime.
+     * A compressed batch is formed by one large op at the first position.
      *
-     * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire
+     * If the op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire
      * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.
      *
-     * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch
-     * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch
-     * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed
-     * (as it is the last chunk).
+     * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch.
+     * This will ensure that the batch semantics of the original (non-compressed) batch are preserved, as the original chunked op
+     * will be unrolled by the runtime when the first message in the batch is processed (as it is the last chunk).
      *
-     * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
+     * To handle legacy compressed batches with empty placeholders this method can attach the empty placeholder ops at the end
+     * of the result batch, ensuring that the batch semantics are preserved.
+     *
+     * To illustrate the current functionality, if the input is `[largeOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
+     * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4]` will be returned.
+     *
+     * With the legacy code, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.
      * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.
      *
      * @remarks - A side effect here is that 1 or more chunks are queued immediately for sending in next JS turn.
      *
      * @param batch - the compressed batch which needs to be processed
-     * @returns A new adjusted batch (last chunk + empty placeholders) which can be sent over the wire
+     * @returns A batch with the last chunk of the original message
      */
     splitFirstBatchMessage(batch) {
         assert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);
@@ -182,7 +185,7 @@ export const splitOp = (op, chunkSizeInBytes, extraOp = false) => {
     for (let chunkId = 1; chunkId <= chunkCount; chunkId++) {
         const chunk = {
             chunkId,
-            contents: op.contents.substr(offset, chunkSizeInBytes),
+            contents: op.contents.slice(offset, offset + chunkSizeInBytes),
             totalChunks: chunkCount,
         };
         if (chunkId === chunkCount) {

package/lib/opLifecycle/opSplitter.js.map

@@ -1 +1 @@
-{"version":3,"file":"opSplitter.js","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAE,MAAM,EAAE,MAAM,qCAAqC,CAAC;AAE7D,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,gCAAgC,GAEhC,MAAM,0CAA0C,CAAC;AAElD,OAAO,EAAE,oBAAoB,EAAoC,MAAM,oBAAoB,CAAC;AAE5F,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAGvD,MAAM,UAAU,gBAAgB,CAAC,OAAkC;IAClE,OAAO,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAC5C,CAAC;AAOD,SAAS,iBAAiB,CAAC,QAAiB;IAC3C,OAAQ,QAAsC,EAAE,IAAI,KAAK,oBAAoB,CAAC,SAAS,CAAC;AACzF,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,UAAU;IAKtB,YACC,MAA4B,EACX,aAEL,EACI,gBAAwB,EACvB,mBAA2B,EAC5C,MAA4B;QALX,kBAAa,GAAb,aAAa,CAElB;QACI,qBAAgB,GAAhB,gBAAgB,CAAQ;QACvB,wBAAmB,GAAnB,mBAAmB,CAAQ;QAG5C,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAmB,MAAM,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,IAAW,sBAAsB;QAChC,OAAO,CACN,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS,CACpF,CAAC;IACH,CAAC;IAED,IAAW,MAAM;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAEM,kBAAkB,CAAC,QAAgB;QACzC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;IACF,CAAC;IAEO,QAAQ,CACf,QAAgB,EAChB,cAA0B,EAC1B,eAA0C;QAE1C,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,cAAc,CAAC,OAAO,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/C,gGAAgG;YAChG,sGAAsG;YACtG,4DAA4D;YAC5D,MAAM,IAAI,mBAAmB,CAAC,mBAAmB,EAAE;gBAClD,GAAG,gCAAgC,CAAC,eAAe,CAAC;gBACpD,cAAc,EAAE,GAAG,CAAC,MAAM;gBAC1B,OAAO,EAAE,cAAc,CAAC,OAAO;gBAC/B,WAAW,EAAE,cAAc,CAAC,WAAW;aACvC,CAAC,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACI,sBAAsB,CAAC,KAAa;QAC1C,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QAC9E,MAAM,CACL,KAAK,CAAC,kBAAkB,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EACzD,KAAK,CAAC,iCAAiC,CACvC,CAAC;QACF,MAAM,CACL,KAAK,CAAC,uBAAuB,KAAK,SAAS,EAC3C,KAAK,CAAC,8DAA8D,CACpE,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACjF,MAAM,CACL,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,mBAAmB,EAChD,KAAK,CAAC,4DAA4D,CAClE,CAAC;QAEF,MAAM,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,wEAAwE;QAChH,MAAM,CACL,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAC7D,KAAK,CAAC,sDAAsD,CAC5D,CAAC;QAEF,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,uEAAuE;QACvH,MAAM,UAAU,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,OAAO,CACrB,YAAY,EACZ,IAAI,CAAC,gBAAgB;QACrB,wEAAwE;QACxE,gFAAgF;QAChF,yDAAyD;QACzD,UAAU,IAAI,IAAI,CAAC,mBAAmB,CACtC,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACnF,wCAAwC;QACxC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,aAAa,CACjB,CAAC,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC,uBAAuB,CAAC,CAAC,EAC3D,KAAK,CAAC,uBAAuB,CAC7B,CAAC;QACH,CAAC;QAED,4DAA4D;QAC5D,oDAAoD;QACpD,MAAM,SAAS,GAAG,mBAAmB,CACpC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EACzB,KAAK,CAAC,uBAAuB,EAC7B,EAAE,KAAK,EAAE,YAAY,CAAC,QAAQ,EAAE,KAAK,EAAE,CACvC,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;YAChC,wCAAwC;YACxC,SAAS,EAAE,wBAAwB;YACnC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM;YAC7B,WAAW,EAAE,KAAK,CAAC,kBAAkB;YACrC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,UAAU;SACV,CAAC,CAAC;QAEH,OAAO;YACN,QAAQ,EAAE,CAAC,SAAS,EAAE,GAAG,cAAc,CAAC;YACxC,kBAAkB,EAAE,SAAS,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC;YACnD,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;SACtD,CAAC;IACH,CAAC;IAEM,YAAY,CAAC,OAAkC;QACrD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvF,MAAM,QAAQ,GAAqB,OAAO,CAAC,QAAQ,CAAC;QAEpD,+FAA+F;QAE/F,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAkB,CAAC;QAC5C,MAAM,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC;QACzC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QAEjD,IAAI,cAAc,CAAC,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE,CAAC;YACzD,yDAAyD;YACzD,6DAA6D;YAC7D,OAAO;gBACN,YAAY,EAAE,KAAK;aACnB,CAAC;QACH,CAAC;QAED,oEAAoE;QACpE,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAElC,wEAAwE;QACxE,oDAAoD;QACpD,MAAM,eAAe,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QACvC,eAAe,CAAC,QAAQ;YACvB,iBAAiB,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACtE,8BAA8B;QAC9B,kEAAkE;QAClE,+GAA+G;QAC/G,mJAAmJ;QACnJ,eAAe,CAAC,IAAI,GAAI,cAAsB,CAAC,YAAY,CAAC;QAC5D,eAAe,CAAC,QAAQ,GAAG,cAAc,CAAC,gBAAgB,CAAC;QAC3D,eAAe,CAAC,WAAW,GAAG,cAAc,CAAC,mBAAmB,CAAC;QACjE,OAAO;YACN,OAAO,EAAE,eAAe;YACxB,YAAY,EAAE,IAAI;SAClB,CAAC;IACH,CAAC;CACD;AAWD,MAAM,mBAAmB,GAAG,CAC3B,KAAiB,EACjB,uBAA+B,EAC/B,WAAgD,SAAS,EAC1C,EAAE;IACjB,MAAM,OAAO,GAAqC;QACjD,IAAI,EAAE,oBAAoB,CAAC,SAAS;QACpC,QAAQ,EAAE,KAAK;KACf,CAAC;IACF,OAAO;QACN,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QACjC,QAAQ;QACR,uBAAuB;KACvB,CAAC;AACH,CAAC,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CACtB,EAAgB,EAChB,gBAAwB,EACxB,UAAmB,KAAK,EACT,EAAE;IACjB,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,CACL,EAAE,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,CAAC,QAAQ,KAAK,IAAI,EACjD,KAAK,CAAC,uCAAuC,CAC7C,CAAC;IAEF,MAAM,aAAa,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;IACzC,MAAM,UAAU,GACf,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,MAAM,KAAK,GAAe;YACzB,OAAO;YACP,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC;YACtD,WAAW,EAAE,UAAU;SACvB,CAAC;QAEF,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;YAC5B,iDAAiD;YACjD,oDAAoD;YACpD,0DAA0D;YAC1D,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,QAAQ,CAAC;YACrC,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,WAAW,CAAC;YAE3C,8BAA8B;YAC9B,kDAAkD;YAClD,yGAAyG;YACzG,kHAAkH;YAClH,6FAA6F;YAC7F,sHAAsH;YACtH,qEAAqE;YACrE,0GAA0G;YACzG,KAAa,CAAC,YAAY,GAAG,WAAW,CAAC;QAC3C,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,IAAI,gBAAgB,CAAC;QAC3B,MAAM,CACL,OAAO,IAAI,UAAU,GAAG,CAAC,IAAI,MAAM,IAAI,aAAa,EACpD,KAAK,CAAC,kCAAkC,CACxC,CAAC;IACH,CAAC;IAED,MAAM,CACL,MAAM,IAAI,aAAa,EACvB,KAAK,CAAC,wDAAwD,CAC9D,CAAC;IACF,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,KAAK,CAAC,+BAA+B,CAAC,CAAC;IAC5E,OAAO,MAAM,CAAC;AACf,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IBatchMessage } from \"@fluidframework/container-definitions/internal\";\nimport { ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport { assert } from \"@fluidframework/core-utils/internal\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/driver-definitions/internal\";\nimport {\n\tDataCorruptionError,\n\tcreateChildLogger,\n\textractSafePropertiesFromMessage,\n\ttype ITelemetryLoggerExt,\n} from \"@fluidframework/telemetry-utils/internal\";\n\nimport { ContainerMessageType, ContainerRuntimeChunkedOpMessage } from \"../messageTypes.js\";\n\nimport { estimateSocketSize } from \"./batchManager.js\";\nimport { BatchMessage, IBatch, IChunkedOp } from \"./definitions.js\";\n\nexport function isChunkedMessage(message: ISequencedDocumentMessage): boolean {\n\treturn isChunkedContents(message.contents);\n}\n\ninterface IChunkedContents {\n\treadonly type: typeof ContainerMessageType.ChunkedOp;\n\treadonly contents: IChunkedOp;\n}\n\nfunction isChunkedContents(contents: unknown): contents is IChunkedContents {\n\treturn (contents as Partial<IChunkedContents>)?.type === ContainerMessageType.ChunkedOp;\n}\n\n/**\n * Responsible for creating and reconstructing chunked messages.\n */\nexport class OpSplitter {\n\t// Local copy of incomplete received chunks.\n\tprivate readonly chunkMap: Map<string, string[]>;\n\tprivate readonly logger: ITelemetryLoggerExt;\n\n\tconstructor(\n\t\tchunks: [string, string[]][],\n\t\tprivate readonly submitBatchFn:\n\t\t\t| ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number)\n\t\t\t| undefined,\n\t\tpublic readonly chunkSizeInBytes: number,\n\t\tprivate readonly maxBatchSizeInBytes: number,\n\t\tlogger: ITelemetryBaseLogger,\n\t) {\n\t\tthis.chunkMap = new Map<string, string[]>(chunks);\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpSplitter\" });\n\t}\n\n\tpublic get isBatchChunkingEnabled(): boolean {\n\t\treturn (\n\t\t\tthis.chunkSizeInBytes < Number.POSITIVE_INFINITY && this.submitBatchFn !== undefined\n\t\t);\n\t}\n\n\tpublic get chunks(): ReadonlyMap<string, string[]> {\n\t\treturn this.chunkMap;\n\t}\n\n\tpublic clearPartialChunks(clientId: string): void {\n\t\tif (this.chunkMap.has(clientId)) {\n\t\t\tthis.chunkMap.delete(clientId);\n\t\t}\n\t}\n\n\tprivate addChunk(\n\t\tclientId: string,\n\t\tchunkedContent: IChunkedOp,\n\t\toriginalMessage: ISequencedDocumentMessage,\n\t): void {\n\t\tlet map = this.chunkMap.get(clientId);\n\t\tif (map === undefined) {\n\t\t\tmap = [];\n\t\t\tthis.chunkMap.set(clientId, map);\n\t\t}\n\n\t\tif (chunkedContent.chunkId !== map.length + 1) {\n\t\t\t// We are expecting the chunks to be processed sequentially, in the same order as they are sent.\n\t\t\t// Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)\n\t\t\t// holding the existing chunks for that particular clientId.\n\t\t\tthrow new DataCorruptionError(\"Chunk Id mismatch\", {\n\t\t\t\t...extractSafePropertiesFromMessage(originalMessage),\n\t\t\t\tchunkMapLength: map.length,\n\t\t\t\tchunkId: chunkedContent.chunkId,\n\t\t\t\ttotalChunks: chunkedContent.totalChunks,\n\t\t\t});\n\t\t}\n\n\t\tmap.push(chunkedContent.contents);\n\t}\n\n\t/**\n\t * Splits the first op of a compressed batch in chunks, sends the chunks separately and\n\t * returns a new batch composed of the last chunk and the rest of the ops in the original batch.\n\t *\n\t * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops\n\t * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original\n\t * uncompressed ops at ingestion in the runtime.\n\t *\n\t * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire\n\t * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.\n\t *\n\t * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch\n\t * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch\n\t * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed\n\t * (as it is the last chunk).\n\t *\n\t * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.\n\t * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.\n\t *\n\t * @remarks - A side effect here is that 1 or more chunks are queued immediately for sending in next JS turn.\n\t *\n\t * @param batch - the compressed batch which needs to be processed\n\t * @returns A new adjusted batch (last chunk + empty placeholders) which can be sent over the wire\n\t */\n\tpublic splitFirstBatchMessage(batch: IBatch): IBatch {\n\t\tassert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);\n\t\tassert(\n\t\t\tbatch.contentSizeInBytes > 0 && batch.messages.length > 0,\n\t\t\t0x514 /* Batch needs to be non-empty */,\n\t\t);\n\t\tassert(\n\t\t\tbatch.referenceSequenceNumber !== undefined,\n\t\t\t0x58a /* Batch must have a reference sequence number if non-empty */,\n\t\t);\n\t\tassert(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);\n\t\tassert(\n\t\t\tthis.chunkSizeInBytes < this.maxBatchSizeInBytes,\n\t\t\t0x516 /* Chunk size needs to be smaller than the max batch size */,\n\t\t);\n\n\t\tconst firstMessage = batch.messages[0]; // we expect this to be the large compressed op, which needs to be split\n\t\tassert(\n\t\t\t(firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,\n\t\t\t0x518 /* First message in the batch needs to be chunkable */,\n\t\t);\n\n\t\tconst restOfMessages = batch.messages.slice(1); // we expect these to be empty ops, created to reserve sequence numbers\n\t\tconst socketSize = estimateSocketSize(batch);\n\t\tconst chunks = splitOp(\n\t\t\tfirstMessage,\n\t\t\tthis.chunkSizeInBytes,\n\t\t\t// If we estimate that the socket batch size will exceed the batch limit\n\t\t\t// we will inject an empty op to minimize the risk of the payload failing due to\n\t\t\t// the overhead from the trailing empty ops in the batch.\n\t\t\tsocketSize >= this.maxBatchSizeInBytes,\n\t\t);\n\n\t\tassert(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);\n\t\t// Send the first N-1 chunks immediately\n\t\tfor (const chunk of chunks.slice(0, -1)) {\n\t\t\tthis.submitBatchFn(\n\t\t\t\t[chunkToBatchMessage(chunk, batch.referenceSequenceNumber)],\n\t\t\t\tbatch.referenceSequenceNumber,\n\t\t\t);\n\t\t}\n\n\t\t// The last chunk will be part of the new batch and needs to\n\t\t// preserve the batch metadata of the original batch\n\t\tconst lastChunk = chunkToBatchMessage(\n\t\t\tchunks[chunks.length - 1],\n\t\t\tbatch.referenceSequenceNumber,\n\t\t\t{ batch: firstMessage.metadata?.batch },\n\t\t);\n\n\t\tthis.logger.sendPerformanceEvent({\n\t\t\t// Used to be \"Chunked compressed batch\"\n\t\t\teventName: \"CompressedChunkedBatch\",\n\t\t\tlength: batch.messages.length,\n\t\t\tsizeInBytes: batch.contentSizeInBytes,\n\t\t\tchunks: chunks.length,\n\t\t\tchunkSizeInBytes: this.chunkSizeInBytes,\n\t\t\tsocketSize,\n\t\t});\n\n\t\treturn {\n\t\t\tmessages: [lastChunk, ...restOfMessages],\n\t\t\tcontentSizeInBytes: lastChunk.contents?.length ?? 0,\n\t\t\treferenceSequenceNumber: batch.referenceSequenceNumber,\n\t\t};\n\t}\n\n\tpublic processChunk(message: ISequencedDocumentMessage): ProcessChunkResult {\n\t\tassert(isChunkedContents(message.contents), 0x948 /* message not of type ChunkedOp */);\n\t\tconst contents: IChunkedContents = message.contents;\n\n\t\t// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)\n\n\t\tconst clientId = message.clientId as string;\n\t\tconst chunkedContent = contents.contents;\n\t\tthis.addChunk(clientId, chunkedContent, message);\n\n\t\tif (chunkedContent.chunkId < chunkedContent.totalChunks) {\n\t\t\t// We are processing the op in chunks but haven't reached\n\t\t\t// the last chunk yet in order to reconstruct the original op\n\t\t\treturn {\n\t\t\t\tisFinalChunk: false,\n\t\t\t};\n\t\t}\n\n\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\tconst serializedContent = this.chunkMap.get(clientId)!.join(\"\");\n\t\tthis.clearPartialChunks(clientId);\n\n\t\t// The final/complete message will contain the data from all the chunks.\n\t\t// It will have the sequenceNumber of the last chunk\n\t\tconst completeMessage = { ...message };\n\t\tcompleteMessage.contents =\n\t\t\tserializedContent === \"\" ? undefined : JSON.parse(serializedContent);\n\t\t// back-compat with 1.x builds\n\t\t// This is only required / present for non-compressed, chunked ops\n\t\t// For compressed ops, we have op grouping enabled, and type of each op is preserved within compressed content.\n\t\t// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment\n\t\tcompleteMessage.type = (chunkedContent as any).originalType;\n\t\tcompleteMessage.metadata = chunkedContent.originalMetadata;\n\t\tcompleteMessage.compression = chunkedContent.originalCompression;\n\t\treturn {\n\t\t\tmessage: completeMessage,\n\t\t\tisFinalChunk: true,\n\t\t};\n\t}\n}\n\ntype ProcessChunkResult =\n\t| {\n\t\t\treadonly isFinalChunk: false;\n\t  }\n\t| {\n\t\t\treadonly isFinalChunk: true;\n\t\t\treadonly message: ISequencedDocumentMessage;\n\t  };\n\nconst chunkToBatchMessage = (\n\tchunk: IChunkedOp,\n\treferenceSequenceNumber: number,\n\tmetadata: Record<string, unknown> | undefined = undefined,\n): BatchMessage => {\n\tconst payload: ContainerRuntimeChunkedOpMessage = {\n\t\ttype: ContainerMessageType.ChunkedOp,\n\t\tcontents: chunk,\n\t};\n\treturn {\n\t\tcontents: JSON.stringify(payload),\n\t\tmetadata,\n\t\treferenceSequenceNumber,\n\t};\n};\n\n/**\n * Splits an op into smaller ops (chunks), based on the size of the op and the `chunkSizeInBytes` parameter.\n *\n * The last op of the result will be bundled with empty ops in the same batch. There is a risk of the batch payload\n * exceeding the 1MB limit due to the overhead from the empty ops. If the last op is large, the risk is even higher.\n * To minimize the odds, an extra empty op can be added to the result using the `extraOp` parameter.\n *\n * @param op - the op to be split\n * @param chunkSizeInBytes - how large should the chunks be\n * @param extraOp - should an extra empty op be added to the result\n * @returns an array of chunked ops\n */\nexport const splitOp = (\n\top: BatchMessage,\n\tchunkSizeInBytes: number,\n\textraOp: boolean = false,\n): IChunkedOp[] => {\n\tconst chunks: IChunkedOp[] = [];\n\tassert(\n\t\top.contents !== undefined && op.contents !== null,\n\t\t0x51a /* We should have something to chunk */,\n\t);\n\n\tconst contentLength = op.contents.length;\n\tconst chunkCount =\n\t\tMath.floor((contentLength - 1) / chunkSizeInBytes) + 1 + (extraOp ? 1 : 0);\n\tlet offset = 0;\n\tfor (let chunkId = 1; chunkId <= chunkCount; chunkId++) {\n\t\tconst chunk: IChunkedOp = {\n\t\t\tchunkId,\n\t\t\tcontents: op.contents.substr(offset, chunkSizeInBytes),\n\t\t\ttotalChunks: chunkCount,\n\t\t};\n\n\t\tif (chunkId === chunkCount) {\n\t\t\t// We don't need to port these to all the chunks,\n\t\t\t// as we rebuild the original op when we process the\n\t\t\t// last chunk, therefore it is the only one that needs it.\n\t\t\tchunk.originalMetadata = op.metadata;\n\t\t\tchunk.originalCompression = op.compression;\n\n\t\t\t// back-compat with 1.x builds\n\t\t\t// 2.x builds only do chunking for compressed ops.\n\t\t\t// originalType is no longer used in such cases, as each op preserves its type within compressed payload.\n\t\t\t// But, if 1.x builds see this op, and there is no type on the message, then it will ignore this message silently.\n\t\t\t// This is really bad, as we will crash on later ops and it's very hard to debug these cases.\n\t\t\t// If we put some known type here, then we will crash on it (as 1.x does not understand compression, and thus will not\n\t\t\t// find info on the op like address of the channel to deliver the op)\n\t\t\t// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access\n\t\t\t(chunk as any).originalType = \"component\";\n\t\t}\n\n\t\tchunks.push(chunk);\n\t\toffset += chunkSizeInBytes;\n\t\tassert(\n\t\t\tchunkId >= chunkCount - 1 || offset <= contentLength,\n\t\t\t0x58b /* Content offset within bounds */,\n\t\t);\n\t}\n\n\tassert(\n\t\toffset >= contentLength,\n\t\t0x58c /* Content offset equal or larger than content length */,\n\t);\n\tassert(chunks.length === chunkCount, 0x5a5 /* Expected number of chunks */);\n\treturn chunks;\n};\n"]}
+{"version":3,"file":"opSplitter.js","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAE,MAAM,EAAE,MAAM,qCAAqC,CAAC;AAE7D,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,gCAAgC,GAEhC,MAAM,0CAA0C,CAAC;AAElD,OAAO,EAAE,oBAAoB,EAAoC,MAAM,oBAAoB,CAAC;AAE5F,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAGvD,MAAM,UAAU,gBAAgB,CAAC,OAAkC;IAClE,OAAO,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAC5C,CAAC;AAOD,SAAS,iBAAiB,CAAC,QAAiB;IAC3C,OAAQ,QAAsC,EAAE,IAAI,KAAK,oBAAoB,CAAC,SAAS,CAAC;AACzF,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,UAAU;IAKtB,YACC,MAA4B,EACX,aAEL,EACI,gBAAwB,EACvB,mBAA2B,EAC5C,MAA4B;QALX,kBAAa,GAAb,aAAa,CAElB;QACI,qBAAgB,GAAhB,gBAAgB,CAAQ;QACvB,wBAAmB,GAAnB,mBAAmB,CAAQ;QAG5C,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAmB,MAAM,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,IAAW,sBAAsB;QAChC,OAAO,CACN,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS,CACpF,CAAC;IACH,CAAC;IAED,IAAW,MAAM;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAEM,kBAAkB,CAAC,QAAgB;QACzC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;IACF,CAAC;IAEO,QAAQ,CACf,QAAgB,EAChB,cAA0B,EAC1B,eAA0C;QAE1C,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,cAAc,CAAC,OAAO,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/C,gGAAgG;YAChG,sGAAsG;YACtG,4DAA4D;YAC5D,MAAM,IAAI,mBAAmB,CAAC,mBAAmB,EAAE;gBAClD,GAAG,gCAAgC,CAAC,eAAe,CAAC;gBACpD,cAAc,EAAE,GAAG,CAAC,MAAM;gBAC1B,OAAO,EAAE,cAAc,CAAC,OAAO;gBAC/B,WAAW,EAAE,cAAc,CAAC,WAAW;aACvC,CAAC,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACI,sBAAsB,CAAC,KAAa;QAC1C,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QAC9E,MAAM,CACL,KAAK,CAAC,kBAAkB,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EACzD,KAAK,CAAC,iCAAiC,CACvC,CAAC;QACF,MAAM,CACL,KAAK,CAAC,uBAAuB,KAAK,SAAS,EAC3C,KAAK,CAAC,8DAA8D,CACpE,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACjF,MAAM,CACL,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,mBAAmB,EAChD,KAAK,CAAC,4DAA4D,CAClE,CAAC;QAEF,MAAM,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,wEAAwE;QAChH,MAAM,CACL,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAC7D,KAAK,CAAC,sDAAsD,CAC5D,CAAC;QAEF,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,uEAAuE;QACvH,MAAM,UAAU,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,OAAO,CACrB,YAAY,EACZ,IAAI,CAAC,gBAAgB;QACrB,wEAAwE;QACxE,gFAAgF;QAChF,yDAAyD;QACzD,UAAU,IAAI,IAAI,CAAC,mBAAmB,CACtC,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACnF,wCAAwC;QACxC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,aAAa,CACjB,CAAC,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC,uBAAuB,CAAC,CAAC,EAC3D,KAAK,CAAC,uBAAuB,CAC7B,CAAC;QACH,CAAC;QAED,4DAA4D;QAC5D,oDAAoD;QACpD,MAAM,SAAS,GAAG,mBAAmB,CACpC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EACzB,KAAK,CAAC,uBAAuB,EAC7B,EAAE,KAAK,EAAE,YAAY,CAAC,QAAQ,EAAE,KAAK,EAAE,CACvC,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;YAChC,wCAAwC;YACxC,SAAS,EAAE,wBAAwB;YACnC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM;YAC7B,WAAW,EAAE,KAAK,CAAC,kBAAkB;YACrC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,UAAU;SACV,CAAC,CAAC;QAEH,OAAO;YACN,QAAQ,EAAE,CAAC,SAAS,EAAE,GAAG,cAAc,CAAC;YACxC,kBAAkB,EAAE,SAAS,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC;YACnD,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;SACtD,CAAC;IACH,CAAC;IAEM,YAAY,CAAC,OAAkC;QACrD,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvF,MAAM,QAAQ,GAAqB,OAAO,CAAC,QAAQ,CAAC;QAEpD,+FAA+F;QAE/F,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAkB,CAAC;QAC5C,MAAM,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC;QACzC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QAEjD,IAAI,cAAc,CAAC,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE,CAAC;YACzD,yDAAyD;YACzD,6DAA6D;YAC7D,OAAO;gBACN,YAAY,EAAE,KAAK;aACnB,CAAC;QACH,CAAC;QAED,oEAAoE;QACpE,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAElC,wEAAwE;QACxE,oDAAoD;QACpD,MAAM,eAAe,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QACvC,eAAe,CAAC,QAAQ;YACvB,iBAAiB,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACtE,8BAA8B;QAC9B,kEAAkE;QAClE,+GAA+G;QAC/G,mJAAmJ;QACnJ,eAAe,CAAC,IAAI,GAAI,cAAsB,CAAC,YAAY,CAAC;QAC5D,eAAe,CAAC,QAAQ,GAAG,cAAc,CAAC,gBAAgB,CAAC;QAC3D,eAAe,CAAC,WAAW,GAAG,cAAc,CAAC,mBAAmB,CAAC;QACjE,OAAO;YACN,OAAO,EAAE,eAAe;YACxB,YAAY,EAAE,IAAI;SAClB,CAAC;IACH,CAAC;CACD;AAWD,MAAM,mBAAmB,GAAG,CAC3B,KAAiB,EACjB,uBAA+B,EAC/B,WAAgD,SAAS,EAC1C,EAAE;IACjB,MAAM,OAAO,GAAqC;QACjD,IAAI,EAAE,oBAAoB,CAAC,SAAS;QACpC,QAAQ,EAAE,KAAK;KACf,CAAC;IACF,OAAO;QACN,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QACjC,QAAQ;QACR,uBAAuB;KACvB,CAAC;AACH,CAAC,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CACtB,EAAgB,EAChB,gBAAwB,EACxB,UAAmB,KAAK,EACT,EAAE;IACjB,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,CACL,EAAE,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,CAAC,QAAQ,KAAK,IAAI,EACjD,KAAK,CAAC,uCAAuC,CAC7C,CAAC;IAEF,MAAM,aAAa,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;IACzC,MAAM,UAAU,GACf,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,MAAM,KAAK,GAAe;YACzB,OAAO;YACP,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,CAAC;YAC9D,WAAW,EAAE,UAAU;SACvB,CAAC;QAEF,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;YAC5B,iDAAiD;YACjD,oDAAoD;YACpD,0DAA0D;YAC1D,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,QAAQ,CAAC;YACrC,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,WAAW,CAAC;YAE3C,8BAA8B;YAC9B,kDAAkD;YAClD,yGAAyG;YACzG,kHAAkH;YAClH,6FAA6F;YAC7F,sHAAsH;YACtH,qEAAqE;YACrE,0GAA0G;YACzG,KAAa,CAAC,YAAY,GAAG,WAAW,CAAC;QAC3C,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,IAAI,gBAAgB,CAAC;QAC3B,MAAM,CACL,OAAO,IAAI,UAAU,GAAG,CAAC,IAAI,MAAM,IAAI,aAAa,EACpD,KAAK,CAAC,kCAAkC,CACxC,CAAC;IACH,CAAC;IAED,MAAM,CACL,MAAM,IAAI,aAAa,EACvB,KAAK,CAAC,wDAAwD,CAC9D,CAAC;IACF,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,KAAK,CAAC,+BAA+B,CAAC,CAAC;IAC5E,OAAO,MAAM,CAAC;AACf,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IBatchMessage } from \"@fluidframework/container-definitions/internal\";\nimport { ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport { assert } from \"@fluidframework/core-utils/internal\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/driver-definitions/internal\";\nimport {\n\tDataCorruptionError,\n\tcreateChildLogger,\n\textractSafePropertiesFromMessage,\n\ttype ITelemetryLoggerExt,\n} from \"@fluidframework/telemetry-utils/internal\";\n\nimport { ContainerMessageType, ContainerRuntimeChunkedOpMessage } from \"../messageTypes.js\";\n\nimport { estimateSocketSize } from \"./batchManager.js\";\nimport { BatchMessage, IBatch, IChunkedOp } from \"./definitions.js\";\n\nexport function isChunkedMessage(message: ISequencedDocumentMessage): boolean {\n\treturn isChunkedContents(message.contents);\n}\n\ninterface IChunkedContents {\n\treadonly type: typeof ContainerMessageType.ChunkedOp;\n\treadonly contents: IChunkedOp;\n}\n\nfunction isChunkedContents(contents: unknown): contents is IChunkedContents {\n\treturn (contents as Partial<IChunkedContents>)?.type === ContainerMessageType.ChunkedOp;\n}\n\n/**\n * Responsible for creating and reconstructing chunked messages.\n */\nexport class OpSplitter {\n\t// Local copy of incomplete received chunks.\n\tprivate readonly chunkMap: Map<string, string[]>;\n\tprivate readonly logger: ITelemetryLoggerExt;\n\n\tconstructor(\n\t\tchunks: [string, string[]][],\n\t\tprivate readonly submitBatchFn:\n\t\t\t| ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number)\n\t\t\t| undefined,\n\t\tpublic readonly chunkSizeInBytes: number,\n\t\tprivate readonly maxBatchSizeInBytes: number,\n\t\tlogger: ITelemetryBaseLogger,\n\t) {\n\t\tthis.chunkMap = new Map<string, string[]>(chunks);\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpSplitter\" });\n\t}\n\n\tpublic get isBatchChunkingEnabled(): boolean {\n\t\treturn (\n\t\t\tthis.chunkSizeInBytes < Number.POSITIVE_INFINITY && this.submitBatchFn !== undefined\n\t\t);\n\t}\n\n\tpublic get chunks(): ReadonlyMap<string, string[]> {\n\t\treturn this.chunkMap;\n\t}\n\n\tpublic clearPartialChunks(clientId: string): void {\n\t\tif (this.chunkMap.has(clientId)) {\n\t\t\tthis.chunkMap.delete(clientId);\n\t\t}\n\t}\n\n\tprivate addChunk(\n\t\tclientId: string,\n\t\tchunkedContent: IChunkedOp,\n\t\toriginalMessage: ISequencedDocumentMessage,\n\t): void {\n\t\tlet map = this.chunkMap.get(clientId);\n\t\tif (map === undefined) {\n\t\t\tmap = [];\n\t\t\tthis.chunkMap.set(clientId, map);\n\t\t}\n\n\t\tif (chunkedContent.chunkId !== map.length + 1) {\n\t\t\t// We are expecting the chunks to be processed sequentially, in the same order as they are sent.\n\t\t\t// Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)\n\t\t\t// holding the existing chunks for that particular clientId.\n\t\t\tthrow new DataCorruptionError(\"Chunk Id mismatch\", {\n\t\t\t\t...extractSafePropertiesFromMessage(originalMessage),\n\t\t\t\tchunkMapLength: map.length,\n\t\t\t\tchunkId: chunkedContent.chunkId,\n\t\t\t\ttotalChunks: chunkedContent.totalChunks,\n\t\t\t});\n\t\t}\n\n\t\tmap.push(chunkedContent.contents);\n\t}\n\n\t/**\n\t * Splits the first op of a compressed batch in chunks, sends the chunks separately and\n\t * returns a new batch composed of the last chunk and the rest of the ops in the original batch.\n\t *\n\t * A compressed batch is formed by one large op at the first position.\n\t *\n\t * If the op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire\n\t * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.\n\t *\n\t * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch.\n\t * This will ensure that the batch semantics of the original (non-compressed) batch are preserved, as the original chunked op\n\t * will be unrolled by the runtime when the first message in the batch is processed (as it is the last chunk).\n\t *\n\t * To handle legacy compressed batches with empty placeholders this method can attach the empty placeholder ops at the end\n\t * of the result batch, ensuring that the batch semantics are preserved.\n\t *\n\t * To illustrate the current functionality, if the input is `[largeOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.\n\t * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4]` will be returned.\n\t *\n\t * With the legacy code, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.\n\t * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.\n\t *\n\t * @remarks - A side effect here is that 1 or more chunks are queued immediately for sending in next JS turn.\n\t *\n\t * @param batch - the compressed batch which needs to be processed\n\t * @returns A batch with the last chunk of the original message\n\t */\n\tpublic splitFirstBatchMessage(batch: IBatch): IBatch {\n\t\tassert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);\n\t\tassert(\n\t\t\tbatch.contentSizeInBytes > 0 && batch.messages.length > 0,\n\t\t\t0x514 /* Batch needs to be non-empty */,\n\t\t);\n\t\tassert(\n\t\t\tbatch.referenceSequenceNumber !== undefined,\n\t\t\t0x58a /* Batch must have a reference sequence number if non-empty */,\n\t\t);\n\t\tassert(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);\n\t\tassert(\n\t\t\tthis.chunkSizeInBytes < this.maxBatchSizeInBytes,\n\t\t\t0x516 /* Chunk size needs to be smaller than the max batch size */,\n\t\t);\n\n\t\tconst firstMessage = batch.messages[0]; // we expect this to be the large compressed op, which needs to be split\n\t\tassert(\n\t\t\t(firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,\n\t\t\t0x518 /* First message in the batch needs to be chunkable */,\n\t\t);\n\n\t\tconst restOfMessages = batch.messages.slice(1); // we expect these to be empty ops, created to reserve sequence numbers\n\t\tconst socketSize = estimateSocketSize(batch);\n\t\tconst chunks = splitOp(\n\t\t\tfirstMessage,\n\t\t\tthis.chunkSizeInBytes,\n\t\t\t// If we estimate that the socket batch size will exceed the batch limit\n\t\t\t// we will inject an empty op to minimize the risk of the payload failing due to\n\t\t\t// the overhead from the trailing empty ops in the batch.\n\t\t\tsocketSize >= this.maxBatchSizeInBytes,\n\t\t);\n\n\t\tassert(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);\n\t\t// Send the first N-1 chunks immediately\n\t\tfor (const chunk of chunks.slice(0, -1)) {\n\t\t\tthis.submitBatchFn(\n\t\t\t\t[chunkToBatchMessage(chunk, batch.referenceSequenceNumber)],\n\t\t\t\tbatch.referenceSequenceNumber,\n\t\t\t);\n\t\t}\n\n\t\t// The last chunk will be part of the new batch and needs to\n\t\t// preserve the batch metadata of the original batch\n\t\tconst lastChunk = chunkToBatchMessage(\n\t\t\tchunks[chunks.length - 1],\n\t\t\tbatch.referenceSequenceNumber,\n\t\t\t{ batch: firstMessage.metadata?.batch },\n\t\t);\n\n\t\tthis.logger.sendPerformanceEvent({\n\t\t\t// Used to be \"Chunked compressed batch\"\n\t\t\teventName: \"CompressedChunkedBatch\",\n\t\t\tlength: batch.messages.length,\n\t\t\tsizeInBytes: batch.contentSizeInBytes,\n\t\t\tchunks: chunks.length,\n\t\t\tchunkSizeInBytes: this.chunkSizeInBytes,\n\t\t\tsocketSize,\n\t\t});\n\n\t\treturn {\n\t\t\tmessages: [lastChunk, ...restOfMessages],\n\t\t\tcontentSizeInBytes: lastChunk.contents?.length ?? 0,\n\t\t\treferenceSequenceNumber: batch.referenceSequenceNumber,\n\t\t};\n\t}\n\n\tpublic processChunk(message: ISequencedDocumentMessage): ProcessChunkResult {\n\t\tassert(isChunkedContents(message.contents), 0x948 /* message not of type ChunkedOp */);\n\t\tconst contents: IChunkedContents = message.contents;\n\n\t\t// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)\n\n\t\tconst clientId = message.clientId as string;\n\t\tconst chunkedContent = contents.contents;\n\t\tthis.addChunk(clientId, chunkedContent, message);\n\n\t\tif (chunkedContent.chunkId < chunkedContent.totalChunks) {\n\t\t\t// We are processing the op in chunks but haven't reached\n\t\t\t// the last chunk yet in order to reconstruct the original op\n\t\t\treturn {\n\t\t\t\tisFinalChunk: false,\n\t\t\t};\n\t\t}\n\n\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\tconst serializedContent = this.chunkMap.get(clientId)!.join(\"\");\n\t\tthis.clearPartialChunks(clientId);\n\n\t\t// The final/complete message will contain the data from all the chunks.\n\t\t// It will have the sequenceNumber of the last chunk\n\t\tconst completeMessage = { ...message };\n\t\tcompleteMessage.contents =\n\t\t\tserializedContent === \"\" ? undefined : JSON.parse(serializedContent);\n\t\t// back-compat with 1.x builds\n\t\t// This is only required / present for non-compressed, chunked ops\n\t\t// For compressed ops, we have op grouping enabled, and type of each op is preserved within compressed content.\n\t\t// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment\n\t\tcompleteMessage.type = (chunkedContent as any).originalType;\n\t\tcompleteMessage.metadata = chunkedContent.originalMetadata;\n\t\tcompleteMessage.compression = chunkedContent.originalCompression;\n\t\treturn {\n\t\t\tmessage: completeMessage,\n\t\t\tisFinalChunk: true,\n\t\t};\n\t}\n}\n\ntype ProcessChunkResult =\n\t| {\n\t\t\treadonly isFinalChunk: false;\n\t  }\n\t| {\n\t\t\treadonly isFinalChunk: true;\n\t\t\treadonly message: ISequencedDocumentMessage;\n\t  };\n\nconst chunkToBatchMessage = (\n\tchunk: IChunkedOp,\n\treferenceSequenceNumber: number,\n\tmetadata: Record<string, unknown> | undefined = undefined,\n): BatchMessage => {\n\tconst payload: ContainerRuntimeChunkedOpMessage = {\n\t\ttype: ContainerMessageType.ChunkedOp,\n\t\tcontents: chunk,\n\t};\n\treturn {\n\t\tcontents: JSON.stringify(payload),\n\t\tmetadata,\n\t\treferenceSequenceNumber,\n\t};\n};\n\n/**\n * Splits an op into smaller ops (chunks), based on the size of the op and the `chunkSizeInBytes` parameter.\n *\n * The last op of the result will be bundled with empty ops in the same batch. There is a risk of the batch payload\n * exceeding the 1MB limit due to the overhead from the empty ops. If the last op is large, the risk is even higher.\n * To minimize the odds, an extra empty op can be added to the result using the `extraOp` parameter.\n *\n * @param op - the op to be split\n * @param chunkSizeInBytes - how large should the chunks be\n * @param extraOp - should an extra empty op be added to the result\n * @returns an array of chunked ops\n */\nexport const splitOp = (\n\top: BatchMessage,\n\tchunkSizeInBytes: number,\n\textraOp: boolean = false,\n): IChunkedOp[] => {\n\tconst chunks: IChunkedOp[] = [];\n\tassert(\n\t\top.contents !== undefined && op.contents !== null,\n\t\t0x51a /* We should have something to chunk */,\n\t);\n\n\tconst contentLength = op.contents.length;\n\tconst chunkCount =\n\t\tMath.floor((contentLength - 1) / chunkSizeInBytes) + 1 + (extraOp ? 1 : 0);\n\tlet offset = 0;\n\tfor (let chunkId = 1; chunkId <= chunkCount; chunkId++) {\n\t\tconst chunk: IChunkedOp = {\n\t\t\tchunkId,\n\t\t\tcontents: op.contents.slice(offset, offset + chunkSizeInBytes),\n\t\t\ttotalChunks: chunkCount,\n\t\t};\n\n\t\tif (chunkId === chunkCount) {\n\t\t\t// We don't need to port these to all the chunks,\n\t\t\t// as we rebuild the original op when we process the\n\t\t\t// last chunk, therefore it is the only one that needs it.\n\t\t\tchunk.originalMetadata = op.metadata;\n\t\t\tchunk.originalCompression = op.compression;\n\n\t\t\t// back-compat with 1.x builds\n\t\t\t// 2.x builds only do chunking for compressed ops.\n\t\t\t// originalType is no longer used in such cases, as each op preserves its type within compressed payload.\n\t\t\t// But, if 1.x builds see this op, and there is no type on the message, then it will ignore this message silently.\n\t\t\t// This is really bad, as we will crash on later ops and it's very hard to debug these cases.\n\t\t\t// If we put some known type here, then we will crash on it (as 1.x does not understand compression, and thus will not\n\t\t\t// find info on the op like address of the channel to deliver the op)\n\t\t\t// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access\n\t\t\t(chunk as any).originalType = \"component\";\n\t\t}\n\n\t\tchunks.push(chunk);\n\t\toffset += chunkSizeInBytes;\n\t\tassert(\n\t\t\tchunkId >= chunkCount - 1 || offset <= contentLength,\n\t\t\t0x58b /* Content offset within bounds */,\n\t\t);\n\t}\n\n\tassert(\n\t\toffset >= contentLength,\n\t\t0x58c /* Content offset equal or larger than content length */,\n\t);\n\tassert(chunks.length === chunkCount, 0x5a5 /* Expected number of chunks */);\n\treturn chunks;\n};\n"]}

package/lib/opLifecycle/outbox.d.ts

@@ -118,10 +118,10 @@ export declare class Outbox {
      * @remarks - If chunking happens, a side effect here is that 1 or more chunks are queued immediately for sending in next JS turn.
      *
      * @param batch - Raw or Grouped batch to consider for compression/chunking
-     * @returns Either (A) the original batch, (B) a compressed batch (same length as original),
-     * or (C) a batch containing the last chunk (plus empty placeholders from compression if applicable).
+     * @returns Either (A) the original batch, (B) a compressed batch (same length as original)
+     * or (C) a batch containing the last chunk.
      */
-    private compressBatch;
+    private compressAndChunkBatch;
     /**
      * Sends the batch object to the container context to be sent over the wire.
      *

package/lib/opLifecycle/outbox.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"outbox.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/outbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,uBAAuB,EAAE,MAAM,uCAAuC,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAC;AAC/E,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AASvE,OAAO,EAAE,0BAA0B,EAAE,MAAM,wBAAwB,CAAC;AACpE,OAAO,EAAE,+BAA+B,EAAE,MAAM,oBAAoB,CAAC;AACrE,OAAO,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAE5F,OAAO,EAEN,oBAAoB,EAGpB,KAAK,OAAO,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAI7C,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,0BAA0B,CAAC;IAExD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,OAAO,CAAC;IACnC,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;IAClD,QAAQ,CAAC,aAAa,EACnB,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,EAAE,uBAAuB,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC,GACtE,SAAS,CAAC;IACb,QAAQ,CAAC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;IACtD,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,YAAY,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC;IACtC,QAAQ,CAAC,eAAe,EAAE,iBAAiB,CAAC;IAC5C,QAAQ,CAAC,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;IAC/D,QAAQ,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACjE,QAAQ,CAAC,YAAY,EAAE,MAAM,OAAO,CAAC;IACrC,QAAQ,CAAC,cAAc,EAAE,CAAC,KAAK,CAAC,EAAE,uBAAuB,KAAK,IAAI,CAAC;CACnE;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,+BAA+B,GAAG,MAAM,CAErF;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAE,MAAW,GAAG,CAAC,CA0BvE;AAED;;;;;;GAMG;AACH,qBAAa,MAAM;IAiBN,OAAO,CAAC,QAAQ,CAAC,MAAM;IAhBnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAe;IACjD,OAAO,CAAC,oBAAoB,CAAK;IACjC,OAAO,CAAC,QAAQ,CAAS;IAEzB;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAK;IAC9C,OAAO,CAAC,qBAAqB,CAAK;gBAEL,MAAM,EAAE,iBAAiB;IAkBtD,IAAW,YAAY,IAAI,MAAM,CAEhC;IAED,IAAW,qBAAqB,IAAI,MAAM,CAEzC;IAED,IAAW,2BAA2B,IAAI,MAAM,CAE/C;IAED,IAAW,6BAA6B,IAAI,MAAM,CAEjD;IAED,IAAW,OAAO,IAAI,OAAO,CAE5B;IAED;;;;;;;;;OASG;IACH,OAAO,CAAC,sBAAsB;IA2CvB,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IAMnC,gBAAgB,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IAkB7C,kBAAkB,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IAMtD,OAAO,CAAC,wBAAwB;IAiBhC;;;;;OAKG;IACI,KAAK,CAAC,mBAAmB,CAAC,EAAE,OAAO,GAAG,IAAI;IAUjD,OAAO,CAAC,QAAQ;IA2BhB,OAAO,CAAC,eAAe;IAsBvB,OAAO,CAAC,aAAa;IA2CrB;;;;;OAKG;IACH,OAAO,CAAC,MAAM;IA8Bd,OAAO,CAAC,kBAAkB;IAI1B;;;;;;;;OAQG;IACH,OAAO,CAAC,aAAa;IAmCrB;;;;;OAKG;IACH,OAAO,CAAC,SAAS;IA6CjB;;OAEG;IACI,mBAAmB,IAAI;QAC7B,SAAS,EAAE,gBAAgB,CAAC;QAC5B,iBAAiB,EAAE,gBAAgB,CAAC;QACpC,eAAe,EAAE,gBAAgB,CAAC;KAClC;CAUD"}
+{"version":3,"file":"outbox.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/outbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,uBAAuB,EAAE,MAAM,uCAAuC,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAC;AAC/E,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AASvE,OAAO,EAAE,0BAA0B,EAAE,MAAM,wBAAwB,CAAC;AACpE,OAAO,EAAE,+BAA+B,EAAE,MAAM,oBAAoB,CAAC;AACrE,OAAO,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAE5F,OAAO,EAEN,oBAAoB,EAGpB,KAAK,OAAO,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAI7C,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,0BAA0B,CAAC;IAExD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,OAAO,CAAC;IACnC,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;IAClD,QAAQ,CAAC,aAAa,EACnB,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,EAAE,uBAAuB,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC,GACtE,SAAS,CAAC;IACb,QAAQ,CAAC,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;IACtD,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,YAAY,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC;IACtC,QAAQ,CAAC,eAAe,EAAE,iBAAiB,CAAC;IAC5C,QAAQ,CAAC,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;IAC/D,QAAQ,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACjE,QAAQ,CAAC,YAAY,EAAE,MAAM,OAAO,CAAC;IACrC,QAAQ,CAAC,cAAc,EAAE,CAAC,KAAK,CAAC,EAAE,uBAAuB,KAAK,IAAI,CAAC;CACnE;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,+BAA+B,GAAG,MAAM,CAErF;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAE,MAAW,GAAG,CAAC,CA0BvE;AAED;;;;;;GAMG;AACH,qBAAa,MAAM;IAiBN,OAAO,CAAC,QAAQ,CAAC,MAAM;IAhBnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAe;IACjD,OAAO,CAAC,oBAAoB,CAAK;IACjC,OAAO,CAAC,QAAQ,CAAS;IAEzB;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAK;IAC9C,OAAO,CAAC,qBAAqB,CAAK;gBAEL,MAAM,EAAE,iBAAiB;IAoBtD,IAAW,YAAY,IAAI,MAAM,CAEhC;IAED,IAAW,qBAAqB,IAAI,MAAM,CAEzC;IAED,IAAW,2BAA2B,IAAI,MAAM,CAE/C;IAED,IAAW,6BAA6B,IAAI,MAAM,CAEjD;IAED,IAAW,OAAO,IAAI,OAAO,CAE5B;IAED;;;;;;;;;OASG;IACH,OAAO,CAAC,sBAAsB;IA2CvB,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IAMnC,gBAAgB,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IAM7C,kBAAkB,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IAMtD,OAAO,CAAC,wBAAwB;IAiBhC;;;;;OAKG;IACI,KAAK,CAAC,mBAAmB,CAAC,EAAE,OAAO,GAAG,IAAI;IAUjD,OAAO,CAAC,QAAQ;IA2BhB,OAAO,CAAC,eAAe;IAsBvB,OAAO,CAAC,aAAa;IA6CrB;;;;;OAKG;IACH,OAAO,CAAC,MAAM;IA8Bd,OAAO,CAAC,kBAAkB;IAI1B;;;;;;;;OAQG;IACH,OAAO,CAAC,qBAAqB;IAoC7B;;;;;OAKG;IACH,OAAO,CAAC,SAAS;IA6CjB;;OAEG;IACI,mBAAmB,IAAI;QAC7B,SAAS,EAAE,gBAAgB,CAAC;QAC5B,iBAAiB,EAAE,gBAAgB,CAAC;QACpC,eAAe,EAAE,gBAAgB,CAAC;KAClC;CAUD"}

package/lib/opLifecycle/outbox.js

@@ -72,7 +72,9 @@ export class Outbox {
         const isCompressionEnabled = this.params.config.compressionOptions.minimumBatchSizeInBytes !==
             Number.POSITIVE_INFINITY;
         // We need to allow infinite size batches if we enable compression
-        const hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;
+        const hardLimit = isCompressionEnabled
+            ? Number.POSITIVE_INFINITY
+            : this.params.config.maxBatchSizeInBytes;
         this.mainBatch = new BatchManager({ hardLimit, canRebase: true });
         this.blobAttachBatch = new BatchManager({ hardLimit, canRebase: true });
         this.idAllocationBatch = new BatchManager({
@@ -143,15 +145,6 @@ export class Outbox {
     submitBlobAttach(message) {
         this.maybeFlushPartialBatch();
         this.addMessageToBatchManager(this.blobAttachBatch, message);
-        // If compression is enabled, we will always successfully receive
-        // blobAttach ops and compress then send them at the next JS turn, regardless
-        // of the overall size of the accumulated ops in the batch.
-        // However, it is more efficient to flush these ops faster, preferably
-        // after they reach a size which would benefit from compression.
-        if (this.blobAttachBatch.contentSizeInBytes >=
-            this.params.config.compressionOptions.minimumBatchSizeInBytes) {
-            this.flushInternal(this.blobAttachBatch);
-        }
     }
     submitIdAllocation(message) {
         this.maybeFlushPartialBatch();
@@ -229,7 +222,9 @@ export class Outbox {
         // If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
         // Because flush() is a task that executes async (on clean stack), we can get here in disconnected state.
         if (this.params.shouldSend()) {
-            const processedBatch = this.compressBatch(shouldGroup ? this.params.groupingManager.groupBatch(rawBatch) : rawBatch);
+            const processedBatch = disableGroupedBatching
+                ? rawBatch
+                : this.compressAndChunkBatch(shouldGroup ? this.params.groupingManager.groupBatch(rawBatch) : rawBatch);
             clientSequenceNumber = this.sendBatch(processedBatch);
             assert(clientSequenceNumber === undefined || clientSequenceNumber >= 0, 0x9d2 /* unexpected negative clientSequenceNumber (empty batch should yield undefined) */);
         }
@@ -273,15 +268,16 @@ export class Outbox {
      * @remarks - If chunking happens, a side effect here is that 1 or more chunks are queued immediately for sending in next JS turn.
      *
      * @param batch - Raw or Grouped batch to consider for compression/chunking
-     * @returns Either (A) the original batch, (B) a compressed batch (same length as original),
-     * or (C) a batch containing the last chunk (plus empty placeholders from compression if applicable).
+     * @returns Either (A) the original batch, (B) a compressed batch (same length as original)
+     * or (C) a batch containing the last chunk.
      */
-    compressBatch(batch) {
+    compressAndChunkBatch(batch) {
         if (batch.messages.length === 0 ||
             this.params.config.compressionOptions === undefined ||
             this.params.config.compressionOptions.minimumBatchSizeInBytes >
                 batch.contentSizeInBytes ||
-            this.params.submitBatchFn === undefined) {
+            this.params.submitBatchFn === undefined ||
+            !this.params.groupingManager.groupedBatchingEnabled()) {
             // Nothing to do if the batch is empty or if compression is disabled or not supported, or if we don't need to compress
             return batch;
         }