@fluidframework/container-runtime 2.0.0-dev.3.1.0.125672 → 2.0.0-dev.4.1.0.148229

package/dist/opLifecycle/opSplitter.d.ts

@@ -11,11 +11,11 @@ import { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from "./de
  */
 export declare class OpSplitter {
     private readonly submitBatchFn;
-    private readonly chunkSizeInBytes;
+    readonly chunkSizeInBytes: number;
     private readonly maxBatchSizeInBytes;
     private readonly chunkMap;
     private readonly logger;
-    constructor(chunks: [string, string[]][], submitBatchFn: ((batch: IBatchMessage[]) => number) | undefined, chunkSizeInBytes: number, maxBatchSizeInBytes: number, logger: ITelemetryLogger);
+    constructor(chunks: [string, string[]][], submitBatchFn: ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number) | undefined, chunkSizeInBytes: number, maxBatchSizeInBytes: number, logger: ITelemetryLogger);
     get isBatchChunkingEnabled(): boolean;
     get chunks(): ReadonlyMap<string, string[]>;
     processRemoteMessage(message: ISequencedDocumentMessage): IMessageProcessingResult;
@@ -45,5 +45,17 @@ export declare class OpSplitter {
      */
     splitCompressedBatch(batch: IBatch): IBatch;
 }
-export declare const splitOp: (op: BatchMessage, chunkSizeInBytes: number) => IChunkedOp[];
+/**
+ * Splits an op into smaller ops (chunks), based on the size of the op and the `chunkSizeInBytes` parameter.
+ *
+ * The last op of the result will be bundled with empty ops in the same batch. There is a risk of the batch payload
+ * exceeding the 1MB limit due to the overhead from the empty ops. If the last op is large, the risk is even higher.
+ * To minimize the odds, an extra empty op can be added to the result using the `extraOp` parameter.
+ *
+ * @param op - the op to be split
+ * @param chunkSizeInBytes - how large should the chunks be
+ * @param extraOp - should an extra empty op be added to the result
+ * @returns an array of chunked ops
+ */
+export declare const splitOp: (op: BatchMessage, chunkSizeInBytes: number, extraOp?: boolean) => IChunkedOp[];
 //# sourceMappingURL=opSplitter.d.ts.map

package/dist/opLifecycle/opSplitter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"opSplitter.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAEtE,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AAKtE,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAGjF,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AAE3F;;GAEG;AACH,qBAAa,UAAU;IAOrB,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IACjC,OAAO,CAAC,QAAQ,CAAC,mBAAmB;IAPrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAGvB,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,EACX,aAAa,EAAE,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,MAAM,CAAC,GAAG,SAAS,EAC/D,gBAAgB,EAAE,MAAM,EACxB,mBAAmB,EAAE,MAAM,EAC5C,MAAM,EAAE,gBAAgB;IAMzB,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,IAAW,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAEjD;IAEM,oBAAoB,CAAC,OAAO,EAAE,yBAAyB,GAAG,wBAAwB;IAoClF,kBAAkB,CAAC,QAAQ,EAAE,MAAM;IAM1C,OAAO,CAAC,QAAQ;IA0BhB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACI,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;CAoDlD;AAoBD,eAAO,MAAM,OAAO,OAAQ,YAAY,oBAAoB,MAAM,KAAG,UAAU,EA+B9E,CAAC"}
+{"version":3,"file":"opSplitter.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAEtE,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AAKtE,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAIjF,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AAE3F;;GAEG;AACH,qBAAa,UAAU;IAOrB,OAAO,CAAC,QAAQ,CAAC,aAAa;aAGd,gBAAgB,EAAE,MAAM;IACxC,OAAO,CAAC,QAAQ,CAAC,mBAAmB;IATrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAGvB,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,EACX,aAAa,EAC3B,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,EAAE,uBAAuB,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC,GACtE,SAAS,EACI,gBAAgB,EAAE,MAAM,EACvB,mBAAmB,EAAE,MAAM,EAC5C,MAAM,EAAE,gBAAgB;IAMzB,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,IAAW,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAEjD;IAEM,oBAAoB,CAAC,OAAO,EAAE,yBAAyB,GAAG,wBAAwB;IAoClF,kBAAkB,CAAC,QAAQ,EAAE,MAAM;IAM1C,OAAO,CAAC,QAAQ;IA0BhB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACI,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;CAsElD;AAoBD;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,OAAO,OACf,YAAY,oBACE,MAAM,YACf,OAAO,KACd,UAAU,EAqCZ,CAAC"}

package/dist/opLifecycle/opSplitter.js

@@ -9,6 +9,7 @@ const common_utils_1 = require("@fluidframework/common-utils");
 const container_utils_1 = require("@fluidframework/container-utils");
 const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
 const containerRuntime_1 = require("../containerRuntime");
+const batchManager_1 = require("./batchManager");
 /**
  * Responsible for creating and reconstructing chunked messages.
  */
@@ -102,31 +103,40 @@ class OpSplitter {
         var _a, _b, _c, _d, _e, _f;
         (0, common_utils_1.assert)(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);
         (0, common_utils_1.assert)(batch.contentSizeInBytes > 0 && batch.content.length > 0, 0x514 /* Batch needs to be non-empty */);
+        (0, common_utils_1.assert)(batch.referenceSequenceNumber !== undefined, 0x58a /* Batch must have a reference sequence number if non-empty */);
         (0, common_utils_1.assert)(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);
         (0, common_utils_1.assert)(this.chunkSizeInBytes < this.maxBatchSizeInBytes, 0x516 /* Chunk size needs to be smaller than the max batch size */);
         const firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split
         (0, common_utils_1.assert)(((_a = firstMessage.metadata) === null || _a === void 0 ? void 0 : _a.compressed) === true || firstMessage.compression !== undefined, 0x517 /* Batch needs to be compressed */);
         (0, common_utils_1.assert)(((_c = (_b = firstMessage.contents) === null || _b === void 0 ? void 0 : _b.length) !== null && _c !== void 0 ? _c : 0) >= this.chunkSizeInBytes, 0x518 /* First message in the batch needs to be chunkable */);
         const restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers
-        const chunks = (0, exports.splitOp)(firstMessage, this.chunkSizeInBytes);
+        const socketSize = (0, batchManager_1.estimateSocketSize)(batch);
+        const chunks = (0, exports.splitOp)(firstMessage, this.chunkSizeInBytes, 
+        // If we estimate that the socket batch size will exceed the batch limit
+        // we will inject an empty op to minimize the risk of the payload failing due to
+        // the overhead from the trailing empty ops in the batch.
+        socketSize >= this.maxBatchSizeInBytes);
         (0, common_utils_1.assert)(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);
         // Send the first N-1 chunks immediately
         for (const chunk of chunks.slice(0, -1)) {
-            this.submitBatchFn([chunkToBatchMessage(chunk, firstMessage.referenceSequenceNumber)]);
+            this.submitBatchFn([chunkToBatchMessage(chunk, batch.referenceSequenceNumber)], batch.referenceSequenceNumber);
         }
         // The last chunk will be part of the new batch and needs to
         // preserve the batch metadata of the original batch
-        const lastChunk = chunkToBatchMessage(chunks[chunks.length - 1], firstMessage.referenceSequenceNumber, { batch: (_d = firstMessage.metadata) === null || _d === void 0 ? void 0 : _d.batch });
+        const lastChunk = chunkToBatchMessage(chunks[chunks.length - 1], batch.referenceSequenceNumber, { batch: (_d = firstMessage.metadata) === null || _d === void 0 ? void 0 : _d.batch });
         this.logger.sendPerformanceEvent({
-            eventName: "Chunked compressed batch",
+            // Used to be "Chunked compressed batch"
+            eventName: "CompressedChunkedBatch",
             length: batch.content.length,
             sizeInBytes: batch.contentSizeInBytes,
             chunks: chunks.length,
             chunkSizeInBytes: this.chunkSizeInBytes,
+            socketSize,
         });
         return {
             content: [lastChunk, ...restOfMessages],
             contentSizeInBytes: (_f = (_e = lastChunk.contents) === null || _e === void 0 ? void 0 : _e.length) !== null && _f !== void 0 ? _f : 0,
+            referenceSequenceNumber: batch.referenceSequenceNumber,
         };
     }
 }
@@ -144,20 +154,32 @@ const chunkToBatchMessage = (chunk, referenceSequenceNumber, metadata = undefine
         referenceSequenceNumber,
     };
 };
-const splitOp = (op, chunkSizeInBytes) => {
+/**
+ * Splits an op into smaller ops (chunks), based on the size of the op and the `chunkSizeInBytes` parameter.
+ *
+ * The last op of the result will be bundled with empty ops in the same batch. There is a risk of the batch payload
+ * exceeding the 1MB limit due to the overhead from the empty ops. If the last op is large, the risk is even higher.
+ * To minimize the odds, an extra empty op can be added to the result using the `extraOp` parameter.
+ *
+ * @param op - the op to be split
+ * @param chunkSizeInBytes - how large should the chunks be
+ * @param extraOp - should an extra empty op be added to the result
+ * @returns an array of chunked ops
+ */
+const splitOp = (op, chunkSizeInBytes, extraOp = false) => {
     const chunks = [];
     (0, common_utils_1.assert)(op.contents !== undefined && op.contents !== null, 0x51a /* We should have something to chunk */);
     const contentLength = op.contents.length;
-    const chunkN = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1;
+    const chunkCount = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1 + (extraOp ? 1 : 0);
     let offset = 0;
-    for (let i = 1; i <= chunkN; i++) {
+    for (let chunkId = 1; chunkId <= chunkCount; chunkId++) {
         const chunk = {
-            chunkId: i,
+            chunkId,
             contents: op.contents.substr(offset, chunkSizeInBytes),
             originalType: op.deserializedContent.type,
-            totalChunks: chunkN,
+            totalChunks: chunkCount,
         };
-        if (i === chunkN) {
+        if (chunkId === chunkCount) {
             // We don't need to port these to all the chunks,
             // as we rebuild the original op when we process the
             // last chunk, therefore it is the only one that needs it.
@@ -166,7 +188,10 @@ const splitOp = (op, chunkSizeInBytes) => {
         }
         chunks.push(chunk);
         offset += chunkSizeInBytes;
+        (0, common_utils_1.assert)(chunkId >= chunkCount - 1 || offset <= contentLength, 0x58b /* Content offset within bounds */);
     }
+    (0, common_utils_1.assert)(offset >= contentLength, 0x58c /* Content offset equal or larger than content length */);
+    (0, common_utils_1.assert)(chunks.length === chunkCount, 0x5a5 /* Expected number of chunks */);
     return chunks;
 };
 exports.splitOp = splitOp;

package/dist/opLifecycle/opSplitter.js.map

@@ -1 +1 @@
-{"version":3,"file":"opSplitter.js","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAGH,+DAAsD;AAEtD,qEAGyC;AAEzC,qEAA8D;AAC9D,0DAAoF;AAGpF;;GAEG;AACH,MAAa,UAAU;IAKtB,YACC,MAA4B,EACX,aAA+D,EAC/D,gBAAwB,EACxB,mBAA2B,EAC5C,MAAwB;QAHP,kBAAa,GAAb,aAAa,CAAkD;QAC/D,qBAAgB,GAAhB,gBAAgB,CAAQ;QACxB,wBAAmB,GAAnB,mBAAmB,CAAQ;QAG5C,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAmB,MAAM,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACxD,CAAC;IAED,IAAW,sBAAsB;QAChC,OAAO,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS,CAAC;IAC7F,CAAC;IAED,IAAW,MAAM;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAEM,oBAAoB,CAAC,OAAkC;QAC7D,IAAI,OAAO,CAAC,IAAI,KAAK,uCAAoB,CAAC,SAAS,EAAE;YACpD,OAAO;gBACN,OAAO;gBACP,KAAK,EAAE,SAAS;aAChB,CAAC;SACF;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,MAAM,cAAc,GAAG,OAAO,CAAC,QAAsB,CAAC;QACtD,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QAEjD,IAAI,cAAc,CAAC,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE;YACxD,yDAAyD;YACzD,6DAA6D;YAC7D,OAAO;gBACN,OAAO;gBACP,KAAK,EAAE,UAAU;aACjB,CAAC;SACF;QAED,oEAAoE;QACpE,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAElC,MAAM,UAAU,qBAAQ,OAAO,CAAE,CAAC;QAClC,UAAU,CAAC,QAAQ,GAAG,iBAAiB,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAC3F,UAAU,CAAC,IAAI,GAAG,cAAc,CAAC,YAAY,CAAC;QAC9C,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC,gBAAgB,CAAC;QACtD,UAAU,CAAC,WAAW,GAAG,cAAc,CAAC,mBAAmB,CAAC;QAC5D,OAAO;YACN,OAAO,EAAE,UAAU;YACnB,KAAK,EAAE,WAAW;SAClB,CAAC;IACH,CAAC;IAEM,kBAAkB,CAAC,QAAgB;QACzC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAChC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;SAC/B;IACF,CAAC;IAEO,QAAQ,CACf,QAAgB,EAChB,cAA0B,EAC1B,eAA0C;QAE1C,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,GAAG,KAAK,SAAS,EAAE;YACtB,GAAG,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;SACjC;QAED,IAAI,cAAc,CAAC,OAAO,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;YAC9C,gGAAgG;YAChG,sGAAsG;YACtG,4DAA4D;YAC5D,MAAM,IAAI,qCAAmB,CAAC,mBAAmB,kCAC7C,IAAA,kDAAgC,EAAC,eAAe,CAAC,KACpD,cAAc,EAAE,GAAG,CAAC,MAAM,EAC1B,OAAO,EAAE,cAAc,CAAC,OAAO,EAC/B,WAAW,EAAE,cAAc,CAAC,WAAW,IACtC,CAAC;SACH;QAED,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACI,oBAAoB,CAAC,KAAa;;QACxC,IAAA,qBAAM,EAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QAC9E,IAAA,qBAAM,EACL,KAAK,CAAC,kBAAkB,GAAG,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EACxD,KAAK,CAAC,iCAAiC,CACvC,CAAC;QACF,IAAA,qBAAM,EAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACjF,IAAA,qBAAM,EACL,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,mBAAmB,EAChD,KAAK,CAAC,4DAA4D,CAClE,CAAC;QAEF,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,wEAAwE;QAC/G,IAAA,qBAAM,EACL,CAAA,MAAA,YAAY,CAAC,QAAQ,0CAAE,UAAU,MAAK,IAAI,IAAI,YAAY,CAAC,WAAW,KAAK,SAAS,EACpF,KAAK,CAAC,kCAAkC,CACxC,CAAC;QACF,IAAA,qBAAM,EACL,CAAC,MAAA,MAAA,YAAY,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAC7D,KAAK,CAAC,sDAAsD,CAC5D,CAAC;QAEF,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,uEAAuE;QACtH,MAAM,MAAM,GAAG,IAAA,eAAO,EAAC,YAAY,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAE5D,IAAA,qBAAM,EAAC,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACnF,wCAAwC;QACxC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;YACxC,IAAI,CAAC,aAAa,CAAC,CAAC,mBAAmB,CAAC,KAAK,EAAE,YAAY,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC;SACvF;QAED,4DAA4D;QAC5D,oDAAoD;QACpD,MAAM,SAAS,GAAG,mBAAmB,CACpC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EACzB,YAAY,CAAC,uBAAuB,EACpC,EAAE,KAAK,EAAE,MAAA,YAAY,CAAC,QAAQ,0CAAE,KAAK,EAAE,CACvC,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;YAChC,SAAS,EAAE,0BAA0B;YACrC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;YAC5B,WAAW,EAAE,KAAK,CAAC,kBAAkB;YACrC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACvC,CAAC,CAAC;QAEH,OAAO;YACN,OAAO,EAAE,CAAC,SAAS,EAAE,GAAG,cAAc,CAAC;YACvC,kBAAkB,EAAE,MAAA,MAAA,SAAS,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC;SACnD,CAAC;IACH,CAAC;CACD;AAtKD,gCAsKC;AAED,MAAM,mBAAmB,GAAG,CAC3B,KAAiB,EACjB,uBAA+B,EAC/B,WAAgD,SAAS,EAC1C,EAAE;IACjB,MAAM,OAAO,GAA4B;QACxC,IAAI,EAAE,uCAAoB,CAAC,SAAS;QACpC,QAAQ,EAAE,KAAK;KACf,CAAC;IACF,OAAO;QACN,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QACjC,mBAAmB,EAAE,OAAO;QAC5B,QAAQ;QACR,eAAe,EAAE,SAAS;QAC1B,uBAAuB;KACvB,CAAC;AACH,CAAC,CAAC;AAEK,MAAM,OAAO,GAAG,CAAC,EAAgB,EAAE,gBAAwB,EAAgB,EAAE;IACnF,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAA,qBAAM,EACL,EAAE,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,CAAC,QAAQ,KAAK,IAAI,EACjD,KAAK,CAAC,uCAAuC,CAC7C,CAAC;IAEF,MAAM,aAAa,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACtE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC,EAAE,EAAE;QACjC,MAAM,KAAK,GAAe;YACzB,OAAO,EAAE,CAAC;YACV,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC;YACtD,YAAY,EAAE,EAAE,CAAC,mBAAmB,CAAC,IAAI;YACzC,WAAW,EAAE,MAAM;SACnB,CAAC;QAEF,IAAI,CAAC,KAAK,MAAM,EAAE;YACjB,iDAAiD;YACjD,oDAAoD;YACpD,0DAA0D;YAC1D,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,QAAQ,CAAC;YACrC,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,WAAW,CAAC;SAC3C;QAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,IAAI,gBAAgB,CAAC;KAC3B;IAED,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AA/BW,QAAA,OAAO,WA+BlB","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert } from \"@fluidframework/common-utils\";\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport {\n\tDataCorruptionError,\n\textractSafePropertiesFromMessage,\n} from \"@fluidframework/container-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { ContainerMessageType, ContainerRuntimeMessage } from \"../containerRuntime\";\nimport { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from \"./definitions\";\n\n/**\n * Responsible for creating and reconstructing chunked messages.\n */\nexport class OpSplitter {\n\t// Local copy of incomplete received chunks.\n\tprivate readonly chunkMap: Map<string, string[]>;\n\tprivate readonly logger;\n\n\tconstructor(\n\t\tchunks: [string, string[]][],\n\t\tprivate readonly submitBatchFn: ((batch: IBatchMessage[]) => number) | undefined,\n\t\tprivate readonly chunkSizeInBytes: number,\n\t\tprivate readonly maxBatchSizeInBytes: number,\n\t\tlogger: ITelemetryLogger,\n\t) {\n\t\tthis.chunkMap = new Map<string, string[]>(chunks);\n\t\tthis.logger = ChildLogger.create(logger, \"OpSplitter\");\n\t}\n\n\tpublic get isBatchChunkingEnabled(): boolean {\n\t\treturn this.chunkSizeInBytes < Number.POSITIVE_INFINITY && this.submitBatchFn !== undefined;\n\t}\n\n\tpublic get chunks(): ReadonlyMap<string, string[]> {\n\t\treturn this.chunkMap;\n\t}\n\n\tpublic processRemoteMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {\n\t\tif (message.type !== ContainerMessageType.ChunkedOp) {\n\t\t\treturn {\n\t\t\t\tmessage,\n\t\t\t\tstate: \"Skipped\",\n\t\t\t};\n\t\t}\n\n\t\tconst clientId = message.clientId;\n\t\tconst chunkedContent = message.contents as IChunkedOp;\n\t\tthis.addChunk(clientId, chunkedContent, message);\n\n\t\tif (chunkedContent.chunkId < chunkedContent.totalChunks) {\n\t\t\t// We are processing the op in chunks but haven't reached\n\t\t\t// the last chunk yet in order to reconstruct the original op\n\t\t\treturn {\n\t\t\t\tmessage,\n\t\t\t\tstate: \"Accepted\",\n\t\t\t};\n\t\t}\n\n\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\tconst serializedContent = this.chunkMap.get(clientId)!.join(\"\");\n\t\tthis.clearPartialChunks(clientId);\n\n\t\tconst newMessage = { ...message };\n\t\tnewMessage.contents = serializedContent === \"\" ? undefined : JSON.parse(serializedContent);\n\t\tnewMessage.type = chunkedContent.originalType;\n\t\tnewMessage.metadata = chunkedContent.originalMetadata;\n\t\tnewMessage.compression = chunkedContent.originalCompression;\n\t\treturn {\n\t\t\tmessage: newMessage,\n\t\t\tstate: \"Processed\",\n\t\t};\n\t}\n\n\tpublic clearPartialChunks(clientId: string) {\n\t\tif (this.chunkMap.has(clientId)) {\n\t\t\tthis.chunkMap.delete(clientId);\n\t\t}\n\t}\n\n\tprivate addChunk(\n\t\tclientId: string,\n\t\tchunkedContent: IChunkedOp,\n\t\toriginalMessage: ISequencedDocumentMessage,\n\t) {\n\t\tlet map = this.chunkMap.get(clientId);\n\t\tif (map === undefined) {\n\t\t\tmap = [];\n\t\t\tthis.chunkMap.set(clientId, map);\n\t\t}\n\n\t\tif (chunkedContent.chunkId !== map.length + 1) {\n\t\t\t// We are expecting the chunks to be processed sequentially, in the same order as they are sent.\n\t\t\t// Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)\n\t\t\t// holding the existing chunks for that particular clientId.\n\t\t\tthrow new DataCorruptionError(\"Chunk Id mismatch\", {\n\t\t\t\t...extractSafePropertiesFromMessage(originalMessage),\n\t\t\t\tchunkMapLength: map.length,\n\t\t\t\tchunkId: chunkedContent.chunkId,\n\t\t\t\ttotalChunks: chunkedContent.totalChunks,\n\t\t\t});\n\t\t}\n\n\t\tmap.push(chunkedContent.contents);\n\t}\n\n\t/**\n\t * Splits the first op of a compressed batch in chunks, sends the chunks separately and\n\t * returns a new batch composed of the last chunk and the rest of the ops in the original batch.\n\t *\n\t * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops\n\t * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original\n\t * uncompressed ops at ingestion in the runtime.\n\t *\n\t * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire\n\t * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.\n\t *\n\t * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch\n\t * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch\n\t * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed\n\t * (as it is the last chunk).\n\t *\n\t * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.\n\t * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.\n\t *\n\t * @param batch - the compressed batch which needs to be processed\n\t * @returns A new adjusted batch which can be sent over the wire\n\t */\n\tpublic splitCompressedBatch(batch: IBatch): IBatch {\n\t\tassert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);\n\t\tassert(\n\t\t\tbatch.contentSizeInBytes > 0 && batch.content.length > 0,\n\t\t\t0x514 /* Batch needs to be non-empty */,\n\t\t);\n\t\tassert(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);\n\t\tassert(\n\t\t\tthis.chunkSizeInBytes < this.maxBatchSizeInBytes,\n\t\t\t0x516 /* Chunk size needs to be smaller than the max batch size */,\n\t\t);\n\n\t\tconst firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split\n\t\tassert(\n\t\t\tfirstMessage.metadata?.compressed === true || firstMessage.compression !== undefined,\n\t\t\t0x517 /* Batch needs to be compressed */,\n\t\t);\n\t\tassert(\n\t\t\t(firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,\n\t\t\t0x518 /* First message in the batch needs to be chunkable */,\n\t\t);\n\n\t\tconst restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers\n\t\tconst chunks = splitOp(firstMessage, this.chunkSizeInBytes);\n\n\t\tassert(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);\n\t\t// Send the first N-1 chunks immediately\n\t\tfor (const chunk of chunks.slice(0, -1)) {\n\t\t\tthis.submitBatchFn([chunkToBatchMessage(chunk, firstMessage.referenceSequenceNumber)]);\n\t\t}\n\n\t\t// The last chunk will be part of the new batch and needs to\n\t\t// preserve the batch metadata of the original batch\n\t\tconst lastChunk = chunkToBatchMessage(\n\t\t\tchunks[chunks.length - 1],\n\t\t\tfirstMessage.referenceSequenceNumber,\n\t\t\t{ batch: firstMessage.metadata?.batch },\n\t\t);\n\n\t\tthis.logger.sendPerformanceEvent({\n\t\t\teventName: \"Chunked compressed batch\",\n\t\t\tlength: batch.content.length,\n\t\t\tsizeInBytes: batch.contentSizeInBytes,\n\t\t\tchunks: chunks.length,\n\t\t\tchunkSizeInBytes: this.chunkSizeInBytes,\n\t\t});\n\n\t\treturn {\n\t\t\tcontent: [lastChunk, ...restOfMessages],\n\t\t\tcontentSizeInBytes: lastChunk.contents?.length ?? 0,\n\t\t};\n\t}\n}\n\nconst chunkToBatchMessage = (\n\tchunk: IChunkedOp,\n\treferenceSequenceNumber: number,\n\tmetadata: Record<string, unknown> | undefined = undefined,\n): BatchMessage => {\n\tconst payload: ContainerRuntimeMessage = {\n\t\ttype: ContainerMessageType.ChunkedOp,\n\t\tcontents: chunk,\n\t};\n\treturn {\n\t\tcontents: JSON.stringify(payload),\n\t\tdeserializedContent: payload,\n\t\tmetadata,\n\t\tlocalOpMetadata: undefined,\n\t\treferenceSequenceNumber,\n\t};\n};\n\nexport const splitOp = (op: BatchMessage, chunkSizeInBytes: number): IChunkedOp[] => {\n\tconst chunks: IChunkedOp[] = [];\n\tassert(\n\t\top.contents !== undefined && op.contents !== null,\n\t\t0x51a /* We should have something to chunk */,\n\t);\n\n\tconst contentLength = op.contents.length;\n\tconst chunkN = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1;\n\tlet offset = 0;\n\tfor (let i = 1; i <= chunkN; i++) {\n\t\tconst chunk: IChunkedOp = {\n\t\t\tchunkId: i,\n\t\t\tcontents: op.contents.substr(offset, chunkSizeInBytes),\n\t\t\toriginalType: op.deserializedContent.type,\n\t\t\ttotalChunks: chunkN,\n\t\t};\n\n\t\tif (i === chunkN) {\n\t\t\t// We don't need to port these to all the chunks,\n\t\t\t// as we rebuild the original op when we process the\n\t\t\t// last chunk, therefore it is the only one that needs it.\n\t\t\tchunk.originalMetadata = op.metadata;\n\t\t\tchunk.originalCompression = op.compression;\n\t\t}\n\n\t\tchunks.push(chunk);\n\t\toffset += chunkSizeInBytes;\n\t}\n\n\treturn chunks;\n};\n"]}
+{"version":3,"file":"opSplitter.js","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAGH,+DAAsD;AAEtD,qEAGyC;AAEzC,qEAA8D;AAC9D,0DAAoF;AACpF,iDAAoD;AAGpD;;GAEG;AACH,MAAa,UAAU;IAKtB,YACC,MAA4B,EACX,aAEL,EACI,gBAAwB,EACvB,mBAA2B,EAC5C,MAAwB;QALP,kBAAa,GAAb,aAAa,CAElB;QACI,qBAAgB,GAAhB,gBAAgB,CAAQ;QACvB,wBAAmB,GAAnB,mBAAmB,CAAQ;QAG5C,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAmB,MAAM,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACxD,CAAC;IAED,IAAW,sBAAsB;QAChC,OAAO,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS,CAAC;IAC7F,CAAC;IAED,IAAW,MAAM;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAEM,oBAAoB,CAAC,OAAkC;QAC7D,IAAI,OAAO,CAAC,IAAI,KAAK,uCAAoB,CAAC,SAAS,EAAE;YACpD,OAAO;gBACN,OAAO;gBACP,KAAK,EAAE,SAAS;aAChB,CAAC;SACF;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,MAAM,cAAc,GAAG,OAAO,CAAC,QAAsB,CAAC;QACtD,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QAEjD,IAAI,cAAc,CAAC,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE;YACxD,yDAAyD;YACzD,6DAA6D;YAC7D,OAAO;gBACN,OAAO;gBACP,KAAK,EAAE,UAAU;aACjB,CAAC;SACF;QAED,oEAAoE;QACpE,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAElC,MAAM,UAAU,qBAAQ,OAAO,CAAE,CAAC;QAClC,UAAU,CAAC,QAAQ,GAAG,iBAAiB,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAC3F,UAAU,CAAC,IAAI,GAAG,cAAc,CAAC,YAAY,CAAC;QAC9C,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC,gBAAgB,CAAC;QACtD,UAAU,CAAC,WAAW,GAAG,cAAc,CAAC,mBAAmB,CAAC;QAC5D,OAAO;YACN,OAAO,EAAE,UAAU;YACnB,KAAK,EAAE,WAAW;SAClB,CAAC;IACH,CAAC;IAEM,kBAAkB,CAAC,QAAgB;QACzC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAChC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;SAC/B;IACF,CAAC;IAEO,QAAQ,CACf,QAAgB,EAChB,cAA0B,EAC1B,eAA0C;QAE1C,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,GAAG,KAAK,SAAS,EAAE;YACtB,GAAG,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;SACjC;QAED,IAAI,cAAc,CAAC,OAAO,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;YAC9C,gGAAgG;YAChG,sGAAsG;YACtG,4DAA4D;YAC5D,MAAM,IAAI,qCAAmB,CAAC,mBAAmB,kCAC7C,IAAA,kDAAgC,EAAC,eAAe,CAAC,KACpD,cAAc,EAAE,GAAG,CAAC,MAAM,EAC1B,OAAO,EAAE,cAAc,CAAC,OAAO,EAC/B,WAAW,EAAE,cAAc,CAAC,WAAW,IACtC,CAAC;SACH;QAED,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACI,oBAAoB,CAAC,KAAa;;QACxC,IAAA,qBAAM,EAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QAC9E,IAAA,qBAAM,EACL,KAAK,CAAC,kBAAkB,GAAG,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EACxD,KAAK,CAAC,iCAAiC,CACvC,CAAC;QACF,IAAA,qBAAM,EACL,KAAK,CAAC,uBAAuB,KAAK,SAAS,EAC3C,KAAK,CAAC,8DAA8D,CACpE,CAAC;QACF,IAAA,qBAAM,EAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACjF,IAAA,qBAAM,EACL,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,mBAAmB,EAChD,KAAK,CAAC,4DAA4D,CAClE,CAAC;QAEF,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,wEAAwE;QAC/G,IAAA,qBAAM,EACL,CAAA,MAAA,YAAY,CAAC,QAAQ,0CAAE,UAAU,MAAK,IAAI,IAAI,YAAY,CAAC,WAAW,KAAK,SAAS,EACpF,KAAK,CAAC,kCAAkC,CACxC,CAAC;QACF,IAAA,qBAAM,EACL,CAAC,MAAA,MAAA,YAAY,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAC7D,KAAK,CAAC,sDAAsD,CAC5D,CAAC;QAEF,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,uEAAuE;QACtH,MAAM,UAAU,GAAG,IAAA,iCAAkB,EAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAA,eAAO,EACrB,YAAY,EACZ,IAAI,CAAC,gBAAgB;QACrB,wEAAwE;QACxE,gFAAgF;QAChF,yDAAyD;QACzD,UAAU,IAAI,IAAI,CAAC,mBAAmB,CACtC,CAAC;QAEF,IAAA,qBAAM,EAAC,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACnF,wCAAwC;QACxC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;YACxC,IAAI,CAAC,aAAa,CACjB,CAAC,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC,uBAAuB,CAAC,CAAC,EAC3D,KAAK,CAAC,uBAAuB,CAC7B,CAAC;SACF;QAED,4DAA4D;QAC5D,oDAAoD;QACpD,MAAM,SAAS,GAAG,mBAAmB,CACpC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EACzB,KAAK,CAAC,uBAAuB,EAC7B,EAAE,KAAK,EAAE,MAAA,YAAY,CAAC,QAAQ,0CAAE,KAAK,EAAE,CACvC,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;YAChC,wCAAwC;YACxC,SAAS,EAAE,wBAAwB;YACnC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;YAC5B,WAAW,EAAE,KAAK,CAAC,kBAAkB;YACrC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,UAAU;SACV,CAAC,CAAC;QAEH,OAAO;YACN,OAAO,EAAE,CAAC,SAAS,EAAE,GAAG,cAAc,CAAC;YACvC,kBAAkB,EAAE,MAAA,MAAA,SAAS,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC;YACnD,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;SACtD,CAAC;IACH,CAAC;CACD;AA1LD,gCA0LC;AAED,MAAM,mBAAmB,GAAG,CAC3B,KAAiB,EACjB,uBAA+B,EAC/B,WAAgD,SAAS,EAC1C,EAAE;IACjB,MAAM,OAAO,GAA4B;QACxC,IAAI,EAAE,uCAAoB,CAAC,SAAS;QACpC,QAAQ,EAAE,KAAK;KACf,CAAC;IACF,OAAO;QACN,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QACjC,mBAAmB,EAAE,OAAO;QAC5B,QAAQ;QACR,eAAe,EAAE,SAAS;QAC1B,uBAAuB;KACvB,CAAC;AACH,CAAC,CAAC;AAEF;;;;;;;;;;;GAWG;AACI,MAAM,OAAO,GAAG,CACtB,EAAgB,EAChB,gBAAwB,EACxB,UAAmB,KAAK,EACT,EAAE;IACjB,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAA,qBAAM,EACL,EAAE,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,CAAC,QAAQ,KAAK,IAAI,EACjD,KAAK,CAAC,uCAAuC,CAC7C,CAAC;IAEF,MAAM,aAAa,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9F,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE;QACvD,MAAM,KAAK,GAAe;YACzB,OAAO;YACP,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC;YACtD,YAAY,EAAE,EAAE,CAAC,mBAAmB,CAAC,IAAI;YACzC,WAAW,EAAE,UAAU;SACvB,CAAC;QAEF,IAAI,OAAO,KAAK,UAAU,EAAE;YAC3B,iDAAiD;YACjD,oDAAoD;YACpD,0DAA0D;YAC1D,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,QAAQ,CAAC;YACrC,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,WAAW,CAAC;SAC3C;QAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,IAAI,gBAAgB,CAAC;QAC3B,IAAA,qBAAM,EACL,OAAO,IAAI,UAAU,GAAG,CAAC,IAAI,MAAM,IAAI,aAAa,EACpD,KAAK,CAAC,kCAAkC,CACxC,CAAC;KACF;IAED,IAAA,qBAAM,EAAC,MAAM,IAAI,aAAa,EAAE,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAChG,IAAA,qBAAM,EAAC,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,KAAK,CAAC,+BAA+B,CAAC,CAAC;IAC5E,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAzCW,QAAA,OAAO,WAyClB","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert } from \"@fluidframework/common-utils\";\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport {\n\tDataCorruptionError,\n\textractSafePropertiesFromMessage,\n} from \"@fluidframework/container-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { ContainerMessageType, ContainerRuntimeMessage } from \"../containerRuntime\";\nimport { estimateSocketSize } from \"./batchManager\";\nimport { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from \"./definitions\";\n\n/**\n * Responsible for creating and reconstructing chunked messages.\n */\nexport class OpSplitter {\n\t// Local copy of incomplete received chunks.\n\tprivate readonly chunkMap: Map<string, string[]>;\n\tprivate readonly logger;\n\n\tconstructor(\n\t\tchunks: [string, string[]][],\n\t\tprivate readonly submitBatchFn:\n\t\t\t| ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number)\n\t\t\t| undefined,\n\t\tpublic readonly chunkSizeInBytes: number,\n\t\tprivate readonly maxBatchSizeInBytes: number,\n\t\tlogger: ITelemetryLogger,\n\t) {\n\t\tthis.chunkMap = new Map<string, string[]>(chunks);\n\t\tthis.logger = ChildLogger.create(logger, \"OpSplitter\");\n\t}\n\n\tpublic get isBatchChunkingEnabled(): boolean {\n\t\treturn this.chunkSizeInBytes < Number.POSITIVE_INFINITY && this.submitBatchFn !== undefined;\n\t}\n\n\tpublic get chunks(): ReadonlyMap<string, string[]> {\n\t\treturn this.chunkMap;\n\t}\n\n\tpublic processRemoteMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {\n\t\tif (message.type !== ContainerMessageType.ChunkedOp) {\n\t\t\treturn {\n\t\t\t\tmessage,\n\t\t\t\tstate: \"Skipped\",\n\t\t\t};\n\t\t}\n\n\t\tconst clientId = message.clientId;\n\t\tconst chunkedContent = message.contents as IChunkedOp;\n\t\tthis.addChunk(clientId, chunkedContent, message);\n\n\t\tif (chunkedContent.chunkId < chunkedContent.totalChunks) {\n\t\t\t// We are processing the op in chunks but haven't reached\n\t\t\t// the last chunk yet in order to reconstruct the original op\n\t\t\treturn {\n\t\t\t\tmessage,\n\t\t\t\tstate: \"Accepted\",\n\t\t\t};\n\t\t}\n\n\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\tconst serializedContent = this.chunkMap.get(clientId)!.join(\"\");\n\t\tthis.clearPartialChunks(clientId);\n\n\t\tconst newMessage = { ...message };\n\t\tnewMessage.contents = serializedContent === \"\" ? undefined : JSON.parse(serializedContent);\n\t\tnewMessage.type = chunkedContent.originalType;\n\t\tnewMessage.metadata = chunkedContent.originalMetadata;\n\t\tnewMessage.compression = chunkedContent.originalCompression;\n\t\treturn {\n\t\t\tmessage: newMessage,\n\t\t\tstate: \"Processed\",\n\t\t};\n\t}\n\n\tpublic clearPartialChunks(clientId: string) {\n\t\tif (this.chunkMap.has(clientId)) {\n\t\t\tthis.chunkMap.delete(clientId);\n\t\t}\n\t}\n\n\tprivate addChunk(\n\t\tclientId: string,\n\t\tchunkedContent: IChunkedOp,\n\t\toriginalMessage: ISequencedDocumentMessage,\n\t) {\n\t\tlet map = this.chunkMap.get(clientId);\n\t\tif (map === undefined) {\n\t\t\tmap = [];\n\t\t\tthis.chunkMap.set(clientId, map);\n\t\t}\n\n\t\tif (chunkedContent.chunkId !== map.length + 1) {\n\t\t\t// We are expecting the chunks to be processed sequentially, in the same order as they are sent.\n\t\t\t// Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)\n\t\t\t// holding the existing chunks for that particular clientId.\n\t\t\tthrow new DataCorruptionError(\"Chunk Id mismatch\", {\n\t\t\t\t...extractSafePropertiesFromMessage(originalMessage),\n\t\t\t\tchunkMapLength: map.length,\n\t\t\t\tchunkId: chunkedContent.chunkId,\n\t\t\t\ttotalChunks: chunkedContent.totalChunks,\n\t\t\t});\n\t\t}\n\n\t\tmap.push(chunkedContent.contents);\n\t}\n\n\t/**\n\t * Splits the first op of a compressed batch in chunks, sends the chunks separately and\n\t * returns a new batch composed of the last chunk and the rest of the ops in the original batch.\n\t *\n\t * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops\n\t * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original\n\t * uncompressed ops at ingestion in the runtime.\n\t *\n\t * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire\n\t * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.\n\t *\n\t * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch\n\t * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch\n\t * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed\n\t * (as it is the last chunk).\n\t *\n\t * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.\n\t * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.\n\t *\n\t * @param batch - the compressed batch which needs to be processed\n\t * @returns A new adjusted batch which can be sent over the wire\n\t */\n\tpublic splitCompressedBatch(batch: IBatch): IBatch {\n\t\tassert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);\n\t\tassert(\n\t\t\tbatch.contentSizeInBytes > 0 && batch.content.length > 0,\n\t\t\t0x514 /* Batch needs to be non-empty */,\n\t\t);\n\t\tassert(\n\t\t\tbatch.referenceSequenceNumber !== undefined,\n\t\t\t0x58a /* Batch must have a reference sequence number if non-empty */,\n\t\t);\n\t\tassert(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);\n\t\tassert(\n\t\t\tthis.chunkSizeInBytes < this.maxBatchSizeInBytes,\n\t\t\t0x516 /* Chunk size needs to be smaller than the max batch size */,\n\t\t);\n\n\t\tconst firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split\n\t\tassert(\n\t\t\tfirstMessage.metadata?.compressed === true || firstMessage.compression !== undefined,\n\t\t\t0x517 /* Batch needs to be compressed */,\n\t\t);\n\t\tassert(\n\t\t\t(firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,\n\t\t\t0x518 /* First message in the batch needs to be chunkable */,\n\t\t);\n\n\t\tconst restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers\n\t\tconst socketSize = estimateSocketSize(batch);\n\t\tconst chunks = splitOp(\n\t\t\tfirstMessage,\n\t\t\tthis.chunkSizeInBytes,\n\t\t\t// If we estimate that the socket batch size will exceed the batch limit\n\t\t\t// we will inject an empty op to minimize the risk of the payload failing due to\n\t\t\t// the overhead from the trailing empty ops in the batch.\n\t\t\tsocketSize >= this.maxBatchSizeInBytes,\n\t\t);\n\n\t\tassert(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);\n\t\t// Send the first N-1 chunks immediately\n\t\tfor (const chunk of chunks.slice(0, -1)) {\n\t\t\tthis.submitBatchFn(\n\t\t\t\t[chunkToBatchMessage(chunk, batch.referenceSequenceNumber)],\n\t\t\t\tbatch.referenceSequenceNumber,\n\t\t\t);\n\t\t}\n\n\t\t// The last chunk will be part of the new batch and needs to\n\t\t// preserve the batch metadata of the original batch\n\t\tconst lastChunk = chunkToBatchMessage(\n\t\t\tchunks[chunks.length - 1],\n\t\t\tbatch.referenceSequenceNumber,\n\t\t\t{ batch: firstMessage.metadata?.batch },\n\t\t);\n\n\t\tthis.logger.sendPerformanceEvent({\n\t\t\t// Used to be \"Chunked compressed batch\"\n\t\t\teventName: \"CompressedChunkedBatch\",\n\t\t\tlength: batch.content.length,\n\t\t\tsizeInBytes: batch.contentSizeInBytes,\n\t\t\tchunks: chunks.length,\n\t\t\tchunkSizeInBytes: this.chunkSizeInBytes,\n\t\t\tsocketSize,\n\t\t});\n\n\t\treturn {\n\t\t\tcontent: [lastChunk, ...restOfMessages],\n\t\t\tcontentSizeInBytes: lastChunk.contents?.length ?? 0,\n\t\t\treferenceSequenceNumber: batch.referenceSequenceNumber,\n\t\t};\n\t}\n}\n\nconst chunkToBatchMessage = (\n\tchunk: IChunkedOp,\n\treferenceSequenceNumber: number,\n\tmetadata: Record<string, unknown> | undefined = undefined,\n): BatchMessage => {\n\tconst payload: ContainerRuntimeMessage = {\n\t\ttype: ContainerMessageType.ChunkedOp,\n\t\tcontents: chunk,\n\t};\n\treturn {\n\t\tcontents: JSON.stringify(payload),\n\t\tdeserializedContent: payload,\n\t\tmetadata,\n\t\tlocalOpMetadata: undefined,\n\t\treferenceSequenceNumber,\n\t};\n};\n\n/**\n * Splits an op into smaller ops (chunks), based on the size of the op and the `chunkSizeInBytes` parameter.\n *\n * The last op of the result will be bundled with empty ops in the same batch. There is a risk of the batch payload\n * exceeding the 1MB limit due to the overhead from the empty ops. If the last op is large, the risk is even higher.\n * To minimize the odds, an extra empty op can be added to the result using the `extraOp` parameter.\n *\n * @param op - the op to be split\n * @param chunkSizeInBytes - how large should the chunks be\n * @param extraOp - should an extra empty op be added to the result\n * @returns an array of chunked ops\n */\nexport const splitOp = (\n\top: BatchMessage,\n\tchunkSizeInBytes: number,\n\textraOp: boolean = false,\n): IChunkedOp[] => {\n\tconst chunks: IChunkedOp[] = [];\n\tassert(\n\t\top.contents !== undefined && op.contents !== null,\n\t\t0x51a /* We should have something to chunk */,\n\t);\n\n\tconst contentLength = op.contents.length;\n\tconst chunkCount = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1 + (extraOp ? 1 : 0);\n\tlet offset = 0;\n\tfor (let chunkId = 1; chunkId <= chunkCount; chunkId++) {\n\t\tconst chunk: IChunkedOp = {\n\t\t\tchunkId,\n\t\t\tcontents: op.contents.substr(offset, chunkSizeInBytes),\n\t\t\toriginalType: op.deserializedContent.type,\n\t\t\ttotalChunks: chunkCount,\n\t\t};\n\n\t\tif (chunkId === chunkCount) {\n\t\t\t// We don't need to port these to all the chunks,\n\t\t\t// as we rebuild the original op when we process the\n\t\t\t// last chunk, therefore it is the only one that needs it.\n\t\t\tchunk.originalMetadata = op.metadata;\n\t\t\tchunk.originalCompression = op.compression;\n\t\t}\n\n\t\tchunks.push(chunk);\n\t\toffset += chunkSizeInBytes;\n\t\tassert(\n\t\t\tchunkId >= chunkCount - 1 || offset <= contentLength,\n\t\t\t0x58b /* Content offset within bounds */,\n\t\t);\n\t}\n\n\tassert(offset >= contentLength, 0x58c /* Content offset equal or larger than content length */);\n\tassert(chunks.length === chunkCount, 0x5a5 /* Expected number of chunks */);\n\treturn chunks;\n};\n"]}

package/dist/opLifecycle/outbox.d.ts

@@ -12,7 +12,7 @@ import { OpSplitter } from "./opSplitter";
 export interface IOutboxConfig {
     readonly compressionOptions: ICompressionRuntimeOptions;
     readonly maxBatchSizeInBytes: number;
-    readonly enableOpReentryCheck?: boolean;
+    readonly disablePartialFlush: boolean;
 }
 export interface IOutboxParameters {
     readonly shouldSend: () => boolean;
@@ -25,11 +25,29 @@ export interface IOutboxParameters {
 }
 export declare class Outbox {
     private readonly params;
+    private readonly mc;
     private readonly attachFlowBatch;
     private readonly mainBatch;
     private readonly defaultAttachFlowSoftLimitInBytes;
+    /**
+     * Track the number of ops which were detected to have a mismatched
+     * reference sequence number, in order to self-throttle the telemetry events.
+     *
+     * This should be removed as part of ADO:2322
+     */
+    private readonly maxMismatchedOpsToReport;
+    private mismatchedOpsReported;
     constructor(params: IOutboxParameters);
     get isEmpty(): boolean;
+    /**
+     * If we detect that the reference sequence number of the incoming message does not match
+     * what was already in the batch managers, this means that batching has been interrupted so
+     * we will flush the accumulated messages to account for that and create a new batch with the new
+     * message as the first message.
+     *
+     * @param message - the incoming message
+     */
+    private maybeFlushPartialBatch;
     submit(message: BatchMessage): void;
     submitAttach(message: BatchMessage): void;
     flush(): void;
@@ -39,8 +57,6 @@ export declare class Outbox {
      * Sends the batch object to the container context to be sent over the wire.
      *
      * @param batch - batch to be sent
-     * @returns the client sequence number of the last batched op which was sent and
-     * -1 if there are no ops or the container cannot send ops.
      */
     private sendBatch;
     private persistBatch;

package/dist/opLifecycle/outbox.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"outbox.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/outbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAEtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,uCAAuC,CAAC;AAG1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,OAAO,EAAE,YAAY,EAAU,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,0BAA0B,CAAC;IAExD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,OAAO,CAAC;CACxC;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,OAAO,CAAC;IACnC,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;IAClD,QAAQ,CAAC,gBAAgB,EAAE,iBAAiB,CAAC;IAC7C,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,YAAY,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;CAClC;AAED,qBAAa,MAAM;IAKN,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJnC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAa;gBAElC,MAAM,EAAE,iBAAiB;IAyBtD,IAAW,OAAO,IAAI,OAAO,CAE5B;IAEM,MAAM,CAAC,OAAO,EAAE,YAAY;IAW5B,YAAY,CAAC,OAAO,EAAE,YAAY;IA6BlC,KAAK;IAKZ,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,aAAa;IA+BrB;;;;;;OAMG;IACH,OAAO,CAAC,SAAS;IA6CjB,OAAO,CAAC,YAAY;IAkBb,UAAU;;;;CAMjB"}
+{"version":3,"file":"outbox.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/outbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAEtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,uCAAuC,CAAC;AAQ1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,OAAO,EAAE,YAAY,EAAU,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,0BAA0B,CAAC;IAExD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,OAAO,CAAC;IACnC,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;IAClD,QAAQ,CAAC,gBAAgB,EAAE,iBAAiB,CAAC;IAC7C,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,YAAY,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;CAClC;AAED,qBAAa,MAAM;IAeN,OAAO,CAAC,QAAQ,CAAC,MAAM;IAdnC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAc;IAEhE;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAK;IAC9C,OAAO,CAAC,qBAAqB,CAAK;gBAEL,MAAM,EAAE,iBAAiB;IAatD,IAAW,OAAO,IAAI,OAAO,CAE5B;IAED;;;;;;;OAOG;IACH,OAAO,CAAC,sBAAsB;IAsCvB,MAAM,CAAC,OAAO,EAAE,YAAY;IAa5B,YAAY,CAAC,OAAO,EAAE,YAAY;IA+BlC,KAAK;IAKZ,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,aAAa;IAmCrB;;;;OAIG;IACH,OAAO,CAAC,SAAS;IAsDjB,OAAO,CAAC,YAAY;IAcb,UAAU;;;;CAMjB"}

package/dist/opLifecycle/outbox.js

@@ -8,31 +8,69 @@ exports.Outbox = void 0;
 const common_utils_1 = require("@fluidframework/common-utils");
 const container_utils_1 = require("@fluidframework/container-utils");
 const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
+const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
 const batchManager_1 = require("./batchManager");
 class Outbox {
     constructor(params) {
         this.params = params;
-        this.defaultAttachFlowSoftLimitInBytes = 64 * 1024;
+        this.defaultAttachFlowSoftLimitInBytes = 320 * 1024;
+        /**
+         * Track the number of ops which were detected to have a mismatched
+         * reference sequence number, in order to self-throttle the telemetry events.
+         *
+         * This should be removed as part of ADO:2322
+         */
+        this.maxMismatchedOpsToReport = 3;
+        this.mismatchedOpsReported = 0;
+        this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(params.logger, "Outbox"));
         const isCompressionEnabled = this.params.config.compressionOptions.minimumBatchSizeInBytes !==
             Number.POSITIVE_INFINITY;
         // We need to allow infinite size batches if we enable compression
         const hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;
         const softLimit = isCompressionEnabled ? Infinity : this.defaultAttachFlowSoftLimitInBytes;
-        this.attachFlowBatch = new batchManager_1.BatchManager({
-            hardLimit,
-            softLimit,
-            enableOpReentryCheck: params.config.enableOpReentryCheck,
-        }, params.logger);
-        this.mainBatch = new batchManager_1.BatchManager({
-            hardLimit,
-            enableOpReentryCheck: params.config.enableOpReentryCheck,
-        }, params.logger);
+        this.attachFlowBatch = new batchManager_1.BatchManager({ hardLimit, softLimit });
+        this.mainBatch = new batchManager_1.BatchManager({ hardLimit });
     }
     get isEmpty() {
         return this.attachFlowBatch.length === 0 && this.mainBatch.length === 0;
     }
+    /**
+     * If we detect that the reference sequence number of the incoming message does not match
+     * what was already in the batch managers, this means that batching has been interrupted so
+     * we will flush the accumulated messages to account for that and create a new batch with the new
+     * message as the first message.
+     *
+     * @param message - the incoming message
+     */
+    maybeFlushPartialBatch(message) {
+        const mainBatchReference = this.mainBatch.referenceSequenceNumber;
+        const attachFlowBatchReference = this.attachFlowBatch.referenceSequenceNumber;
+        (0, common_utils_1.assert)(this.params.config.disablePartialFlush ||
+            mainBatchReference === undefined ||
+            attachFlowBatchReference === undefined ||
+            mainBatchReference === attachFlowBatchReference, 0x58d /* Reference sequence numbers from both batches must be in sync */);
+        if ((mainBatchReference === undefined ||
+            mainBatchReference === message.referenceSequenceNumber) &&
+            (attachFlowBatchReference === undefined ||
+                attachFlowBatchReference === message.referenceSequenceNumber)) {
+            // The reference sequence numbers are stable, there is nothing to do
+            return;
+        }
+        if (++this.mismatchedOpsReported <= this.maxMismatchedOpsToReport) {
+            this.mc.logger.sendErrorEvent({
+                eventName: "ReferenceSequenceNumberMismatch",
+                mainReferenceSequenceNumber: mainBatchReference,
+                attachReferenceSequenceNumber: attachFlowBatchReference,
+                messageReferenceSequenceNumber: message.referenceSequenceNumber,
+            }, new container_utils_1.UsageError("Submission of an out of order message"));
+        }
+        if (!this.params.config.disablePartialFlush) {
+            this.flush();
+        }
+    }
     submit(message) {
         var _a, _b;
+        this.maybeFlushPartialBatch(message);
         if (!this.mainBatch.push(message)) {
             throw new container_utils_1.GenericError("BatchTooLarge", /* error */ undefined, {
                 opSize: (_b = (_a = message.contents) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0,
@@ -44,6 +82,7 @@ class Outbox {
     }
     submitAttach(message) {
         var _a, _b;
+        this.maybeFlushPartialBatch(message);
         if (!this.attachFlowBatch.push(message)) {
             // BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
             // when queue is not empty.
@@ -74,83 +113,83 @@ class Outbox {
     }
     flushInternal(rawBatch) {
         const processedBatch = this.compressBatch(rawBatch);
-        const clientSequenceNumber = this.sendBatch(processedBatch);
-        this.persistBatch(clientSequenceNumber, rawBatch.content);
+        this.sendBatch(processedBatch);
+        this.persistBatch(rawBatch.content);
     }
     compressBatch(batch) {
         if (batch.content.length === 0 ||
             this.params.config.compressionOptions === undefined ||
-            this.params.config.compressionOptions.minimumBatchSizeInBytes > batch.contentSizeInBytes) {
-            // Nothing to do if the batch is empty or if compression is disabled or if we don't need to compress
+            this.params.config.compressionOptions.minimumBatchSizeInBytes >
+                batch.contentSizeInBytes ||
+            this.params.containerContext.submitBatchFn === undefined) {
+            // Nothing to do if the batch is empty or if compression is disabled or not supported, or if we don't need to compress
             return batch;
         }
         const compressedBatch = this.params.compressor.compressBatch(batch);
-        if (compressedBatch.contentSizeInBytes <= this.params.config.maxBatchSizeInBytes) {
-            // If we don't reach the maximum supported size of a batch, it can safely be sent as is
-            return compressedBatch;
-        }
         if (this.params.splitter.isBatchChunkingEnabled) {
-            return this.params.splitter.splitCompressedBatch(compressedBatch);
+            return compressedBatch.contentSizeInBytes <= this.params.splitter.chunkSizeInBytes
+                ? compressedBatch
+                : this.params.splitter.splitCompressedBatch(compressedBatch);
+        }
+        if (compressedBatch.contentSizeInBytes >= this.params.config.maxBatchSizeInBytes) {
+            throw new container_utils_1.GenericError("BatchTooLarge", /* error */ undefined, {
+                batchSize: batch.contentSizeInBytes,
+                compressedBatchSize: compressedBatch.contentSizeInBytes,
+                count: compressedBatch.content.length,
+                limit: this.params.config.maxBatchSizeInBytes,
+                chunkingEnabled: this.params.splitter.isBatchChunkingEnabled,
+                compressionOptions: JSON.stringify(this.params.config.compressionOptions),
+                socketSize: (0, batchManager_1.estimateSocketSize)(batch),
+            });
         }
-        // If we've reached this point, the runtime would attempt to send a batch larger than the allowed size
-        throw new container_utils_1.GenericError("BatchTooLarge", /* error */ undefined, {
-            batchSize: batch.contentSizeInBytes,
-            compressedBatchSize: compressedBatch.contentSizeInBytes,
-            count: compressedBatch.content.length,
-            limit: this.params.config.maxBatchSizeInBytes,
-            chunkingEnabled: this.params.splitter.isBatchChunkingEnabled,
-            compressionOptions: JSON.stringify(this.params.config.compressionOptions),
-        });
+        return compressedBatch;
     }
     /**
      * Sends the batch object to the container context to be sent over the wire.
      *
      * @param batch - batch to be sent
-     * @returns the client sequence number of the last batched op which was sent and
-     * -1 if there are no ops or the container cannot send ops.
      */
     sendBatch(batch) {
-        var _a;
-        let clientSequenceNumber = -1;
         const length = batch.content.length;
         // Did we disconnect in the middle of turn-based batch?
         // If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
         if (length === 0 || !this.params.shouldSend()) {
-            return clientSequenceNumber;
+            return;
+        }
+        const socketSize = (0, batchManager_1.estimateSocketSize)(batch);
+        if (socketSize >= this.params.config.maxBatchSizeInBytes) {
+            this.mc.logger.sendPerformanceEvent({
+                eventName: "LargeBatch",
+                length: batch.content.length,
+                sizeInBytes: batch.contentSizeInBytes,
+                socketSize,
+            });
         }
         if (this.params.containerContext.submitBatchFn === undefined) {
             // Legacy path - supporting old loader versions. Can be removed only when LTS moves above
             // version that has support for batches (submitBatchFn)
+            (0, common_utils_1.assert)(batch.content[0].compression === undefined, 0x5a6 /* Compression should not have happened if the loader does not support it */);
             for (const message of batch.content) {
-                // Legacy path doesn't support compressed payloads and will submit uncompressed payload anyways
-                if ((_a = message.metadata) === null || _a === void 0 ? void 0 : _a.compressed) {
-                    delete message.metadata.compressed;
-                }
-                clientSequenceNumber = this.params.containerContext.submitFn(protocol_definitions_1.MessageType.Operation, message.deserializedContent, true, // batch
+                this.params.containerContext.submitFn(protocol_definitions_1.MessageType.Operation, message.deserializedContent, true, // batch
                 message.metadata);
             }
             this.params.containerContext.deltaManager.flush();
         }
         else {
-            // returns clientSequenceNumber of last message in a batch
-            clientSequenceNumber = this.params.containerContext.submitBatchFn(batch.content.map((message) => ({
+            (0, common_utils_1.assert)(batch.referenceSequenceNumber !== undefined, 0x58e /* Batch must not be empty */);
+            this.params.containerContext.submitBatchFn(batch.content.map((message) => ({
                 contents: message.contents,
                 metadata: message.metadata,
                 compression: message.compression,
-            })));
+                referenceSequenceNumber: message.referenceSequenceNumber,
+            })), batch.referenceSequenceNumber);
         }
-        // Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.
-        clientSequenceNumber -= length - 1;
-        (0, common_utils_1.assert)(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);
-        return clientSequenceNumber;
     }
-    persistBatch(initialClientSequenceNumber, batch) {
-        let clientSequenceNumber = initialClientSequenceNumber;
+    persistBatch(batch) {
         // Let the PendingStateManager know that a message was submitted.
         // In future, need to shift toward keeping batch as a whole!
         for (const message of batch) {
-            this.params.pendingStateManager.onSubmitMessage(message.deserializedContent.type, clientSequenceNumber, message.referenceSequenceNumber, message.deserializedContent.contents, message.localOpMetadata, message.metadata);
-            clientSequenceNumber++;
+            this.params.pendingStateManager.onSubmitMessage(message.deserializedContent.type, message.referenceSequenceNumber, message.deserializedContent.contents, message.localOpMetadata, message.metadata);
         }
     }
     checkpoint() {

package/dist/opLifecycle/outbox.js.map

@@ -1 +1 @@
-{"version":3,"file":"outbox.js","sourceRoot":"","sources":["../../src/opLifecycle/outbox.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAGH,+DAAsD;AAEtD,qEAA+D;AAC/D,+EAAmE;AAGnE,iDAA8C;AAsB9C,MAAa,MAAM;IAKlB,YAA6B,MAAyB;QAAzB,WAAM,GAAN,MAAM,CAAmB;QAFrC,sCAAiC,GAAG,EAAE,GAAG,IAAI,CAAC;QAG9D,MAAM,oBAAoB,GACzB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,uBAAuB;YAC7D,MAAM,CAAC,iBAAiB,CAAC;QAC1B,kEAAkE;QAClE,MAAM,SAAS,GAAG,oBAAoB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC;QAC3F,MAAM,SAAS,GAAG,oBAAoB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,iCAAiC,CAAC;QAE3F,IAAI,CAAC,eAAe,GAAG,IAAI,2BAAY,CACtC;YACC,SAAS;YACT,SAAS;YACT,oBAAoB,EAAE,MAAM,CAAC,MAAM,CAAC,oBAAoB;SACxD,EACD,MAAM,CAAC,MAAM,CACb,CAAC;QACF,IAAI,CAAC,SAAS,GAAG,IAAI,2BAAY,CAChC;YACC,SAAS;YACT,oBAAoB,EAAE,MAAM,CAAC,MAAM,CAAC,oBAAoB;SACxD,EACD,MAAM,CAAC,MAAM,CACb,CAAC;IACH,CAAC;IAED,IAAW,OAAO;QACjB,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC;IACzE,CAAC;IAEM,MAAM,CAAC,OAAqB;;QAClC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YAClC,MAAM,IAAI,8BAAY,CAAC,eAAe,EAAE,WAAW,CAAC,SAAS,EAAE;gBAC9D,MAAM,EAAE,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC;gBACrC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB;gBAC5C,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM;gBAC5B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS;aACvC,CAAC,CAAC;SACH;IACF,CAAC;IAEM,YAAY,CAAC,OAAqB;;QACxC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YACxC,oFAAoF;YACpF,2BAA2B;YAC3B,8FAA8F;YAC9F,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;gBACxC,MAAM,IAAI,8BAAY,CAAC,eAAe,EAAE,WAAW,CAAC,SAAS,EAAE;oBAC9D,MAAM,EAAE,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC;oBACrC,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,kBAAkB;oBAClD,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;oBAClC,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS;iBAC7C,CAAC,CAAC;aACH;SACD;QAED,iEAAiE;QACjE,yEAAyE;QACzE,2DAA2D;QAC3D,sEAAsE;QACtE,gEAAgE;QAChE,IACC,IAAI,CAAC,eAAe,CAAC,kBAAkB;YACvC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,uBAAuB,EAC5D;YACD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;SACpD;IACF,CAAC;IAEM,KAAK;QACX,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC/C,CAAC;IAEO,aAAa,CAAC,QAAgB;QACrC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,oBAAoB,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QAE5D,IAAI,CAAC,YAAY,CAAC,oBAAoB,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC3D,CAAC;IAEO,aAAa,CAAC,KAAa;QAClC,IACC,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,KAAK,SAAS;YACnD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,uBAAuB,GAAG,KAAK,CAAC,kBAAkB,EACvF;YACD,oGAAoG;YACpG,OAAO,KAAK,CAAC;SACb;QAED,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACpE,IAAI,eAAe,CAAC,kBAAkB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,mBAAmB,EAAE;YACjF,uFAAuF;YACvF,OAAO,eAAe,CAAC;SACvB;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,EAAE;YAChD,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;SAClE;QAED,sGAAsG;QACtG,MAAM,IAAI,8BAAY,CAAC,eAAe,EAAE,WAAW,CAAC,SAAS,EAAE;YAC9D,SAAS,EAAE,KAAK,CAAC,kBAAkB;YACnC,mBAAmB,EAAE,eAAe,CAAC,kBAAkB;YACvD,KAAK,EAAE,eAAe,CAAC,OAAO,CAAC,MAAM;YACrC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,mBAAmB;YAC7C,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB;YAC5D,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC;SACzE,CAAC,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACK,SAAS,CAAC,KAAa;;QAC9B,IAAI,oBAAoB,GAAW,CAAC,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAEpC,uDAAuD;QACvD,uFAAuF;QACvF,IAAI,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE;YAC9C,OAAO,oBAAoB,CAAC;SAC5B;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,aAAa,KAAK,SAAS,EAAE;YAC7D,yFAAyF;YACzF,uDAAuD;YACvD,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE;gBACpC,+FAA+F;gBAC/F,IAAI,MAAA,OAAO,CAAC,QAAQ,0CAAE,UAAU,EAAE;oBACjC,OAAO,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;iBACnC;gBAED,oBAAoB,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,QAAQ,CAC3D,kCAAW,CAAC,SAAS,EACrB,OAAO,CAAC,mBAAmB,EAC3B,IAAI,EAAE,QAAQ;gBACd,OAAO,CAAC,QAAQ,CAChB,CAAC;aACF;YAED,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;SAClD;aAAM;YACN,0DAA0D;YAC1D,oBAAoB,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,aAAa,CAChE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC/B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;aAChC,CAAC,CAAC,CACH,CAAC;SACF;QAED,2GAA2G;QAC3G,oBAAoB,IAAI,MAAM,GAAG,CAAC,CAAC;QACnC,IAAA,qBAAM,EAAC,oBAAoB,IAAI,CAAC,EAAE,KAAK,CAAC,4CAA4C,CAAC,CAAC;QACtF,OAAO,oBAAoB,CAAC;IAC7B,CAAC;IAEO,YAAY,CAAC,2BAAmC,EAAE,KAAqB;QAC9E,IAAI,oBAAoB,GAAG,2BAA2B,CAAC;QACvD,iEAAiE;QACjE,4DAA4D;QAC5D,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE;YAC5B,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,eAAe,CAC9C,OAAO,CAAC,mBAAmB,CAAC,IAAI,EAChC,oBAAoB,EACpB,OAAO,CAAC,uBAAuB,EAC/B,OAAO,CAAC,mBAAmB,CAAC,QAAQ,EACpC,OAAO,CAAC,eAAe,EACvB,OAAO,CAAC,QAAQ,CAChB,CAAC;YAEF,oBAAoB,EAAE,CAAC;SACvB;IACF,CAAC;IAEM,UAAU;QAChB,OAAO;YACN,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE;YACtC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE;SAClD,CAAC;IACH,CAAC;CACD;AAjMD,wBAiMC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert } from \"@fluidframework/common-utils\";\nimport { IContainerContext } from \"@fluidframework/container-definitions\";\nimport { GenericError } from \"@fluidframework/container-utils\";\nimport { MessageType } from \"@fluidframework/protocol-definitions\";\nimport { ICompressionRuntimeOptions } from \"../containerRuntime\";\nimport { PendingStateManager } from \"../pendingStateManager\";\nimport { BatchManager } from \"./batchManager\";\nimport { BatchMessage, IBatch } from \"./definitions\";\nimport { OpCompressor } from \"./opCompressor\";\nimport { OpSplitter } from \"./opSplitter\";\n\nexport interface IOutboxConfig {\n\treadonly compressionOptions: ICompressionRuntimeOptions;\n\t// The maximum size of a batch that we can send over the wire.\n\treadonly maxBatchSizeInBytes: number;\n\treadonly enableOpReentryCheck?: boolean;\n}\n\nexport interface IOutboxParameters {\n\treadonly shouldSend: () => boolean;\n\treadonly pendingStateManager: PendingStateManager;\n\treadonly containerContext: IContainerContext;\n\treadonly config: IOutboxConfig;\n\treadonly compressor: OpCompressor;\n\treadonly splitter: OpSplitter;\n\treadonly logger: ITelemetryLogger;\n}\n\nexport class Outbox {\n\tprivate readonly attachFlowBatch: BatchManager;\n\tprivate readonly mainBatch: BatchManager;\n\tprivate readonly defaultAttachFlowSoftLimitInBytes = 64 * 1024;\n\n\tconstructor(private readonly params: IOutboxParameters) {\n\t\tconst isCompressionEnabled =\n\t\t\tthis.params.config.compressionOptions.minimumBatchSizeInBytes !==\n\t\t\tNumber.POSITIVE_INFINITY;\n\t\t// We need to allow infinite size batches if we enable compression\n\t\tconst hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;\n\t\tconst softLimit = isCompressionEnabled ? Infinity : this.defaultAttachFlowSoftLimitInBytes;\n\n\t\tthis.attachFlowBatch = new BatchManager(\n\t\t\t{\n\t\t\t\thardLimit,\n\t\t\t\tsoftLimit,\n\t\t\t\tenableOpReentryCheck: params.config.enableOpReentryCheck,\n\t\t\t},\n\t\t\tparams.logger,\n\t\t);\n\t\tthis.mainBatch = new BatchManager(\n\t\t\t{\n\t\t\t\thardLimit,\n\t\t\t\tenableOpReentryCheck: params.config.enableOpReentryCheck,\n\t\t\t},\n\t\t\tparams.logger,\n\t\t);\n\t}\n\n\tpublic get isEmpty(): boolean {\n\t\treturn this.attachFlowBatch.length === 0 && this.mainBatch.length === 0;\n\t}\n\n\tpublic submit(message: BatchMessage) {\n\t\tif (!this.mainBatch.push(message)) {\n\t\t\tthrow new GenericError(\"BatchTooLarge\", /* error */ undefined, {\n\t\t\t\topSize: message.contents?.length ?? 0,\n\t\t\t\tbatchSize: this.mainBatch.contentSizeInBytes,\n\t\t\t\tcount: this.mainBatch.length,\n\t\t\t\tlimit: this.mainBatch.options.hardLimit,\n\t\t\t});\n\t\t}\n\t}\n\n\tpublic submitAttach(message: BatchMessage) {\n\t\tif (!this.attachFlowBatch.push(message)) {\n\t\t\t// BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged\n\t\t\t// when queue is not empty.\n\t\t\t// Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit\n\t\t\tthis.flushInternal(this.attachFlowBatch.popBatch());\n\t\t\tif (!this.attachFlowBatch.push(message)) {\n\t\t\t\tthrow new GenericError(\"BatchTooLarge\", /* error */ undefined, {\n\t\t\t\t\topSize: message.contents?.length ?? 0,\n\t\t\t\t\tbatchSize: this.attachFlowBatch.contentSizeInBytes,\n\t\t\t\t\tcount: this.attachFlowBatch.length,\n\t\t\t\t\tlimit: this.attachFlowBatch.options.hardLimit,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\n\t\t// If compression is enabled, we will always successfully receive\n\t\t// attach ops and compress then send them at the next JS turn, regardless\n\t\t// of the overall size of the accumulated ops in the batch.\n\t\t// However, it is more efficient to flush these ops faster, preferably\n\t\t// after they reach a size which would benefit from compression.\n\t\tif (\n\t\t\tthis.attachFlowBatch.contentSizeInBytes >=\n\t\t\tthis.params.config.compressionOptions.minimumBatchSizeInBytes\n\t\t) {\n\t\t\tthis.flushInternal(this.attachFlowBatch.popBatch());\n\t\t}\n\t}\n\n\tpublic flush() {\n\t\tthis.flushInternal(this.attachFlowBatch.popBatch());\n\t\tthis.flushInternal(this.mainBatch.popBatch());\n\t}\n\n\tprivate flushInternal(rawBatch: IBatch) {\n\t\tconst processedBatch = this.compressBatch(rawBatch);\n\t\tconst clientSequenceNumber = this.sendBatch(processedBatch);\n\n\t\tthis.persistBatch(clientSequenceNumber, rawBatch.content);\n\t}\n\n\tprivate compressBatch(batch: IBatch): IBatch {\n\t\tif (\n\t\t\tbatch.content.length === 0 ||\n\t\t\tthis.params.config.compressionOptions === undefined ||\n\t\t\tthis.params.config.compressionOptions.minimumBatchSizeInBytes > batch.contentSizeInBytes\n\t\t) {\n\t\t\t// Nothing to do if the batch is empty or if compression is disabled or if we don't need to compress\n\t\t\treturn batch;\n\t\t}\n\n\t\tconst compressedBatch = this.params.compressor.compressBatch(batch);\n\t\tif (compressedBatch.contentSizeInBytes <= this.params.config.maxBatchSizeInBytes) {\n\t\t\t// If we don't reach the maximum supported size of a batch, it can safely be sent as is\n\t\t\treturn compressedBatch;\n\t\t}\n\n\t\tif (this.params.splitter.isBatchChunkingEnabled) {\n\t\t\treturn this.params.splitter.splitCompressedBatch(compressedBatch);\n\t\t}\n\n\t\t// If we've reached this point, the runtime would attempt to send a batch larger than the allowed size\n\t\tthrow new GenericError(\"BatchTooLarge\", /* error */ undefined, {\n\t\t\tbatchSize: batch.contentSizeInBytes,\n\t\t\tcompressedBatchSize: compressedBatch.contentSizeInBytes,\n\t\t\tcount: compressedBatch.content.length,\n\t\t\tlimit: this.params.config.maxBatchSizeInBytes,\n\t\t\tchunkingEnabled: this.params.splitter.isBatchChunkingEnabled,\n\t\t\tcompressionOptions: JSON.stringify(this.params.config.compressionOptions),\n\t\t});\n\t}\n\n\t/**\n\t * Sends the batch object to the container context to be sent over the wire.\n\t *\n\t * @param batch - batch to be sent\n\t * @returns the client sequence number of the last batched op which was sent and\n\t * -1 if there are no ops or the container cannot send ops.\n\t */\n\tprivate sendBatch(batch: IBatch): number {\n\t\tlet clientSequenceNumber: number = -1;\n\t\tconst length = batch.content.length;\n\n\t\t// Did we disconnect in the middle of turn-based batch?\n\t\t// If so, do nothing, as pending state manager will resubmit it correctly on reconnect.\n\t\tif (length === 0 || !this.params.shouldSend()) {\n\t\t\treturn clientSequenceNumber;\n\t\t}\n\n\t\tif (this.params.containerContext.submitBatchFn === undefined) {\n\t\t\t// Legacy path - supporting old loader versions. Can be removed only when LTS moves above\n\t\t\t// version that has support for batches (submitBatchFn)\n\t\t\tfor (const message of batch.content) {\n\t\t\t\t// Legacy path doesn't support compressed payloads and will submit uncompressed payload anyways\n\t\t\t\tif (message.metadata?.compressed) {\n\t\t\t\t\tdelete message.metadata.compressed;\n\t\t\t\t}\n\n\t\t\t\tclientSequenceNumber = this.params.containerContext.submitFn(\n\t\t\t\t\tMessageType.Operation,\n\t\t\t\t\tmessage.deserializedContent,\n\t\t\t\t\ttrue, // batch\n\t\t\t\t\tmessage.metadata,\n\t\t\t\t);\n\t\t\t}\n\n\t\t\tthis.params.containerContext.deltaManager.flush();\n\t\t} else {\n\t\t\t// returns clientSequenceNumber of last message in a batch\n\t\t\tclientSequenceNumber = this.params.containerContext.submitBatchFn(\n\t\t\t\tbatch.content.map((message) => ({\n\t\t\t\t\tcontents: message.contents,\n\t\t\t\t\tmetadata: message.metadata,\n\t\t\t\t\tcompression: message.compression,\n\t\t\t\t})),\n\t\t\t);\n\t\t}\n\n\t\t// Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.\n\t\tclientSequenceNumber -= length - 1;\n\t\tassert(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);\n\t\treturn clientSequenceNumber;\n\t}\n\n\tprivate persistBatch(initialClientSequenceNumber: number, batch: BatchMessage[]) {\n\t\tlet clientSequenceNumber = initialClientSequenceNumber;\n\t\t// Let the PendingStateManager know that a message was submitted.\n\t\t// In future, need to shift toward keeping batch as a whole!\n\t\tfor (const message of batch) {\n\t\t\tthis.params.pendingStateManager.onSubmitMessage(\n\t\t\t\tmessage.deserializedContent.type,\n\t\t\t\tclientSequenceNumber,\n\t\t\t\tmessage.referenceSequenceNumber,\n\t\t\t\tmessage.deserializedContent.contents,\n\t\t\t\tmessage.localOpMetadata,\n\t\t\t\tmessage.metadata,\n\t\t\t);\n\n\t\t\tclientSequenceNumber++;\n\t\t}\n\t}\n\n\tpublic checkpoint() {\n\t\treturn {\n\t\t\tmainBatch: this.mainBatch.checkpoint(),\n\t\t\tattachFlowBatch: this.attachFlowBatch.checkpoint(),\n\t\t};\n\t}\n}\n"]}
+{"version":3,"file":"outbox.js","sourceRoot":"","sources":["../../src/opLifecycle/outbox.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAGH,+DAAsD;AAEtD,qEAA2E;AAC3E,+EAAmE;AACnE,qEAIyC;AAGzC,iDAAkE;AAsBlE,MAAa,MAAM;IAelB,YAA6B,MAAyB;QAAzB,WAAM,GAAN,MAAM,CAAmB;QAXrC,sCAAiC,GAAG,GAAG,GAAG,IAAI,CAAC;QAEhE;;;;;WAKG;QACc,6BAAwB,GAAG,CAAC,CAAC;QACtC,0BAAqB,GAAG,CAAC,CAAC;QAGjC,IAAI,CAAC,EAAE,GAAG,IAAA,2CAAyB,EAAC,6BAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;QACjF,MAAM,oBAAoB,GACzB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,uBAAuB;YAC7D,MAAM,CAAC,iBAAiB,CAAC;QAC1B,kEAAkE;QAClE,MAAM,SAAS,GAAG,oBAAoB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC;QAC3F,MAAM,SAAS,GAAG,oBAAoB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,iCAAiC,CAAC;QAE3F,IAAI,CAAC,eAAe,GAAG,IAAI,2BAAY,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,SAAS,GAAG,IAAI,2BAAY,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,IAAW,OAAO;QACjB,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC;IACzE,CAAC;IAED;;;;;;;OAOG;IACK,sBAAsB,CAAC,OAAqB;QACnD,MAAM,kBAAkB,GAAG,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;QAClE,MAAM,wBAAwB,GAAG,IAAI,CAAC,eAAe,CAAC,uBAAuB,CAAC;QAC9E,IAAA,qBAAM,EACL,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,mBAAmB;YACrC,kBAAkB,KAAK,SAAS;YAChC,wBAAwB,KAAK,SAAS;YACtC,kBAAkB,KAAK,wBAAwB,EAChD,KAAK,CAAC,kEAAkE,CACxE,CAAC;QAEF,IACC,CAAC,kBAAkB,KAAK,SAAS;YAChC,kBAAkB,KAAK,OAAO,CAAC,uBAAuB,CAAC;YACxD,CAAC,wBAAwB,KAAK,SAAS;gBACtC,wBAAwB,KAAK,OAAO,CAAC,uBAAuB,CAAC,EAC7D;YACD,oEAAoE;YACpE,OAAO;SACP;QAED,IAAI,EAAE,IAAI,CAAC,qBAAqB,IAAI,IAAI,CAAC,wBAAwB,EAAE;YAClE,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,CAC5B;gBACC,SAAS,EAAE,iCAAiC;gBAC5C,2BAA2B,EAAE,kBAAkB;gBAC/C,6BAA6B,EAAE,wBAAwB;gBACvD,8BAA8B,EAAE,OAAO,CAAC,uBAAuB;aAC/D,EACD,IAAI,4BAAU,CAAC,uCAAuC,CAAC,CACvD,CAAC;SACF;QAED,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,mBAAmB,EAAE;YAC5C,IAAI,CAAC,KAAK,EAAE,CAAC;SACb;IACF,CAAC;IAEM,MAAM,CAAC,OAAqB;;QAClC,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAErC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YAClC,MAAM,IAAI,8BAAY,CAAC,eAAe,EAAE,WAAW,CAAC,SAAS,EAAE;gBAC9D,MAAM,EAAE,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC;gBACrC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB;gBAC5C,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM;gBAC5B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS;aACvC,CAAC,CAAC;SACH;IACF,CAAC;IAEM,YAAY,CAAC,OAAqB;;QACxC,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAErC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YACxC,oFAAoF;YACpF,2BAA2B;YAC3B,8FAA8F;YAC9F,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;gBACxC,MAAM,IAAI,8BAAY,CAAC,eAAe,EAAE,WAAW,CAAC,SAAS,EAAE;oBAC9D,MAAM,EAAE,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC;oBACrC,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,kBAAkB;oBAClD,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;oBAClC,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS;iBAC7C,CAAC,CAAC;aACH;SACD;QAED,iEAAiE;QACjE,yEAAyE;QACzE,2DAA2D;QAC3D,sEAAsE;QACtE,gEAAgE;QAChE,IACC,IAAI,CAAC,eAAe,CAAC,kBAAkB;YACvC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,uBAAuB,EAC5D;YACD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;SACpD;IACF,CAAC;IAEM,KAAK;QACX,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC/C,CAAC;IAEO,aAAa,CAAC,QAAgB;QACrC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QAE/B,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAEO,aAAa,CAAC,KAAa;QAClC,IACC,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,KAAK,SAAS;YACnD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,uBAAuB;gBAC5D,KAAK,CAAC,kBAAkB;YACzB,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,aAAa,KAAK,SAAS,EACvD;YACD,sHAAsH;YACtH,OAAO,KAAK,CAAC;SACb;QAED,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAEpE,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,EAAE;YAChD,OAAO,eAAe,CAAC,kBAAkB,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB;gBACjF,CAAC,CAAC,eAAe;gBACjB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;SAC9D;QAED,IAAI,eAAe,CAAC,kBAAkB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,mBAAmB,EAAE;YACjF,MAAM,IAAI,8BAAY,CAAC,eAAe,EAAE,WAAW,CAAC,SAAS,EAAE;gBAC9D,SAAS,EAAE,KAAK,CAAC,kBAAkB;gBACnC,mBAAmB,EAAE,eAAe,CAAC,kBAAkB;gBACvD,KAAK,EAAE,eAAe,CAAC,OAAO,CAAC,MAAM;gBACrC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,mBAAmB;gBAC7C,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB;gBAC5D,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC;gBACzE,UAAU,EAAE,IAAA,iCAAkB,EAAC,KAAK,CAAC;aACrC,CAAC,CAAC;SACH;QAED,OAAO,eAAe,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACK,SAAS,CAAC,KAAa;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAEpC,uDAAuD;QACvD,uFAAuF;QACvF,IAAI,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE;YAC9C,OAAO;SACP;QAED,MAAM,UAAU,GAAG,IAAA,iCAAkB,EAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,mBAAmB,EAAE;YACzD,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBACnC,SAAS,EAAE,YAAY;gBACvB,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;gBAC5B,WAAW,EAAE,KAAK,CAAC,kBAAkB;gBACrC,UAAU;aACV,CAAC,CAAC;SACH;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,aAAa,KAAK,SAAS,EAAE;YAC7D,yFAAyF;YACzF,uDAAuD;YACvD,IAAA,qBAAM,EACL,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,EAC1C,KAAK,CAAC,4EAA4E,CAClF,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE;gBACpC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,QAAQ,CACpC,kCAAW,CAAC,SAAS,EACrB,OAAO,CAAC,mBAAmB,EAC3B,IAAI,EAAE,QAAQ;gBACd,OAAO,CAAC,QAAQ,CAChB,CAAC;aACF;YAED,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;SAClD;aAAM;YACN,IAAA,qBAAM,EACL,KAAK,CAAC,uBAAuB,KAAK,SAAS,EAC3C,KAAK,CAAC,6BAA6B,CACnC,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,aAAa,CACzC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC/B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;aACxD,CAAC,CAAC,EACH,KAAK,CAAC,uBAAuB,CAC7B,CAAC;SACF;IACF,CAAC;IAEO,YAAY,CAAC,KAAqB;QACzC,iEAAiE;QACjE,4DAA4D;QAC5D,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE;YAC5B,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,eAAe,CAC9C,OAAO,CAAC,mBAAmB,CAAC,IAAI,EAChC,OAAO,CAAC,uBAAuB,EAC/B,OAAO,CAAC,mBAAmB,CAAC,QAAQ,EACpC,OAAO,CAAC,eAAe,EACvB,OAAO,CAAC,QAAQ,CAChB,CAAC;SACF;IACF,CAAC;IAEM,UAAU;QAChB,OAAO;YACN,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE;YACtC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE;SAClD,CAAC;IACH,CAAC;CACD;AAxPD,wBAwPC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert } from \"@fluidframework/common-utils\";\nimport { IContainerContext } from \"@fluidframework/container-definitions\";\nimport { GenericError, UsageError } from \"@fluidframework/container-utils\";\nimport { MessageType } from \"@fluidframework/protocol-definitions\";\nimport {\n\tChildLogger,\n\tloggerToMonitoringContext,\n\tMonitoringContext,\n} from \"@fluidframework/telemetry-utils\";\nimport { ICompressionRuntimeOptions } from \"../containerRuntime\";\nimport { PendingStateManager } from \"../pendingStateManager\";\nimport { BatchManager, estimateSocketSize } from \"./batchManager\";\nimport { BatchMessage, IBatch } from \"./definitions\";\nimport { OpCompressor } from \"./opCompressor\";\nimport { OpSplitter } from \"./opSplitter\";\n\nexport interface IOutboxConfig {\n\treadonly compressionOptions: ICompressionRuntimeOptions;\n\t// The maximum size of a batch that we can send over the wire.\n\treadonly maxBatchSizeInBytes: number;\n\treadonly disablePartialFlush: boolean;\n}\n\nexport interface IOutboxParameters {\n\treadonly shouldSend: () => boolean;\n\treadonly pendingStateManager: PendingStateManager;\n\treadonly containerContext: IContainerContext;\n\treadonly config: IOutboxConfig;\n\treadonly compressor: OpCompressor;\n\treadonly splitter: OpSplitter;\n\treadonly logger: ITelemetryLogger;\n}\n\nexport class Outbox {\n\tprivate readonly mc: MonitoringContext;\n\tprivate readonly attachFlowBatch: BatchManager;\n\tprivate readonly mainBatch: BatchManager;\n\tprivate readonly defaultAttachFlowSoftLimitInBytes = 320 * 1024;\n\n\t/**\n\t * Track the number of ops which were detected to have a mismatched\n\t * reference sequence number, in order to self-throttle the telemetry events.\n\t *\n\t * This should be removed as part of ADO:2322\n\t */\n\tprivate readonly maxMismatchedOpsToReport = 3;\n\tprivate mismatchedOpsReported = 0;\n\n\tconstructor(private readonly params: IOutboxParameters) {\n\t\tthis.mc = loggerToMonitoringContext(ChildLogger.create(params.logger, \"Outbox\"));\n\t\tconst isCompressionEnabled =\n\t\t\tthis.params.config.compressionOptions.minimumBatchSizeInBytes !==\n\t\t\tNumber.POSITIVE_INFINITY;\n\t\t// We need to allow infinite size batches if we enable compression\n\t\tconst hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;\n\t\tconst softLimit = isCompressionEnabled ? Infinity : this.defaultAttachFlowSoftLimitInBytes;\n\n\t\tthis.attachFlowBatch = new BatchManager({ hardLimit, softLimit });\n\t\tthis.mainBatch = new BatchManager({ hardLimit });\n\t}\n\n\tpublic get isEmpty(): boolean {\n\t\treturn this.attachFlowBatch.length === 0 && this.mainBatch.length === 0;\n\t}\n\n\t/**\n\t * If we detect that the reference sequence number of the incoming message does not match\n\t * what was already in the batch managers, this means that batching has been interrupted so\n\t * we will flush the accumulated messages to account for that and create a new batch with the new\n\t * message as the first message.\n\t *\n\t * @param message - the incoming message\n\t */\n\tprivate maybeFlushPartialBatch(message: BatchMessage) {\n\t\tconst mainBatchReference = this.mainBatch.referenceSequenceNumber;\n\t\tconst attachFlowBatchReference = this.attachFlowBatch.referenceSequenceNumber;\n\t\tassert(\n\t\t\tthis.params.config.disablePartialFlush ||\n\t\t\t\tmainBatchReference === undefined ||\n\t\t\t\tattachFlowBatchReference === undefined ||\n\t\t\t\tmainBatchReference === attachFlowBatchReference,\n\t\t\t0x58d /* Reference sequence numbers from both batches must be in sync */,\n\t\t);\n\n\t\tif (\n\t\t\t(mainBatchReference === undefined ||\n\t\t\t\tmainBatchReference === message.referenceSequenceNumber) &&\n\t\t\t(attachFlowBatchReference === undefined ||\n\t\t\t\tattachFlowBatchReference === message.referenceSequenceNumber)\n\t\t) {\n\t\t\t// The reference sequence numbers are stable, there is nothing to do\n\t\t\treturn;\n\t\t}\n\n\t\tif (++this.mismatchedOpsReported <= this.maxMismatchedOpsToReport) {\n\t\t\tthis.mc.logger.sendErrorEvent(\n\t\t\t\t{\n\t\t\t\t\teventName: \"ReferenceSequenceNumberMismatch\",\n\t\t\t\t\tmainReferenceSequenceNumber: mainBatchReference,\n\t\t\t\t\tattachReferenceSequenceNumber: attachFlowBatchReference,\n\t\t\t\t\tmessageReferenceSequenceNumber: message.referenceSequenceNumber,\n\t\t\t\t},\n\t\t\t\tnew UsageError(\"Submission of an out of order message\"),\n\t\t\t);\n\t\t}\n\n\t\tif (!this.params.config.disablePartialFlush) {\n\t\t\tthis.flush();\n\t\t}\n\t}\n\n\tpublic submit(message: BatchMessage) {\n\t\tthis.maybeFlushPartialBatch(message);\n\n\t\tif (!this.mainBatch.push(message)) {\n\t\t\tthrow new GenericError(\"BatchTooLarge\", /* error */ undefined, {\n\t\t\t\topSize: message.contents?.length ?? 0,\n\t\t\t\tbatchSize: this.mainBatch.contentSizeInBytes,\n\t\t\t\tcount: this.mainBatch.length,\n\t\t\t\tlimit: this.mainBatch.options.hardLimit,\n\t\t\t});\n\t\t}\n\t}\n\n\tpublic submitAttach(message: BatchMessage) {\n\t\tthis.maybeFlushPartialBatch(message);\n\n\t\tif (!this.attachFlowBatch.push(message)) {\n\t\t\t// BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged\n\t\t\t// when queue is not empty.\n\t\t\t// Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit\n\t\t\tthis.flushInternal(this.attachFlowBatch.popBatch());\n\t\t\tif (!this.attachFlowBatch.push(message)) {\n\t\t\t\tthrow new GenericError(\"BatchTooLarge\", /* error */ undefined, {\n\t\t\t\t\topSize: message.contents?.length ?? 0,\n\t\t\t\t\tbatchSize: this.attachFlowBatch.contentSizeInBytes,\n\t\t\t\t\tcount: this.attachFlowBatch.length,\n\t\t\t\t\tlimit: this.attachFlowBatch.options.hardLimit,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\n\t\t// If compression is enabled, we will always successfully receive\n\t\t// attach ops and compress then send them at the next JS turn, regardless\n\t\t// of the overall size of the accumulated ops in the batch.\n\t\t// However, it is more efficient to flush these ops faster, preferably\n\t\t// after they reach a size which would benefit from compression.\n\t\tif (\n\t\t\tthis.attachFlowBatch.contentSizeInBytes >=\n\t\t\tthis.params.config.compressionOptions.minimumBatchSizeInBytes\n\t\t) {\n\t\t\tthis.flushInternal(this.attachFlowBatch.popBatch());\n\t\t}\n\t}\n\n\tpublic flush() {\n\t\tthis.flushInternal(this.attachFlowBatch.popBatch());\n\t\tthis.flushInternal(this.mainBatch.popBatch());\n\t}\n\n\tprivate flushInternal(rawBatch: IBatch) {\n\t\tconst processedBatch = this.compressBatch(rawBatch);\n\t\tthis.sendBatch(processedBatch);\n\n\t\tthis.persistBatch(rawBatch.content);\n\t}\n\n\tprivate compressBatch(batch: IBatch): IBatch {\n\t\tif (\n\t\t\tbatch.content.length === 0 ||\n\t\t\tthis.params.config.compressionOptions === undefined ||\n\t\t\tthis.params.config.compressionOptions.minimumBatchSizeInBytes >\n\t\t\t\tbatch.contentSizeInBytes ||\n\t\t\tthis.params.containerContext.submitBatchFn === undefined\n\t\t) {\n\t\t\t// Nothing to do if the batch is empty or if compression is disabled or not supported, or if we don't need to compress\n\t\t\treturn batch;\n\t\t}\n\n\t\tconst compressedBatch = this.params.compressor.compressBatch(batch);\n\n\t\tif (this.params.splitter.isBatchChunkingEnabled) {\n\t\t\treturn compressedBatch.contentSizeInBytes <= this.params.splitter.chunkSizeInBytes\n\t\t\t\t? compressedBatch\n\t\t\t\t: this.params.splitter.splitCompressedBatch(compressedBatch);\n\t\t}\n\n\t\tif (compressedBatch.contentSizeInBytes >= this.params.config.maxBatchSizeInBytes) {\n\t\t\tthrow new GenericError(\"BatchTooLarge\", /* error */ undefined, {\n\t\t\t\tbatchSize: batch.contentSizeInBytes,\n\t\t\t\tcompressedBatchSize: compressedBatch.contentSizeInBytes,\n\t\t\t\tcount: compressedBatch.content.length,\n\t\t\t\tlimit: this.params.config.maxBatchSizeInBytes,\n\t\t\t\tchunkingEnabled: this.params.splitter.isBatchChunkingEnabled,\n\t\t\t\tcompressionOptions: JSON.stringify(this.params.config.compressionOptions),\n\t\t\t\tsocketSize: estimateSocketSize(batch),\n\t\t\t});\n\t\t}\n\n\t\treturn compressedBatch;\n\t}\n\n\t/**\n\t * Sends the batch object to the container context to be sent over the wire.\n\t *\n\t * @param batch - batch to be sent\n\t */\n\tprivate sendBatch(batch: IBatch) {\n\t\tconst length = batch.content.length;\n\n\t\t// Did we disconnect in the middle of turn-based batch?\n\t\t// If so, do nothing, as pending state manager will resubmit it correctly on reconnect.\n\t\tif (length === 0 || !this.params.shouldSend()) {\n\t\t\treturn;\n\t\t}\n\n\t\tconst socketSize = estimateSocketSize(batch);\n\t\tif (socketSize >= this.params.config.maxBatchSizeInBytes) {\n\t\t\tthis.mc.logger.sendPerformanceEvent({\n\t\t\t\teventName: \"LargeBatch\",\n\t\t\t\tlength: batch.content.length,\n\t\t\t\tsizeInBytes: batch.contentSizeInBytes,\n\t\t\t\tsocketSize,\n\t\t\t});\n\t\t}\n\n\t\tif (this.params.containerContext.submitBatchFn === undefined) {\n\t\t\t// Legacy path - supporting old loader versions. Can be removed only when LTS moves above\n\t\t\t// version that has support for batches (submitBatchFn)\n\t\t\tassert(\n\t\t\t\tbatch.content[0].compression === undefined,\n\t\t\t\t0x5a6 /* Compression should not have happened if the loader does not support it */,\n\t\t\t);\n\n\t\t\tfor (const message of batch.content) {\n\t\t\t\tthis.params.containerContext.submitFn(\n\t\t\t\t\tMessageType.Operation,\n\t\t\t\t\tmessage.deserializedContent,\n\t\t\t\t\ttrue, // batch\n\t\t\t\t\tmessage.metadata,\n\t\t\t\t);\n\t\t\t}\n\n\t\t\tthis.params.containerContext.deltaManager.flush();\n\t\t} else {\n\t\t\tassert(\n\t\t\t\tbatch.referenceSequenceNumber !== undefined,\n\t\t\t\t0x58e /* Batch must not be empty */,\n\t\t\t);\n\t\t\tthis.params.containerContext.submitBatchFn(\n\t\t\t\tbatch.content.map((message) => ({\n\t\t\t\t\tcontents: message.contents,\n\t\t\t\t\tmetadata: message.metadata,\n\t\t\t\t\tcompression: message.compression,\n\t\t\t\t\treferenceSequenceNumber: message.referenceSequenceNumber,\n\t\t\t\t})),\n\t\t\t\tbatch.referenceSequenceNumber,\n\t\t\t);\n\t\t}\n\t}\n\n\tprivate persistBatch(batch: BatchMessage[]) {\n\t\t// Let the PendingStateManager know that a message was submitted.\n\t\t// In future, need to shift toward keeping batch as a whole!\n\t\tfor (const message of batch) {\n\t\t\tthis.params.pendingStateManager.onSubmitMessage(\n\t\t\t\tmessage.deserializedContent.type,\n\t\t\t\tmessage.referenceSequenceNumber,\n\t\t\t\tmessage.deserializedContent.contents,\n\t\t\t\tmessage.localOpMetadata,\n\t\t\t\tmessage.metadata,\n\t\t\t);\n\t\t}\n\t}\n\n\tpublic checkpoint() {\n\t\treturn {\n\t\t\tmainBatch: this.mainBatch.checkpoint(),\n\t\t\tattachFlowBatch: this.attachFlowBatch.checkpoint(),\n\t\t};\n\t}\n}\n"]}

package/dist/packageVersion.d.ts

@@ -5,5 +5,5 @@
  * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
  */
 export declare const pkgName = "@fluidframework/container-runtime";
-export declare const pkgVersion = "2.0.0-dev.3.1.0.125672";
+export declare const pkgVersion = "2.0.0-dev.4.1.0.148229";
 //# sourceMappingURL=packageVersion.d.ts.map

package/dist/packageVersion.js

@@ -8,5 +8,5 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.pkgVersion = exports.pkgName = void 0;
 exports.pkgName = "@fluidframework/container-runtime";
-exports.pkgVersion = "2.0.0-dev.3.1.0.125672";
+exports.pkgVersion = "2.0.0-dev.4.1.0.148229";
 //# sourceMappingURL=packageVersion.js.map

package/dist/packageVersion.js.map

@@ -1 +1 @@
-{"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAEU,QAAA,OAAO,GAAG,mCAAmC,CAAC;AAC9C,QAAA,UAAU,GAAG,wBAAwB,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/container-runtime\";\nexport const pkgVersion = \"2.0.0-dev.3.1.0.125672\";\n"]}
+{"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAEU,QAAA,OAAO,GAAG,mCAAmC,CAAC;AAC9C,QAAA,UAAU,GAAG,wBAAwB,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/container-runtime\";\nexport const pkgVersion = \"2.0.0-dev.4.1.0.148229\";\n"]}

package/dist/pendingStateManager.d.ts

@@ -22,11 +22,12 @@ export interface IPendingMessage {
 /**
  * This represents an explicit flush call and is added to the pending queue when flush is called on the ContainerRuntime
  * to flush pending messages.
- * @deprecated Use batch metadata on IPendingMessage instead. To be removed in 2.0.0-internal.4.0.0 (AB#2496)
+ * ! TODO: Remove in "2.0.0-internal.5.0.0" AB#2496
  */
 export interface IPendingFlush {
     type: "flush";
 }
+/** ! TODO: Remove in "2.0.0-internal.5.0.0" AB#2496 */
 export declare type IPendingState = IPendingMessage | IPendingFlush;
 export interface IPendingLocalState {
     /**
@@ -74,11 +75,10 @@ export declare class PendingStateManager implements IDisposable {
      * Called when a message is submitted locally. Adds the message and the associated details to the pending state
      * queue.
      * @param type - The container message type.
-     * @param clientSequenceNumber - The clientSequenceNumber associated with the message.
      * @param content - The message content.
      * @param localOpMetadata - The local metadata associated with the message.
      */
-    onSubmitMessage(type: ContainerMessageType, clientSequenceNumber: number, referenceSequenceNumber: number, content: any, localOpMetadata: unknown, opMetadata: Record<string, unknown> | undefined): void;
+    onSubmitMessage(type: ContainerMessageType, referenceSequenceNumber: number, content: any, localOpMetadata: unknown, opMetadata: Record<string, unknown> | undefined): void;
     /**
      * Applies stashed ops at their reference sequence number so they are ready to be ACKed or resubmitted
      * @param seqNum - Sequence number at which to apply ops. Will apply all ops if seqNum is undefined.