@fluidframework/container-runtime 2.0.0-internal.3.1.0 → 2.0.0-internal.3.2.1

package/lib/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACN,oBAAoB,EACpB,uBAAuB,EACvB,iBAAiB,EACjB,sBAAsB,EACtB,yBAAyB,EACzB,+BAA+B,EAC/B,sCAAsC,EACtC,sCAAsC,EACtC,wBAAwB,EACxB,yBAAyB,EACzB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,8BAA8B,EAC9B,0BAA0B,EAC1B,qBAAqB,EACrB,2BAA2B,EAC3B,0BAA0B,EAC1B,qBAAqB,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EACN,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,aAAa,GACb,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EACN,sBAAsB,EACtB,iBAAiB,EACjB,oBAAoB,EACpB,uBAAuB,EACvB,kBAAkB,EAClB,mBAAmB,EACnB,wBAAwB,EACxB,0BAA0B,EAC1B,sBAAsB,EACtB,kBAAkB,EAClB,yBAAyB,EACzB,kBAAkB,EAClB,yBAAyB,EACzB,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,WAAW,EACX,iBAAiB,EACjB,4BAA4B,EAC5B,kBAAkB,EAClB,mBAAmB,EACnB,yBAAyB,EACzB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACN,aAAa,EACb,qBAAqB,EACrB,QAAQ,EACR,0BAA0B,EAC1B,kBAAkB,EAClB,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACN,gCAAgC,EAChC,0BAA0B,GAC1B,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACN,oBAAoB,EACpB,uBAAuB,EACvB,iBAAiB,EACjB,sBAAsB,EACtB,yBAAyB,EACzB,+BAA+B,EAC/B,sCAAsC,EACtC,sCAAsC,EACtC,wBAAwB,EACxB,yBAAyB,EACzB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,8BAA8B,EAC9B,0BAA0B,EAC1B,qBAAqB,EACrB,2BAA2B,EAC3B,0BAA0B,EAC1B,qBAAqB,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EACN,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,aAAa,GACb,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACN,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,yBAAyB,EACzB,0BAA0B,EAC1B,UAAU,EACV,oBAAoB,EACpB,iBAAiB,EACjB,sBAAsB,EACtB,iBAAiB,EACjB,oBAAoB,EACpB,uBAAuB,EACvB,kBAAkB,EAClB,mBAAmB,EACnB,wBAAwB,EACxB,0BAA0B,EAC1B,sBAAsB,EACtB,kBAAkB,EAClB,yBAAyB,EACzB,kBAAkB,EAClB,yBAAyB,EACzB,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,4BAA4B,EAC5B,kBAAkB,EAClB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EACrB,QAAQ,EACR,0BAA0B,EAC1B,kBAAkB,EAClB,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,iBAAiB,EACjB,gCAAgC,GAChC,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC"}

package/lib/index.js

@@ -4,9 +4,6 @@
  */
 export { ContainerMessageType, isRuntimeMessage, RuntimeMessage, agentSchedulerId, ContainerRuntime, RuntimeHeaders, AllowTombstoneRequestHeaderKey, TombstoneResponseHeaderKey, DefaultSummaryConfiguration, CompressionAlgorithms, } from "./containerRuntime";
 export { FluidDataStoreRegistry } from "./dataStoreRegistry";
-export { Summarizer } from "./summarizer";
-export { ISummarizer, } from "./summarizerTypes";
-export { SummaryCollection, } from "./summaryCollection";
-export { neverCancelledSummaryToken, } from "./runWhileConnectedCoordinator";
+export { ISummarizer, neverCancelledSummaryToken, Summarizer, SummaryCollection, } from "./summary";
 export { unpackRuntimeMessage } from "./opLifecycle";
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACN,oBAAoB,EAUpB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,8BAA8B,EAC9B,0BAA0B,EAE1B,2BAA2B,EAE3B,qBAAqB,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAQ7D,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAkBN,WAAW,GAUX,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAUN,iBAAiB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAEN,0BAA0B,GAC1B,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAc,oBAAoB,EAAE,MAAM,eAAe,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nexport {\n\tContainerMessageType,\n\tContainerRuntimeMessage,\n\tIGCRuntimeOptions,\n\tISummaryRuntimeOptions,\n\tISummaryBaseConfiguration,\n\tISummaryConfigurationHeuristics,\n\tISummaryConfigurationDisableSummarizer,\n\tISummaryConfigurationDisableHeuristics,\n\tIContainerRuntimeOptions,\n\tIRootSummaryTreeWithStats,\n\tisRuntimeMessage,\n\tRuntimeMessage,\n\tagentSchedulerId,\n\tContainerRuntime,\n\tRuntimeHeaders,\n\tAllowTombstoneRequestHeaderKey,\n\tTombstoneResponseHeaderKey,\n\tISummaryConfiguration,\n\tDefaultSummaryConfiguration,\n\tICompressionRuntimeOptions,\n\tCompressionAlgorithms,\n} from \"./containerRuntime\";\nexport { FluidDataStoreRegistry } from \"./dataStoreRegistry\";\nexport { IGCStats } from \"./garbageCollection\";\nexport {\n\tIPendingFlush,\n\tIPendingLocalState,\n\tIPendingMessage,\n\tIPendingState,\n} from \"./pendingStateManager\";\nexport { Summarizer } from \"./summarizer\";\nexport {\n\tEnqueueSummarizeResult,\n\tIAckSummaryResult,\n\tIBaseSummarizeResult,\n\tIBroadcastSummaryResult,\n\tICancellationToken,\n\tIConnectableRuntime,\n\tIEnqueueSummarizeOptions,\n\tIGenerateSummaryTreeResult,\n\tIGeneratedSummaryStats,\n\tINackSummaryResult,\n\tIOnDemandSummarizeOptions,\n\tIProvideSummarizer,\n\tIRefreshSummaryAckOptions,\n\tISubmitSummaryOpResult,\n\tISubmitSummaryOptions,\n\tISummarizeOptions,\n\tISummarizeResults,\n\tISummarizer,\n\tISummarizerEvents,\n\tISummarizerInternalsProvider,\n\tISummarizerRuntime,\n\tISummarizingWarning,\n\tISummaryCancellationToken,\n\tIUploadSummaryResult,\n\tSubmitSummaryResult,\n\tSummarizeResultPart,\n\tSummarizerStopReason,\n} from \"./summarizerTypes\";\nexport {\n\tIAckedSummary,\n\tIClientSummaryWatcher,\n\tISummary,\n\tISummaryCollectionOpEvents,\n\tISummaryAckMessage,\n\tISummaryNackMessage,\n\tISummaryOpMessage,\n\tOpActionEventListener,\n\tOpActionEventName,\n\tSummaryCollection,\n} from \"./summaryCollection\";\nexport {\n\tICancellableSummarizerController,\n\tneverCancelledSummaryToken,\n} from \"./runWhileConnectedCoordinator\";\nexport { IChunkedOp, unpackRuntimeMessage } from \"./opLifecycle\";\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACN,oBAAoB,EAUpB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,8BAA8B,EAC9B,0BAA0B,EAE1B,2BAA2B,EAE3B,qBAAqB,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAQ7D,OAAO,EAEN,WAAW,EAGX,0BAA0B,EAC1B,UAAU,EAEV,iBAAiB,GAiCjB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAc,oBAAoB,EAAE,MAAM,eAAe,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nexport {\n\tContainerMessageType,\n\tContainerRuntimeMessage,\n\tIGCRuntimeOptions,\n\tISummaryRuntimeOptions,\n\tISummaryBaseConfiguration,\n\tISummaryConfigurationHeuristics,\n\tISummaryConfigurationDisableSummarizer,\n\tISummaryConfigurationDisableHeuristics,\n\tIContainerRuntimeOptions,\n\tIRootSummaryTreeWithStats,\n\tisRuntimeMessage,\n\tRuntimeMessage,\n\tagentSchedulerId,\n\tContainerRuntime,\n\tRuntimeHeaders,\n\tAllowTombstoneRequestHeaderKey,\n\tTombstoneResponseHeaderKey,\n\tISummaryConfiguration,\n\tDefaultSummaryConfiguration,\n\tICompressionRuntimeOptions,\n\tCompressionAlgorithms,\n} from \"./containerRuntime\";\nexport { FluidDataStoreRegistry } from \"./dataStoreRegistry\";\nexport { IGCStats } from \"./gc\";\nexport {\n\tIPendingFlush,\n\tIPendingLocalState,\n\tIPendingMessage,\n\tIPendingState,\n} from \"./pendingStateManager\";\nexport {\n\tIAckedSummary,\n\tISummarizer,\n\tISummarizeResults,\n\tISummaryCancellationToken,\n\tneverCancelledSummaryToken,\n\tSummarizer,\n\tSummarizerStopReason,\n\tSummaryCollection,\n\tEnqueueSummarizeResult,\n\tIAckSummaryResult,\n\tIBaseSummarizeResult,\n\tIBroadcastSummaryResult,\n\tICancellationToken,\n\tIConnectableRuntime,\n\tIEnqueueSummarizeOptions,\n\tIGenerateSummaryTreeResult,\n\tIGeneratedSummaryStats,\n\tINackSummaryResult,\n\tIOnDemandSummarizeOptions,\n\tIProvideSummarizer,\n\tIRefreshSummaryAckOptions,\n\tISubmitSummaryOpResult,\n\tISubmitSummaryOptions,\n\tISummarizeOptions,\n\tISummarizerEvents,\n\tISummarizerInternalsProvider,\n\tISummarizerRuntime,\n\tISummarizingWarning,\n\tIUploadSummaryResult,\n\tSubmitSummaryResult,\n\tSummarizeResultPart,\n\tIClientSummaryWatcher,\n\tISummary,\n\tISummaryCollectionOpEvents,\n\tISummaryAckMessage,\n\tISummaryNackMessage,\n\tISummaryOpMessage,\n\tOpActionEventListener,\n\tOpActionEventName,\n\tICancellableSummarizerController,\n} from \"./summary\";\nexport { IChunkedOp, unpackRuntimeMessage } from \"./opLifecycle\";\n"]}

package/lib/opLifecycle/batchManager.d.ts

@@ -2,11 +2,9 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { ICompressionRuntimeOptions } from "../containerRuntime";
 import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions";
 export interface IBatchManagerOptions {
-    readonly enableOpReentryCheck?: boolean;
     readonly hardLimit: number;
     readonly softLimit?: number;
     readonly compressionOptions?: ICompressionRuntimeOptions;
@@ -16,20 +14,12 @@ export interface IBatchManagerOptions {
  */
 export declare class BatchManager {
     readonly options: IBatchManagerOptions;
-    private readonly logger;
     private pendingBatch;
     private batchContentSize;
-    /**
-     * Track the number of ops which were detected to have a mismatched
-     * reference sequence number, in order to self-throttle the telemetry events.
-     *
-     * This should be removed as part of ADO:2322
-     */
-    private readonly maxMismatchedOpsToReport;
-    private mismatchedOpsReported;
     get length(): number;
     get contentSizeInBytes(): number;
-    constructor(options: IBatchManagerOptions, logger: ITelemetryLogger);
+    get referenceSequenceNumber(): number | undefined;
+    constructor(options: IBatchManagerOptions);
     push(message: BatchMessage): boolean;
     get empty(): boolean;
     popBatch(): IBatch;
@@ -37,6 +27,5 @@ export declare class BatchManager {
      * Capture the pending state at this point
      */
     checkpoint(): IBatchCheckpoint;
-    private checkReferenceSequenceNumber;
 }
 //# sourceMappingURL=batchManager.d.ts.map

package/lib/opLifecycle/batchManager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"batchManager.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/batchManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAGtE,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEvE,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,OAAO,CAAC;IACxC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,0BAA0B,CAAC;CACzD;AAED;;GAEG;AACH,qBAAa,YAAY;aAoBI,OAAO,EAAE,oBAAoB;IAnBzD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IACxB,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,gBAAgB,CAAK;IAC7B;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAK;IAC9C,OAAO,CAAC,qBAAqB,CAAK;IAElC,IAAW,MAAM,WAEhB;IACD,IAAW,kBAAkB,WAE5B;gBAE2B,OAAO,EAAE,oBAAoB,EAAE,MAAM,EAAE,gBAAgB;IAI5E,IAAI,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO;IAmC3C,IAAW,KAAK,YAEf;IAEM,QAAQ,IAAI,MAAM;IAYzB;;OAEG;IACI,UAAU,IAAI,gBAAgB;IAgBrC,OAAO,CAAC,4BAA4B;CAoCpC"}
+{"version":3,"file":"batchManager.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/batchManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEvE,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,0BAA0B,CAAC;CACzD;AAED;;GAEG;AACH,qBAAa,YAAY;aAiBI,OAAO,EAAE,oBAAoB;IAhBzD,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,gBAAgB,CAAK;IAE7B,IAAW,MAAM,WAEhB;IACD,IAAW,kBAAkB,WAE5B;IAED,IAAW,uBAAuB,IAAI,MAAM,GAAG,SAAS,CAIvD;gBAE2B,OAAO,EAAE,oBAAoB;IAElD,IAAI,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO;IAiC3C,IAAW,KAAK,YAEf;IAEM,QAAQ,IAAI,MAAM;IAazB;;OAEG;IACI,UAAU,IAAI,gBAAgB;CAerC"}

package/lib/opLifecycle/batchManager.js

@@ -2,25 +2,14 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { UsageError } from "@fluidframework/driver-utils";
-import { ChildLogger } from "@fluidframework/telemetry-utils";
 /**
  * Helper class that manages partial batch & rollback.
  */
 export class BatchManager {
-    constructor(options, logger) {
+    constructor(options) {
         this.options = options;
         this.pendingBatch = [];
         this.batchContentSize = 0;
-        /**
-         * Track the number of ops which were detected to have a mismatched
-         * reference sequence number, in order to self-throttle the telemetry events.
-         *
-         * This should be removed as part of ADO:2322
-         */
-        this.maxMismatchedOpsToReport = 5;
-        this.mismatchedOpsReported = 0;
-        this.logger = ChildLogger.create(logger, "BatchManager");
     }
     get length() {
         return this.pendingBatch.length;
@@ -28,9 +17,13 @@ export class BatchManager {
     get contentSizeInBytes() {
         return this.batchContentSize;
     }
+    get referenceSequenceNumber() {
+        return this.pendingBatch.length === 0
+            ? undefined
+            : this.pendingBatch[this.pendingBatch.length - 1].referenceSequenceNumber;
+    }
     push(message) {
         var _a, _b;
-        this.checkReferenceSequenceNumber(message);
         const contentSize = this.batchContentSize + ((_b = (_a = message.contents) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0);
         const opCount = this.pendingBatch.length;
         // Attempt to estimate batch size, aka socket message size.
@@ -63,6 +56,7 @@ export class BatchManager {
         const batch = {
             content: this.pendingBatch,
             contentSizeInBytes: this.batchContentSize,
+            referenceSequenceNumber: this.referenceSequenceNumber,
         };
         this.pendingBatch = [];
         this.batchContentSize = 0;
@@ -86,29 +80,6 @@ export class BatchManager {
             },
         };
     }
-    checkReferenceSequenceNumber(message) {
-        if (this.pendingBatch.length === 0 ||
-            message.referenceSequenceNumber === this.pendingBatch[0].referenceSequenceNumber) {
-            // The reference sequence numbers are stable
-            return;
-        }
-        const telemetryProperties = {
-            referenceSequenceNumber: this.pendingBatch[0].referenceSequenceNumber,
-            messageReferenceSequenceNumber: message.referenceSequenceNumber,
-            type: message.deserializedContent.type,
-            length: this.pendingBatch.length,
-            enableOpReentryCheck: this.options.enableOpReentryCheck === true,
-        };
-        const error = new UsageError("Submission of an out of order message");
-        const eventName = "ReferenceSequenceNumberMismatch";
-        if (this.options.enableOpReentryCheck === true) {
-            this.logger.sendErrorEvent(Object.assign({ eventName }, telemetryProperties), error);
-            throw error;
-        }
-        if (++this.mismatchedOpsReported <= this.maxMismatchedOpsToReport) {
-            this.logger.sendErrorEvent(Object.assign(Object.assign({ eventName }, telemetryProperties), { ops: this.mismatchedOpsReported, maxOps: this.maxMismatchedOpsToReport }), error);
-        }
-    }
 }
 const addBatchMetadata = (batch) => {
     if (batch.content.length > 1) {

package/lib/opLifecycle/batchManager.js.map

@@ -1 +1 @@
-{"version":3,"file":"batchManager.js","sourceRoot":"","sources":["../../src/opLifecycle/batchManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAW9D;;GAEG;AACH,MAAM,OAAO,YAAY;IAoBxB,YAA4B,OAA6B,EAAE,MAAwB;QAAvD,YAAO,GAAP,OAAO,CAAsB;QAlBjD,iBAAY,GAAmB,EAAE,CAAC;QAClC,qBAAgB,GAAG,CAAC,CAAC;QAC7B;;;;;WAKG;QACc,6BAAwB,GAAG,CAAC,CAAC;QACtC,0BAAqB,GAAG,CAAC,CAAC;QAUjC,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC1D,CAAC;IATD,IAAW,MAAM;QAChB,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;IACjC,CAAC;IACD,IAAW,kBAAkB;QAC5B,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC9B,CAAC;IAMM,IAAI,CAAC,OAAqB;;QAChC,IAAI,CAAC,4BAA4B,CAAC,OAAO,CAAC,CAAC;QAE3C,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,GAAG,CAAC,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC,CAAC;QAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAEzC,2DAA2D;QAC3D,iEAAiE;QACjE,sGAAsG;QACtG,iGAAiG;QACjG,gFAAgF;QAChF,MAAM,iBAAiB,GAAG,WAAW,GAAG,GAAG,GAAG,OAAO,CAAC;QAEtD,0DAA0D;QAC1D,wGAAwG;QACxG,0DAA0D;QAC1D,yGAAyG;QACzG,mGAAmG;QACnG,IACC,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS;YACpC,IAAI,CAAC,MAAM,GAAG,CAAC;YACf,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAC1C;YACD,OAAO,KAAK,CAAC;SACb;QAED,IAAI,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAChD,OAAO,KAAK,CAAC;SACb;QAED,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC;QACpC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAW,KAAK;QACf,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC;IACvC,CAAC;IAEM,QAAQ;QACd,MAAM,KAAK,GAAW;YACrB,OAAO,EAAE,IAAI,CAAC,YAAY;YAC1B,kBAAkB,EAAE,IAAI,CAAC,gBAAgB;SACzC,CAAC;QAEF,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAE1B,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACI,UAAU;QAChB,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5C,OAAO;YACN,QAAQ,EAAE,CAAC,OAAwC,EAAE,EAAE;;gBACtD,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,GAAG,UAAU,GAAI;oBACxD,CAAC,EAAE,CAAC;oBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBACrC,IAAI,CAAC,gBAAgB,IAAI,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC;oBACvD,OAAO,CAAC,OAAO,CAAC,CAAC;iBACjB;gBAED,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU,CAAC;YACvC,CAAC;SACD,CAAC;IACH,CAAC;IAEO,4BAA4B,CAAC,OAAqB;QACzD,IACC,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAC9B,OAAO,CAAC,uBAAuB,KAAK,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,uBAAuB,EAC/E;YACD,4CAA4C;YAC5C,OAAO;SACP;QAED,MAAM,mBAAmB,GAAG;YAC3B,uBAAuB,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,uBAAuB;YACrE,8BAA8B,EAAE,OAAO,CAAC,uBAAuB;YAC/D,IAAI,EAAE,OAAO,CAAC,mBAAmB,CAAC,IAAI;YACtC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM;YAChC,oBAAoB,EAAE,IAAI,CAAC,OAAO,CAAC,oBAAoB,KAAK,IAAI;SAChE,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,uCAAuC,CAAC,CAAC;QACtE,MAAM,SAAS,GAAG,iCAAiC,CAAC;QAEpD,IAAI,IAAI,CAAC,OAAO,CAAC,oBAAoB,KAAK,IAAI,EAAE;YAC/C,IAAI,CAAC,MAAM,CAAC,cAAc,iBAAG,SAAS,IAAK,mBAAmB,GAAI,KAAK,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC;SACZ;QAED,IAAI,EAAE,IAAI,CAAC,qBAAqB,IAAI,IAAI,CAAC,wBAAwB,EAAE;YAClE,IAAI,CAAC,MAAM,CAAC,cAAc,+BAExB,SAAS,IACN,mBAAmB,KACtB,GAAG,EAAE,IAAI,CAAC,qBAAqB,EAC/B,MAAM,EAAE,IAAI,CAAC,wBAAwB,KAEtC,KAAK,CACL,CAAC;SACF;IACF,CAAC;CACD;AAED,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAU,EAAE;IAClD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,mCACrB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,KAC5B,KAAK,EAAE,IAAI,GACX,CAAC;QACF,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,mCAC5C,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,KACnD,KAAK,EAAE,KAAK,GACZ,CAAC;KACF;IAED,OAAO,KAAK,CAAC;AACd,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { UsageError } from \"@fluidframework/driver-utils\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { ICompressionRuntimeOptions } from \"../containerRuntime\";\nimport { BatchMessage, IBatch, IBatchCheckpoint } from \"./definitions\";\n\nexport interface IBatchManagerOptions {\n\treadonly enableOpReentryCheck?: boolean;\n\treadonly hardLimit: number;\n\treadonly softLimit?: number;\n\treadonly compressionOptions?: ICompressionRuntimeOptions;\n}\n\n/**\n * Helper class that manages partial batch & rollback.\n */\nexport class BatchManager {\n\tprivate readonly logger;\n\tprivate pendingBatch: BatchMessage[] = [];\n\tprivate batchContentSize = 0;\n\t/**\n\t * Track the number of ops which were detected to have a mismatched\n\t * reference sequence number, in order to self-throttle the telemetry events.\n\t *\n\t * This should be removed as part of ADO:2322\n\t */\n\tprivate readonly maxMismatchedOpsToReport = 5;\n\tprivate mismatchedOpsReported = 0;\n\n\tpublic get length() {\n\t\treturn this.pendingBatch.length;\n\t}\n\tpublic get contentSizeInBytes() {\n\t\treturn this.batchContentSize;\n\t}\n\n\tconstructor(public readonly options: IBatchManagerOptions, logger: ITelemetryLogger) {\n\t\tthis.logger = ChildLogger.create(logger, \"BatchManager\");\n\t}\n\n\tpublic push(message: BatchMessage): boolean {\n\t\tthis.checkReferenceSequenceNumber(message);\n\n\t\tconst contentSize = this.batchContentSize + (message.contents?.length ?? 0);\n\t\tconst opCount = this.pendingBatch.length;\n\n\t\t// Attempt to estimate batch size, aka socket message size.\n\t\t// Each op has pretty large envelope, estimating to be 200 bytes.\n\t\t// Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.\n\t\t// Not taking it into account, as compression work should help there - compressed payload will be\n\t\t// initially stored as base64, and that requires only 2 extra escape characters.\n\t\tconst socketMessageSize = contentSize + 200 * opCount;\n\n\t\t// If we were provided soft limit, check for exceeding it.\n\t\t// But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)\n\t\t// and start over. That's not an option if we have no ops.\n\t\t// If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.\n\t\t// Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.\n\t\tif (\n\t\t\tthis.options.softLimit !== undefined &&\n\t\t\tthis.length > 0 &&\n\t\t\tsocketMessageSize >= this.options.softLimit\n\t\t) {\n\t\t\treturn false;\n\t\t}\n\n\t\tif (socketMessageSize >= this.options.hardLimit) {\n\t\t\treturn false;\n\t\t}\n\n\t\tthis.batchContentSize = contentSize;\n\t\tthis.pendingBatch.push(message);\n\t\treturn true;\n\t}\n\n\tpublic get empty() {\n\t\treturn this.pendingBatch.length === 0;\n\t}\n\n\tpublic popBatch(): IBatch {\n\t\tconst batch: IBatch = {\n\t\t\tcontent: this.pendingBatch,\n\t\t\tcontentSizeInBytes: this.batchContentSize,\n\t\t};\n\n\t\tthis.pendingBatch = [];\n\t\tthis.batchContentSize = 0;\n\n\t\treturn addBatchMetadata(batch);\n\t}\n\n\t/**\n\t * Capture the pending state at this point\n\t */\n\tpublic checkpoint(): IBatchCheckpoint {\n\t\tconst startPoint = this.pendingBatch.length;\n\t\treturn {\n\t\t\trollback: (process: (message: BatchMessage) => void) => {\n\t\t\t\tfor (let i = this.pendingBatch.length; i > startPoint; ) {\n\t\t\t\t\ti--;\n\t\t\t\t\tconst message = this.pendingBatch[i];\n\t\t\t\t\tthis.batchContentSize -= message.contents?.length ?? 0;\n\t\t\t\t\tprocess(message);\n\t\t\t\t}\n\n\t\t\t\tthis.pendingBatch.length = startPoint;\n\t\t\t},\n\t\t};\n\t}\n\n\tprivate checkReferenceSequenceNumber(message: BatchMessage) {\n\t\tif (\n\t\t\tthis.pendingBatch.length === 0 ||\n\t\t\tmessage.referenceSequenceNumber === this.pendingBatch[0].referenceSequenceNumber\n\t\t) {\n\t\t\t// The reference sequence numbers are stable\n\t\t\treturn;\n\t\t}\n\n\t\tconst telemetryProperties = {\n\t\t\treferenceSequenceNumber: this.pendingBatch[0].referenceSequenceNumber,\n\t\t\tmessageReferenceSequenceNumber: message.referenceSequenceNumber,\n\t\t\ttype: message.deserializedContent.type,\n\t\t\tlength: this.pendingBatch.length,\n\t\t\tenableOpReentryCheck: this.options.enableOpReentryCheck === true,\n\t\t};\n\t\tconst error = new UsageError(\"Submission of an out of order message\");\n\t\tconst eventName = \"ReferenceSequenceNumberMismatch\";\n\n\t\tif (this.options.enableOpReentryCheck === true) {\n\t\t\tthis.logger.sendErrorEvent({ eventName, ...telemetryProperties }, error);\n\t\t\tthrow error;\n\t\t}\n\n\t\tif (++this.mismatchedOpsReported <= this.maxMismatchedOpsToReport) {\n\t\t\tthis.logger.sendErrorEvent(\n\t\t\t\t{\n\t\t\t\t\teventName,\n\t\t\t\t\t...telemetryProperties,\n\t\t\t\t\tops: this.mismatchedOpsReported,\n\t\t\t\t\tmaxOps: this.maxMismatchedOpsToReport,\n\t\t\t\t},\n\t\t\t\terror,\n\t\t\t);\n\t\t}\n\t}\n}\n\nconst addBatchMetadata = (batch: IBatch): IBatch => {\n\tif (batch.content.length > 1) {\n\t\tbatch.content[0].metadata = {\n\t\t\t...batch.content[0].metadata,\n\t\t\tbatch: true,\n\t\t};\n\t\tbatch.content[batch.content.length - 1].metadata = {\n\t\t\t...batch.content[batch.content.length - 1].metadata,\n\t\t\tbatch: false,\n\t\t};\n\t}\n\n\treturn batch;\n};\n"]}
+{"version":3,"file":"batchManager.js","sourceRoot":"","sources":["../../src/opLifecycle/batchManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAWH;;GAEG;AACH,MAAM,OAAO,YAAY;IAiBxB,YAA4B,OAA6B;QAA7B,YAAO,GAAP,OAAO,CAAsB;QAhBjD,iBAAY,GAAmB,EAAE,CAAC;QAClC,qBAAgB,GAAG,CAAC,CAAC;IAe+B,CAAC;IAb7D,IAAW,MAAM;QAChB,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;IACjC,CAAC;IACD,IAAW,kBAAkB;QAC5B,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC9B,CAAC;IAED,IAAW,uBAAuB;QACjC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YACpC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,uBAAuB,CAAC;IAC5E,CAAC;IAIM,IAAI,CAAC,OAAqB;;QAChC,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,GAAG,CAAC,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC,CAAC;QAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAEzC,2DAA2D;QAC3D,iEAAiE;QACjE,sGAAsG;QACtG,iGAAiG;QACjG,gFAAgF;QAChF,MAAM,iBAAiB,GAAG,WAAW,GAAG,GAAG,GAAG,OAAO,CAAC;QAEtD,0DAA0D;QAC1D,wGAAwG;QACxG,0DAA0D;QAC1D,yGAAyG;QACzG,mGAAmG;QACnG,IACC,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS;YACpC,IAAI,CAAC,MAAM,GAAG,CAAC;YACf,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAC1C;YACD,OAAO,KAAK,CAAC;SACb;QAED,IAAI,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAChD,OAAO,KAAK,CAAC;SACb;QAED,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC;QACpC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAW,KAAK;QACf,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC;IACvC,CAAC;IAEM,QAAQ;QACd,MAAM,KAAK,GAAW;YACrB,OAAO,EAAE,IAAI,CAAC,YAAY;YAC1B,kBAAkB,EAAE,IAAI,CAAC,gBAAgB;YACzC,uBAAuB,EAAE,IAAI,CAAC,uBAAuB;SACrD,CAAC;QAEF,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAE1B,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACI,UAAU;QAChB,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5C,OAAO;YACN,QAAQ,EAAE,CAAC,OAAwC,EAAE,EAAE;;gBACtD,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,GAAG,UAAU,GAAI;oBACxD,CAAC,EAAE,CAAC;oBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBACrC,IAAI,CAAC,gBAAgB,IAAI,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC;oBACvD,OAAO,CAAC,OAAO,CAAC,CAAC;iBACjB;gBAED,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU,CAAC;YACvC,CAAC;SACD,CAAC;IACH,CAAC;CACD;AAED,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAU,EAAE;IAClD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,mCACrB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,KAC5B,KAAK,EAAE,IAAI,GACX,CAAC;QACF,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,mCAC5C,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,KACnD,KAAK,EAAE,KAAK,GACZ,CAAC;KACF;IAED,OAAO,KAAK,CAAC;AACd,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ICompressionRuntimeOptions } from \"../containerRuntime\";\nimport { BatchMessage, IBatch, IBatchCheckpoint } from \"./definitions\";\n\nexport interface IBatchManagerOptions {\n\treadonly hardLimit: number;\n\treadonly softLimit?: number;\n\treadonly compressionOptions?: ICompressionRuntimeOptions;\n}\n\n/**\n * Helper class that manages partial batch & rollback.\n */\nexport class BatchManager {\n\tprivate pendingBatch: BatchMessage[] = [];\n\tprivate batchContentSize = 0;\n\n\tpublic get length() {\n\t\treturn this.pendingBatch.length;\n\t}\n\tpublic get contentSizeInBytes() {\n\t\treturn this.batchContentSize;\n\t}\n\n\tpublic get referenceSequenceNumber(): number | undefined {\n\t\treturn this.pendingBatch.length === 0\n\t\t\t? undefined\n\t\t\t: this.pendingBatch[this.pendingBatch.length - 1].referenceSequenceNumber;\n\t}\n\n\tconstructor(public readonly options: IBatchManagerOptions) {}\n\n\tpublic push(message: BatchMessage): boolean {\n\t\tconst contentSize = this.batchContentSize + (message.contents?.length ?? 0);\n\t\tconst opCount = this.pendingBatch.length;\n\n\t\t// Attempt to estimate batch size, aka socket message size.\n\t\t// Each op has pretty large envelope, estimating to be 200 bytes.\n\t\t// Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.\n\t\t// Not taking it into account, as compression work should help there - compressed payload will be\n\t\t// initially stored as base64, and that requires only 2 extra escape characters.\n\t\tconst socketMessageSize = contentSize + 200 * opCount;\n\n\t\t// If we were provided soft limit, check for exceeding it.\n\t\t// But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)\n\t\t// and start over. That's not an option if we have no ops.\n\t\t// If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.\n\t\t// Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.\n\t\tif (\n\t\t\tthis.options.softLimit !== undefined &&\n\t\t\tthis.length > 0 &&\n\t\t\tsocketMessageSize >= this.options.softLimit\n\t\t) {\n\t\t\treturn false;\n\t\t}\n\n\t\tif (socketMessageSize >= this.options.hardLimit) {\n\t\t\treturn false;\n\t\t}\n\n\t\tthis.batchContentSize = contentSize;\n\t\tthis.pendingBatch.push(message);\n\t\treturn true;\n\t}\n\n\tpublic get empty() {\n\t\treturn this.pendingBatch.length === 0;\n\t}\n\n\tpublic popBatch(): IBatch {\n\t\tconst batch: IBatch = {\n\t\t\tcontent: this.pendingBatch,\n\t\t\tcontentSizeInBytes: this.batchContentSize,\n\t\t\treferenceSequenceNumber: this.referenceSequenceNumber,\n\t\t};\n\n\t\tthis.pendingBatch = [];\n\t\tthis.batchContentSize = 0;\n\n\t\treturn addBatchMetadata(batch);\n\t}\n\n\t/**\n\t * Capture the pending state at this point\n\t */\n\tpublic checkpoint(): IBatchCheckpoint {\n\t\tconst startPoint = this.pendingBatch.length;\n\t\treturn {\n\t\t\trollback: (process: (message: BatchMessage) => void) => {\n\t\t\t\tfor (let i = this.pendingBatch.length; i > startPoint; ) {\n\t\t\t\t\ti--;\n\t\t\t\t\tconst message = this.pendingBatch[i];\n\t\t\t\t\tthis.batchContentSize -= message.contents?.length ?? 0;\n\t\t\t\t\tprocess(message);\n\t\t\t\t}\n\n\t\t\t\tthis.pendingBatch.length = startPoint;\n\t\t\t},\n\t\t};\n\t}\n}\n\nconst addBatchMetadata = (batch: IBatch): IBatch => {\n\tif (batch.content.length > 1) {\n\t\tbatch.content[0].metadata = {\n\t\t\t...batch.content[0].metadata,\n\t\t\tbatch: true,\n\t\t};\n\t\tbatch.content[batch.content.length - 1].metadata = {\n\t\t\t...batch.content[batch.content.length - 1].metadata,\n\t\t\tbatch: false,\n\t\t};\n\t}\n\n\treturn batch;\n};\n"]}

package/lib/opLifecycle/definitions.d.ts

@@ -27,6 +27,10 @@ export interface IBatch {
      * All the messages in the batch
      */
     readonly content: BatchMessage[];
+    /**
+     * The reference sequence number for the batch
+     */
+    readonly referenceSequenceNumber: number | undefined;
 }
 export interface IBatchCheckpoint {
     rollback: (action: (message: BatchMessage) => void) => void;

package/lib/opLifecycle/definitions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"definitions.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/definitions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AACtE,OAAO,EAAE,yBAAyB,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAC9F,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,IAAI,CAAC;AAE1F;;GAEG;AACH,oBAAY,YAAY,GAAG,aAAa,GAAG;IAC1C,eAAe,EAAE,OAAO,CAAC;IACzB,mBAAmB,EAAE,uBAAuB,CAAC;IAC7C,uBAAuB,EAAE,MAAM,CAAC;IAChC,WAAW,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,MAAM;IACtB;;;OAGG;IACH,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,KAAK,IAAI,CAAC;CAC5D;AAED,MAAM,WAAW,UAAU;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,WAAW,GAAG,oBAAoB,CAAC;IACjD,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3C,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,oBAAY,eAAe,GAAG,WAAW,GAAG,SAAS,GAAG,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,yBAAyB,CAAC;IAC5C;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;CAChC"}
+{"version":3,"file":"definitions.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/definitions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AACtE,OAAO,EAAE,yBAAyB,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAC9F,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,IAAI,CAAC;AAE1F;;GAEG;AACH,oBAAY,YAAY,GAAG,aAAa,GAAG;IAC1C,eAAe,EAAE,OAAO,CAAC;IACzB,mBAAmB,EAAE,uBAAuB,CAAC;IAC7C,uBAAuB,EAAE,MAAM,CAAC;IAChC,WAAW,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,MAAM;IACtB;;;OAGG;IACH,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC;IACjC;;OAEG;IACH,QAAQ,CAAC,uBAAuB,EAAE,MAAM,GAAG,SAAS,CAAC;CACrD;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,KAAK,IAAI,CAAC;CAC5D;AAED,MAAM,WAAW,UAAU;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,WAAW,GAAG,oBAAoB,CAAC;IACjD,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3C,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,oBAAY,eAAe,GAAG,WAAW,GAAG,SAAS,GAAG,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,yBAAyB,CAAC;IAC5C;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;CAChC"}

package/lib/opLifecycle/definitions.js.map

@@ -1 +1 @@
-{"version":3,"file":"definitions.js","sourceRoot":"","sources":["../../src/opLifecycle/definitions.ts"],"names":[],"mappings":"AAAA;;;GAGG","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport { ISequencedDocumentMessage, MessageType } from \"@fluidframework/protocol-definitions\";\nimport { CompressionAlgorithms, ContainerMessageType, ContainerRuntimeMessage } from \"..\";\n\n/**\n * Batch message type used internally by the runtime\n */\nexport type BatchMessage = IBatchMessage & {\n\tlocalOpMetadata: unknown;\n\tdeserializedContent: ContainerRuntimeMessage;\n\treferenceSequenceNumber: number;\n\tcompression?: CompressionAlgorithms;\n};\n\n/**\n * Batch interface used internally by the runtime.\n */\nexport interface IBatch {\n\t/**\n\t * Sum of the in-memory content sizes of all messages in the batch.\n\t * If the batch is compressed, this number reflects the post-compression size.\n\t */\n\treadonly contentSizeInBytes: number;\n\t/**\n\t * All the messages in the batch\n\t */\n\treadonly content: BatchMessage[];\n}\n\nexport interface IBatchCheckpoint {\n\trollback: (action: (message: BatchMessage) => void) => void;\n}\n\nexport interface IChunkedOp {\n\tchunkId: number;\n\ttotalChunks: number;\n\tcontents: string;\n\toriginalType: MessageType | ContainerMessageType;\n\toriginalMetadata?: Record<string, unknown>;\n\toriginalCompression?: string;\n}\n\n/**\n * The state of remote message processing:\n * `Processed` - the message can be considered processed\n * `Skipped` - the message was ignored by the processor\n * `Accepted` - the message was processed partially. Eventually, a message\n * will make the processor return `Processed`.\n */\nexport type ProcessingState = \"Processed\" | \"Skipped\" | \"Accepted\";\n\n/**\n * Return type for functions which process remote messages\n */\nexport interface IMessageProcessingResult {\n\t/**\n\t * A shallow copy of the input message if processing happened, or\n\t * the original message otherwise\n\t */\n\treadonly message: ISequencedDocumentMessage;\n\t/**\n\t * Processing result of the input message.\n\t */\n\treadonly state: ProcessingState;\n}\n"]}
+{"version":3,"file":"definitions.js","sourceRoot":"","sources":["../../src/opLifecycle/definitions.ts"],"names":[],"mappings":"AAAA;;;GAGG","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport { ISequencedDocumentMessage, MessageType } from \"@fluidframework/protocol-definitions\";\nimport { CompressionAlgorithms, ContainerMessageType, ContainerRuntimeMessage } from \"..\";\n\n/**\n * Batch message type used internally by the runtime\n */\nexport type BatchMessage = IBatchMessage & {\n\tlocalOpMetadata: unknown;\n\tdeserializedContent: ContainerRuntimeMessage;\n\treferenceSequenceNumber: number;\n\tcompression?: CompressionAlgorithms;\n};\n\n/**\n * Batch interface used internally by the runtime.\n */\nexport interface IBatch {\n\t/**\n\t * Sum of the in-memory content sizes of all messages in the batch.\n\t * If the batch is compressed, this number reflects the post-compression size.\n\t */\n\treadonly contentSizeInBytes: number;\n\t/**\n\t * All the messages in the batch\n\t */\n\treadonly content: BatchMessage[];\n\t/**\n\t * The reference sequence number for the batch\n\t */\n\treadonly referenceSequenceNumber: number | undefined;\n}\n\nexport interface IBatchCheckpoint {\n\trollback: (action: (message: BatchMessage) => void) => void;\n}\n\nexport interface IChunkedOp {\n\tchunkId: number;\n\ttotalChunks: number;\n\tcontents: string;\n\toriginalType: MessageType | ContainerMessageType;\n\toriginalMetadata?: Record<string, unknown>;\n\toriginalCompression?: string;\n}\n\n/**\n * The state of remote message processing:\n * `Processed` - the message can be considered processed\n * `Skipped` - the message was ignored by the processor\n * `Accepted` - the message was processed partially. Eventually, a message\n * will make the processor return `Processed`.\n */\nexport type ProcessingState = \"Processed\" | \"Skipped\" | \"Accepted\";\n\n/**\n * Return type for functions which process remote messages\n */\nexport interface IMessageProcessingResult {\n\t/**\n\t * A shallow copy of the input message if processing happened, or\n\t * the original message otherwise\n\t */\n\treadonly message: ISequencedDocumentMessage;\n\t/**\n\t * Processing result of the input message.\n\t */\n\treadonly state: ProcessingState;\n}\n"]}

package/lib/opLifecycle/opCompressor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"opCompressor.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAMtE,OAAO,EAAE,MAAM,EAAgB,MAAM,eAAe,CAAC;AAErD;;;;GAIG;AACH,qBAAa,YAAY;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAEZ,MAAM,EAAE,gBAAgB;IAI7B,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAkC3C,OAAO,CAAC,cAAc;CAsBtB"}
+{"version":3,"file":"opCompressor.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAMtE,OAAO,EAAE,MAAM,EAAgB,MAAM,eAAe,CAAC;AAErD;;;;GAIG;AACH,qBAAa,YAAY;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAEZ,MAAM,EAAE,gBAAgB;IAI7B,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAmC3C,OAAO,CAAC,cAAc;CAsBtB"}

package/lib/opLifecycle/opCompressor.js

@@ -38,6 +38,7 @@ export class OpCompressor {
         return {
             contentSizeInBytes: compressedContent.length,
             content: messages,
+            referenceSequenceNumber: batch.referenceSequenceNumber,
         };
     }
     serializeBatch(batch) {

package/lib/opLifecycle/opCompressor.js.map

@@ -1 +1 @@
-{"version":3,"file":"opCompressor.js","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AACjC,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAG5D;;;;GAIG;AACH,MAAM,OAAO,YAAY;IAGxB,YAAY,MAAwB;QACnC,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC1D,CAAC;IAEM,aAAa,CAAC,KAAa;QACjC,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9E,MAAM,kBAAkB,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;QAE/C,IAAI,KAAK,CAAC,kBAAkB,GAAG,MAAM,EAAE;YACtC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAChC,SAAS,EAAE,iBAAiB;gBAC5B,QAAQ;gBACR,qBAAqB,EAAE,KAAK,CAAC,kBAAkB;gBAC/C,oBAAoB,EAAE,iBAAiB,CAAC,MAAM;aAC9C,CAAC,CAAC;SACH;QAED,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,QAAQ,CAAC,IAAI,iCACT,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KACnB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC,EAC/D,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,EACnC,WAAW,EAAE,qBAAqB,CAAC,GAAG,IACrC,CAAC;QAEH,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC7C,QAAQ,CAAC,IAAI,iCAAM,OAAO,KAAE,QAAQ,EAAE,SAAS,IAAG,CAAC;SACnD;QAED,OAAO;YACN,kBAAkB,EAAE,iBAAiB,CAAC,MAAM;YAC5C,OAAO,EAAE,QAAQ;SACjB,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,KAAa;QACnC,IAAI;YACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC;SACnF;QAAC,OAAO,CAAM,EAAE;YAChB,IAAI,CAAC,CAAC,OAAO,KAAK,uBAAuB,EAAE;gBAC1C,0CAA0C;gBAC1C,wCAAwC;gBACxC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,mBAAmB,CAAC,CAAC;gBAClD,IAAI,CAAC,MAAM,CAAC,cAAc,CACzB;oBACC,SAAS,EAAE,eAAe;oBAC1B,IAAI,EAAE,KAAK,CAAC,kBAAkB;oBAC9B,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;iBAC5B,EACD,KAAK,CACL,CAAC;gBACF,MAAM,KAAK,CAAC;aACZ;YAED,MAAM,CAAC,CAAC;SACR;IACF,CAAC;CACD","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { IsoBuffer } from \"@fluidframework/common-utils\";\nimport { UsageError } from \"@fluidframework/container-utils\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { compress } from \"lz4js\";\nimport { CompressionAlgorithms } from \"../containerRuntime\";\nimport { IBatch, BatchMessage } from \"./definitions\";\n\n/**\n * Compresses batches of ops. It generates a single compressed op that contains\n * the contents of each op in the batch. It then submits empty ops for each original\n * op to reserve sequence numbers.\n */\nexport class OpCompressor {\n\tprivate readonly logger;\n\n\tconstructor(logger: ITelemetryLogger) {\n\t\tthis.logger = ChildLogger.create(logger, \"OpCompressor\");\n\t}\n\n\tpublic compressBatch(batch: IBatch): IBatch {\n\t\tconst compressionStart = Date.now();\n\t\tconst contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));\n\t\tconst compressedContents = compress(contentsAsBuffer);\n\t\tconst compressedContent = IsoBuffer.from(compressedContents).toString(\"base64\");\n\t\tconst duration = Date.now() - compressionStart;\n\n\t\tif (batch.contentSizeInBytes > 200000) {\n\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\teventName: \"CompressedBatch\",\n\t\t\t\tduration,\n\t\t\t\tsizeBeforeCompression: batch.contentSizeInBytes,\n\t\t\t\tsizeAfterCompression: compressedContent.length,\n\t\t\t});\n\t\t}\n\n\t\tconst messages: BatchMessage[] = [];\n\t\tmessages.push({\n\t\t\t...batch.content[0],\n\t\t\tcontents: JSON.stringify({ packedContents: compressedContent }),\n\t\t\tmetadata: batch.content[0].metadata,\n\t\t\tcompression: CompressionAlgorithms.lz4,\n\t\t});\n\n\t\tfor (const message of batch.content.slice(1)) {\n\t\t\tmessages.push({ ...message, contents: undefined });\n\t\t}\n\n\t\treturn {\n\t\t\tcontentSizeInBytes: compressedContent.length,\n\t\t\tcontent: messages,\n\t\t};\n\t}\n\n\tprivate serializeBatch(batch: IBatch): string {\n\t\ttry {\n\t\t\treturn JSON.stringify(batch.content.map((message) => message.deserializedContent));\n\t\t} catch (e: any) {\n\t\t\tif (e.message === \"Invalid string length\") {\n\t\t\t\t// This is how JSON.stringify signals that\n\t\t\t\t// the content size exceeds its capacity\n\t\t\t\tconst error = new UsageError(\"Payload too large\");\n\t\t\t\tthis.logger.sendErrorEvent(\n\t\t\t\t\t{\n\t\t\t\t\t\teventName: \"BatchTooLarge\",\n\t\t\t\t\t\tsize: batch.contentSizeInBytes,\n\t\t\t\t\t\tlength: batch.content.length,\n\t\t\t\t\t},\n\t\t\t\t\terror,\n\t\t\t\t);\n\t\t\t\tthrow error;\n\t\t\t}\n\n\t\t\tthrow e;\n\t\t}\n\t}\n}\n"]}
+{"version":3,"file":"opCompressor.js","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AACjC,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAG5D;;;;GAIG;AACH,MAAM,OAAO,YAAY;IAGxB,YAAY,MAAwB;QACnC,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC1D,CAAC;IAEM,aAAa,CAAC,KAAa;QACjC,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9E,MAAM,kBAAkB,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;QAE/C,IAAI,KAAK,CAAC,kBAAkB,GAAG,MAAM,EAAE;YACtC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAChC,SAAS,EAAE,iBAAiB;gBAC5B,QAAQ;gBACR,qBAAqB,EAAE,KAAK,CAAC,kBAAkB;gBAC/C,oBAAoB,EAAE,iBAAiB,CAAC,MAAM;aAC9C,CAAC,CAAC;SACH;QAED,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,QAAQ,CAAC,IAAI,iCACT,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KACnB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC,EAC/D,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,EACnC,WAAW,EAAE,qBAAqB,CAAC,GAAG,IACrC,CAAC;QAEH,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC7C,QAAQ,CAAC,IAAI,iCAAM,OAAO,KAAE,QAAQ,EAAE,SAAS,IAAG,CAAC;SACnD;QAED,OAAO;YACN,kBAAkB,EAAE,iBAAiB,CAAC,MAAM;YAC5C,OAAO,EAAE,QAAQ;YACjB,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;SACtD,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,KAAa;QACnC,IAAI;YACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC;SACnF;QAAC,OAAO,CAAM,EAAE;YAChB,IAAI,CAAC,CAAC,OAAO,KAAK,uBAAuB,EAAE;gBAC1C,0CAA0C;gBAC1C,wCAAwC;gBACxC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,mBAAmB,CAAC,CAAC;gBAClD,IAAI,CAAC,MAAM,CAAC,cAAc,CACzB;oBACC,SAAS,EAAE,eAAe;oBAC1B,IAAI,EAAE,KAAK,CAAC,kBAAkB;oBAC9B,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;iBAC5B,EACD,KAAK,CACL,CAAC;gBACF,MAAM,KAAK,CAAC;aACZ;YAED,MAAM,CAAC,CAAC;SACR;IACF,CAAC;CACD","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { IsoBuffer } from \"@fluidframework/common-utils\";\nimport { UsageError } from \"@fluidframework/container-utils\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { compress } from \"lz4js\";\nimport { CompressionAlgorithms } from \"../containerRuntime\";\nimport { IBatch, BatchMessage } from \"./definitions\";\n\n/**\n * Compresses batches of ops. It generates a single compressed op that contains\n * the contents of each op in the batch. It then submits empty ops for each original\n * op to reserve sequence numbers.\n */\nexport class OpCompressor {\n\tprivate readonly logger;\n\n\tconstructor(logger: ITelemetryLogger) {\n\t\tthis.logger = ChildLogger.create(logger, \"OpCompressor\");\n\t}\n\n\tpublic compressBatch(batch: IBatch): IBatch {\n\t\tconst compressionStart = Date.now();\n\t\tconst contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));\n\t\tconst compressedContents = compress(contentsAsBuffer);\n\t\tconst compressedContent = IsoBuffer.from(compressedContents).toString(\"base64\");\n\t\tconst duration = Date.now() - compressionStart;\n\n\t\tif (batch.contentSizeInBytes > 200000) {\n\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\teventName: \"CompressedBatch\",\n\t\t\t\tduration,\n\t\t\t\tsizeBeforeCompression: batch.contentSizeInBytes,\n\t\t\t\tsizeAfterCompression: compressedContent.length,\n\t\t\t});\n\t\t}\n\n\t\tconst messages: BatchMessage[] = [];\n\t\tmessages.push({\n\t\t\t...batch.content[0],\n\t\t\tcontents: JSON.stringify({ packedContents: compressedContent }),\n\t\t\tmetadata: batch.content[0].metadata,\n\t\t\tcompression: CompressionAlgorithms.lz4,\n\t\t});\n\n\t\tfor (const message of batch.content.slice(1)) {\n\t\t\tmessages.push({ ...message, contents: undefined });\n\t\t}\n\n\t\treturn {\n\t\t\tcontentSizeInBytes: compressedContent.length,\n\t\t\tcontent: messages,\n\t\t\treferenceSequenceNumber: batch.referenceSequenceNumber,\n\t\t};\n\t}\n\n\tprivate serializeBatch(batch: IBatch): string {\n\t\ttry {\n\t\t\treturn JSON.stringify(batch.content.map((message) => message.deserializedContent));\n\t\t} catch (e: any) {\n\t\t\tif (e.message === \"Invalid string length\") {\n\t\t\t\t// This is how JSON.stringify signals that\n\t\t\t\t// the content size exceeds its capacity\n\t\t\t\tconst error = new UsageError(\"Payload too large\");\n\t\t\t\tthis.logger.sendErrorEvent(\n\t\t\t\t\t{\n\t\t\t\t\t\teventName: \"BatchTooLarge\",\n\t\t\t\t\t\tsize: batch.contentSizeInBytes,\n\t\t\t\t\t\tlength: batch.content.length,\n\t\t\t\t\t},\n\t\t\t\t\terror,\n\t\t\t\t);\n\t\t\t\tthrow error;\n\t\t\t}\n\n\t\t\tthrow e;\n\t\t}\n\t}\n}\n"]}

package/lib/opLifecycle/opSplitter.d.ts

@@ -15,7 +15,7 @@ export declare class OpSplitter {
     private readonly maxBatchSizeInBytes;
     private readonly chunkMap;
     private readonly logger;
-    constructor(chunks: [string, string[]][], submitBatchFn: ((batch: IBatchMessage[]) => number) | undefined, chunkSizeInBytes: number, maxBatchSizeInBytes: number, logger: ITelemetryLogger);
+    constructor(chunks: [string, string[]][], submitBatchFn: ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number) | undefined, chunkSizeInBytes: number, maxBatchSizeInBytes: number, logger: ITelemetryLogger);
     get isBatchChunkingEnabled(): boolean;
     get chunks(): ReadonlyMap<string, string[]>;
     processRemoteMessage(message: ISequencedDocumentMessage): IMessageProcessingResult;

package/lib/opLifecycle/opSplitter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"opSplitter.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAEtE,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AAKtE,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAGjF,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AAE3F;;GAEG;AACH,qBAAa,UAAU;IAOrB,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IACjC,OAAO,CAAC,QAAQ,CAAC,mBAAmB;IAPrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAGvB,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,EACX,aAAa,EAAE,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,MAAM,CAAC,GAAG,SAAS,EAC/D,gBAAgB,EAAE,MAAM,EACxB,mBAAmB,EAAE,MAAM,EAC5C,MAAM,EAAE,gBAAgB;IAMzB,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,IAAW,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAEjD;IAEM,oBAAoB,CAAC,OAAO,EAAE,yBAAyB,GAAG,wBAAwB;IAoClF,kBAAkB,CAAC,QAAQ,EAAE,MAAM;IAM1C,OAAO,CAAC,QAAQ;IA0BhB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACI,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;CAoDlD;AAoBD,eAAO,MAAM,OAAO,OAAQ,YAAY,oBAAoB,MAAM,KAAG,UAAU,EA+B9E,CAAC"}
+{"version":3,"file":"opSplitter.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAEtE,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AAKtE,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAGjF,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AAE3F;;GAEG;AACH,qBAAa,UAAU;IAOrB,OAAO,CAAC,QAAQ,CAAC,aAAa;IAG9B,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IACjC,OAAO,CAAC,QAAQ,CAAC,mBAAmB;IATrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAGvB,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,EACX,aAAa,EAC3B,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,EAAE,uBAAuB,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC,GACtE,SAAS,EACK,gBAAgB,EAAE,MAAM,EACxB,mBAAmB,EAAE,MAAM,EAC5C,MAAM,EAAE,gBAAgB;IAMzB,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,IAAW,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAEjD;IAEM,oBAAoB,CAAC,OAAO,EAAE,yBAAyB,GAAG,wBAAwB;IAoClF,kBAAkB,CAAC,QAAQ,EAAE,MAAM;IAM1C,OAAO,CAAC,QAAQ;IA0BhB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACI,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;CA4DlD;AAoBD,eAAO,MAAM,OAAO,OAAQ,YAAY,oBAAoB,MAAM,KAAG,UAAU,EAwC9E,CAAC"}

package/lib/opLifecycle/opSplitter.js

@@ -99,6 +99,7 @@ export class OpSplitter {
         var _a, _b, _c, _d, _e, _f;
         assert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);
         assert(batch.contentSizeInBytes > 0 && batch.content.length > 0, 0x514 /* Batch needs to be non-empty */);
+        assert(batch.referenceSequenceNumber !== undefined, 0x58a /* Batch must have a reference sequence number if non-empty */);
         assert(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);
         assert(this.chunkSizeInBytes < this.maxBatchSizeInBytes, 0x516 /* Chunk size needs to be smaller than the max batch size */);
         const firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split
@@ -109,11 +110,11 @@ export class OpSplitter {
         assert(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);
         // Send the first N-1 chunks immediately
         for (const chunk of chunks.slice(0, -1)) {
-            this.submitBatchFn([chunkToBatchMessage(chunk, firstMessage.referenceSequenceNumber)]);
+            this.submitBatchFn([chunkToBatchMessage(chunk, batch.referenceSequenceNumber)], batch.referenceSequenceNumber);
         }
         // The last chunk will be part of the new batch and needs to
         // preserve the batch metadata of the original batch
-        const lastChunk = chunkToBatchMessage(chunks[chunks.length - 1], firstMessage.referenceSequenceNumber, { batch: (_d = firstMessage.metadata) === null || _d === void 0 ? void 0 : _d.batch });
+        const lastChunk = chunkToBatchMessage(chunks[chunks.length - 1], batch.referenceSequenceNumber, { batch: (_d = firstMessage.metadata) === null || _d === void 0 ? void 0 : _d.batch });
         this.logger.sendPerformanceEvent({
             eventName: "Chunked compressed batch",
             length: batch.content.length,
@@ -124,6 +125,7 @@ export class OpSplitter {
         return {
             content: [lastChunk, ...restOfMessages],
             contentSizeInBytes: (_f = (_e = lastChunk.contents) === null || _e === void 0 ? void 0 : _e.length) !== null && _f !== void 0 ? _f : 0,
+            referenceSequenceNumber: batch.referenceSequenceNumber,
         };
     }
 }
@@ -144,25 +146,31 @@ export const splitOp = (op, chunkSizeInBytes) => {
     const chunks = [];
     assert(op.contents !== undefined && op.contents !== null, 0x51a /* We should have something to chunk */);
     const contentLength = op.contents.length;
-    const chunkN = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1;
+    const chunkCount = Math.floor((contentLength - 1) / chunkSizeInBytes) + 2;
     let offset = 0;
-    for (let i = 1; i <= chunkN; i++) {
+    for (let i = 1; i < chunkCount; i++) {
         const chunk = {
             chunkId: i,
             contents: op.contents.substr(offset, chunkSizeInBytes),
             originalType: op.deserializedContent.type,
-            totalChunks: chunkN,
+            totalChunks: chunkCount,
         };
-        if (i === chunkN) {
-            // We don't need to port these to all the chunks,
-            // as we rebuild the original op when we process the
-            // last chunk, therefore it is the only one that needs it.
-            chunk.originalMetadata = op.metadata;
-            chunk.originalCompression = op.compression;
-        }
         chunks.push(chunk);
         offset += chunkSizeInBytes;
+        assert(i === chunkCount - 1 || offset <= contentLength, 0x58b /* Content offset within bounds */);
     }
+    assert(offset >= contentLength, 0x58c /* Content offset equal or larger than content length */);
+    // The last chunk has empty contents, to minimize the risk of the
+    // resulting payload exceeding 1MB due to the overhead from the empty ops
+    // which will be bundled with this op.
+    chunks.push({
+        chunkId: chunkCount,
+        contents: "",
+        originalType: op.deserializedContent.type,
+        totalChunks: chunkCount,
+        originalMetadata: op.metadata,
+        originalCompression: op.compression,
+    });
     return chunks;
 };
 //# sourceMappingURL=opSplitter.js.map

package/lib/opLifecycle/opSplitter.js.map

@@ -1 +1 @@
-{"version":3,"file":"opSplitter.js","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAEtD,OAAO,EACN,mBAAmB,EACnB,gCAAgC,GAChC,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,oBAAoB,EAA2B,MAAM,qBAAqB,CAAC;AAGpF;;GAEG;AACH,MAAM,OAAO,UAAU;IAKtB,YACC,MAA4B,EACX,aAA+D,EAC/D,gBAAwB,EACxB,mBAA2B,EAC5C,MAAwB;QAHP,kBAAa,GAAb,aAAa,CAAkD;QAC/D,qBAAgB,GAAhB,gBAAgB,CAAQ;QACxB,wBAAmB,GAAnB,mBAAmB,CAAQ;QAG5C,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAmB,MAAM,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACxD,CAAC;IAED,IAAW,sBAAsB;QAChC,OAAO,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS,CAAC;IAC7F,CAAC;IAED,IAAW,MAAM;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAEM,oBAAoB,CAAC,OAAkC;QAC7D,IAAI,OAAO,CAAC,IAAI,KAAK,oBAAoB,CAAC,SAAS,EAAE;YACpD,OAAO;gBACN,OAAO;gBACP,KAAK,EAAE,SAAS;aAChB,CAAC;SACF;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,MAAM,cAAc,GAAG,OAAO,CAAC,QAAsB,CAAC;QACtD,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QAEjD,IAAI,cAAc,CAAC,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE;YACxD,yDAAyD;YACzD,6DAA6D;YAC7D,OAAO;gBACN,OAAO;gBACP,KAAK,EAAE,UAAU;aACjB,CAAC;SACF;QAED,oEAAoE;QACpE,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAElC,MAAM,UAAU,qBAAQ,OAAO,CAAE,CAAC;QAClC,UAAU,CAAC,QAAQ,GAAG,iBAAiB,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAC3F,UAAU,CAAC,IAAI,GAAG,cAAc,CAAC,YAAY,CAAC;QAC9C,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC,gBAAgB,CAAC;QACtD,UAAU,CAAC,WAAW,GAAG,cAAc,CAAC,mBAAmB,CAAC;QAC5D,OAAO;YACN,OAAO,EAAE,UAAU;YACnB,KAAK,EAAE,WAAW;SAClB,CAAC;IACH,CAAC;IAEM,kBAAkB,CAAC,QAAgB;QACzC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAChC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;SAC/B;IACF,CAAC;IAEO,QAAQ,CACf,QAAgB,EAChB,cAA0B,EAC1B,eAA0C;QAE1C,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,GAAG,KAAK,SAAS,EAAE;YACtB,GAAG,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;SACjC;QAED,IAAI,cAAc,CAAC,OAAO,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;YAC9C,gGAAgG;YAChG,sGAAsG;YACtG,4DAA4D;YAC5D,MAAM,IAAI,mBAAmB,CAAC,mBAAmB,kCAC7C,gCAAgC,CAAC,eAAe,CAAC,KACpD,cAAc,EAAE,GAAG,CAAC,MAAM,EAC1B,OAAO,EAAE,cAAc,CAAC,OAAO,EAC/B,WAAW,EAAE,cAAc,CAAC,WAAW,IACtC,CAAC;SACH;QAED,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACI,oBAAoB,CAAC,KAAa;;QACxC,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QAC9E,MAAM,CACL,KAAK,CAAC,kBAAkB,GAAG,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EACxD,KAAK,CAAC,iCAAiC,CACvC,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACjF,MAAM,CACL,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,mBAAmB,EAChD,KAAK,CAAC,4DAA4D,CAClE,CAAC;QAEF,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,wEAAwE;QAC/G,MAAM,CACL,CAAA,MAAA,YAAY,CAAC,QAAQ,0CAAE,UAAU,MAAK,IAAI,IAAI,YAAY,CAAC,WAAW,KAAK,SAAS,EACpF,KAAK,CAAC,kCAAkC,CACxC,CAAC;QACF,MAAM,CACL,CAAC,MAAA,MAAA,YAAY,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAC7D,KAAK,CAAC,sDAAsD,CAC5D,CAAC;QAEF,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,uEAAuE;QACtH,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAE5D,MAAM,CAAC,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACnF,wCAAwC;QACxC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;YACxC,IAAI,CAAC,aAAa,CAAC,CAAC,mBAAmB,CAAC,KAAK,EAAE,YAAY,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC;SACvF;QAED,4DAA4D;QAC5D,oDAAoD;QACpD,MAAM,SAAS,GAAG,mBAAmB,CACpC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EACzB,YAAY,CAAC,uBAAuB,EACpC,EAAE,KAAK,EAAE,MAAA,YAAY,CAAC,QAAQ,0CAAE,KAAK,EAAE,CACvC,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;YAChC,SAAS,EAAE,0BAA0B;YACrC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;YAC5B,WAAW,EAAE,KAAK,CAAC,kBAAkB;YACrC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACvC,CAAC,CAAC;QAEH,OAAO;YACN,OAAO,EAAE,CAAC,SAAS,EAAE,GAAG,cAAc,CAAC;YACvC,kBAAkB,EAAE,MAAA,MAAA,SAAS,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC;SACnD,CAAC;IACH,CAAC;CACD;AAED,MAAM,mBAAmB,GAAG,CAC3B,KAAiB,EACjB,uBAA+B,EAC/B,WAAgD,SAAS,EAC1C,EAAE;IACjB,MAAM,OAAO,GAA4B;QACxC,IAAI,EAAE,oBAAoB,CAAC,SAAS;QACpC,QAAQ,EAAE,KAAK;KACf,CAAC;IACF,OAAO;QACN,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QACjC,mBAAmB,EAAE,OAAO;QAC5B,QAAQ;QACR,eAAe,EAAE,SAAS;QAC1B,uBAAuB;KACvB,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAgB,EAAE,gBAAwB,EAAgB,EAAE;IACnF,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,CACL,EAAE,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,CAAC,QAAQ,KAAK,IAAI,EACjD,KAAK,CAAC,uCAAuC,CAC7C,CAAC;IAEF,MAAM,aAAa,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACtE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC,EAAE,EAAE;QACjC,MAAM,KAAK,GAAe;YACzB,OAAO,EAAE,CAAC;YACV,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC;YACtD,YAAY,EAAE,EAAE,CAAC,mBAAmB,CAAC,IAAI;YACzC,WAAW,EAAE,MAAM;SACnB,CAAC;QAEF,IAAI,CAAC,KAAK,MAAM,EAAE;YACjB,iDAAiD;YACjD,oDAAoD;YACpD,0DAA0D;YAC1D,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC,QAAQ,CAAC;YACrC,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,WAAW,CAAC;SAC3C;QAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,IAAI,gBAAgB,CAAC;KAC3B;IAED,OAAO,MAAM,CAAC;AACf,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert } from \"@fluidframework/common-utils\";\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport {\n\tDataCorruptionError,\n\textractSafePropertiesFromMessage,\n} from \"@fluidframework/container-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { ContainerMessageType, ContainerRuntimeMessage } from \"../containerRuntime\";\nimport { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from \"./definitions\";\n\n/**\n * Responsible for creating and reconstructing chunked messages.\n */\nexport class OpSplitter {\n\t// Local copy of incomplete received chunks.\n\tprivate readonly chunkMap: Map<string, string[]>;\n\tprivate readonly logger;\n\n\tconstructor(\n\t\tchunks: [string, string[]][],\n\t\tprivate readonly submitBatchFn: ((batch: IBatchMessage[]) => number) | undefined,\n\t\tprivate readonly chunkSizeInBytes: number,\n\t\tprivate readonly maxBatchSizeInBytes: number,\n\t\tlogger: ITelemetryLogger,\n\t) {\n\t\tthis.chunkMap = new Map<string, string[]>(chunks);\n\t\tthis.logger = ChildLogger.create(logger, \"OpSplitter\");\n\t}\n\n\tpublic get isBatchChunkingEnabled(): boolean {\n\t\treturn this.chunkSizeInBytes < Number.POSITIVE_INFINITY && this.submitBatchFn !== undefined;\n\t}\n\n\tpublic get chunks(): ReadonlyMap<string, string[]> {\n\t\treturn this.chunkMap;\n\t}\n\n\tpublic processRemoteMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {\n\t\tif (message.type !== ContainerMessageType.ChunkedOp) {\n\t\t\treturn {\n\t\t\t\tmessage,\n\t\t\t\tstate: \"Skipped\",\n\t\t\t};\n\t\t}\n\n\t\tconst clientId = message.clientId;\n\t\tconst chunkedContent = message.contents as IChunkedOp;\n\t\tthis.addChunk(clientId, chunkedContent, message);\n\n\t\tif (chunkedContent.chunkId < chunkedContent.totalChunks) {\n\t\t\t// We are processing the op in chunks but haven't reached\n\t\t\t// the last chunk yet in order to reconstruct the original op\n\t\t\treturn {\n\t\t\t\tmessage,\n\t\t\t\tstate: \"Accepted\",\n\t\t\t};\n\t\t}\n\n\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\tconst serializedContent = this.chunkMap.get(clientId)!.join(\"\");\n\t\tthis.clearPartialChunks(clientId);\n\n\t\tconst newMessage = { ...message };\n\t\tnewMessage.contents = serializedContent === \"\" ? undefined : JSON.parse(serializedContent);\n\t\tnewMessage.type = chunkedContent.originalType;\n\t\tnewMessage.metadata = chunkedContent.originalMetadata;\n\t\tnewMessage.compression = chunkedContent.originalCompression;\n\t\treturn {\n\t\t\tmessage: newMessage,\n\t\t\tstate: \"Processed\",\n\t\t};\n\t}\n\n\tpublic clearPartialChunks(clientId: string) {\n\t\tif (this.chunkMap.has(clientId)) {\n\t\t\tthis.chunkMap.delete(clientId);\n\t\t}\n\t}\n\n\tprivate addChunk(\n\t\tclientId: string,\n\t\tchunkedContent: IChunkedOp,\n\t\toriginalMessage: ISequencedDocumentMessage,\n\t) {\n\t\tlet map = this.chunkMap.get(clientId);\n\t\tif (map === undefined) {\n\t\t\tmap = [];\n\t\t\tthis.chunkMap.set(clientId, map);\n\t\t}\n\n\t\tif (chunkedContent.chunkId !== map.length + 1) {\n\t\t\t// We are expecting the chunks to be processed sequentially, in the same order as they are sent.\n\t\t\t// Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)\n\t\t\t// holding the existing chunks for that particular clientId.\n\t\t\tthrow new DataCorruptionError(\"Chunk Id mismatch\", {\n\t\t\t\t...extractSafePropertiesFromMessage(originalMessage),\n\t\t\t\tchunkMapLength: map.length,\n\t\t\t\tchunkId: chunkedContent.chunkId,\n\t\t\t\ttotalChunks: chunkedContent.totalChunks,\n\t\t\t});\n\t\t}\n\n\t\tmap.push(chunkedContent.contents);\n\t}\n\n\t/**\n\t * Splits the first op of a compressed batch in chunks, sends the chunks separately and\n\t * returns a new batch composed of the last chunk and the rest of the ops in the original batch.\n\t *\n\t * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops\n\t * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original\n\t * uncompressed ops at ingestion in the runtime.\n\t *\n\t * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire\n\t * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.\n\t *\n\t * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch\n\t * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch\n\t * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed\n\t * (as it is the last chunk).\n\t *\n\t * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.\n\t * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.\n\t *\n\t * @param batch - the compressed batch which needs to be processed\n\t * @returns A new adjusted batch which can be sent over the wire\n\t */\n\tpublic splitCompressedBatch(batch: IBatch): IBatch {\n\t\tassert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);\n\t\tassert(\n\t\t\tbatch.contentSizeInBytes > 0 && batch.content.length > 0,\n\t\t\t0x514 /* Batch needs to be non-empty */,\n\t\t);\n\t\tassert(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);\n\t\tassert(\n\t\t\tthis.chunkSizeInBytes < this.maxBatchSizeInBytes,\n\t\t\t0x516 /* Chunk size needs to be smaller than the max batch size */,\n\t\t);\n\n\t\tconst firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split\n\t\tassert(\n\t\t\tfirstMessage.metadata?.compressed === true || firstMessage.compression !== undefined,\n\t\t\t0x517 /* Batch needs to be compressed */,\n\t\t);\n\t\tassert(\n\t\t\t(firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,\n\t\t\t0x518 /* First message in the batch needs to be chunkable */,\n\t\t);\n\n\t\tconst restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers\n\t\tconst chunks = splitOp(firstMessage, this.chunkSizeInBytes);\n\n\t\tassert(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);\n\t\t// Send the first N-1 chunks immediately\n\t\tfor (const chunk of chunks.slice(0, -1)) {\n\t\t\tthis.submitBatchFn([chunkToBatchMessage(chunk, firstMessage.referenceSequenceNumber)]);\n\t\t}\n\n\t\t// The last chunk will be part of the new batch and needs to\n\t\t// preserve the batch metadata of the original batch\n\t\tconst lastChunk = chunkToBatchMessage(\n\t\t\tchunks[chunks.length - 1],\n\t\t\tfirstMessage.referenceSequenceNumber,\n\t\t\t{ batch: firstMessage.metadata?.batch },\n\t\t);\n\n\t\tthis.logger.sendPerformanceEvent({\n\t\t\teventName: \"Chunked compressed batch\",\n\t\t\tlength: batch.content.length,\n\t\t\tsizeInBytes: batch.contentSizeInBytes,\n\t\t\tchunks: chunks.length,\n\t\t\tchunkSizeInBytes: this.chunkSizeInBytes,\n\t\t});\n\n\t\treturn {\n\t\t\tcontent: [lastChunk, ...restOfMessages],\n\t\t\tcontentSizeInBytes: lastChunk.contents?.length ?? 0,\n\t\t};\n\t}\n}\n\nconst chunkToBatchMessage = (\n\tchunk: IChunkedOp,\n\treferenceSequenceNumber: number,\n\tmetadata: Record<string, unknown> | undefined = undefined,\n): BatchMessage => {\n\tconst payload: ContainerRuntimeMessage = {\n\t\ttype: ContainerMessageType.ChunkedOp,\n\t\tcontents: chunk,\n\t};\n\treturn {\n\t\tcontents: JSON.stringify(payload),\n\t\tdeserializedContent: payload,\n\t\tmetadata,\n\t\tlocalOpMetadata: undefined,\n\t\treferenceSequenceNumber,\n\t};\n};\n\nexport const splitOp = (op: BatchMessage, chunkSizeInBytes: number): IChunkedOp[] => {\n\tconst chunks: IChunkedOp[] = [];\n\tassert(\n\t\top.contents !== undefined && op.contents !== null,\n\t\t0x51a /* We should have something to chunk */,\n\t);\n\n\tconst contentLength = op.contents.length;\n\tconst chunkN = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1;\n\tlet offset = 0;\n\tfor (let i = 1; i <= chunkN; i++) {\n\t\tconst chunk: IChunkedOp = {\n\t\t\tchunkId: i,\n\t\t\tcontents: op.contents.substr(offset, chunkSizeInBytes),\n\t\t\toriginalType: op.deserializedContent.type,\n\t\t\ttotalChunks: chunkN,\n\t\t};\n\n\t\tif (i === chunkN) {\n\t\t\t// We don't need to port these to all the chunks,\n\t\t\t// as we rebuild the original op when we process the\n\t\t\t// last chunk, therefore it is the only one that needs it.\n\t\t\tchunk.originalMetadata = op.metadata;\n\t\t\tchunk.originalCompression = op.compression;\n\t\t}\n\n\t\tchunks.push(chunk);\n\t\toffset += chunkSizeInBytes;\n\t}\n\n\treturn chunks;\n};\n"]}
+{"version":3,"file":"opSplitter.js","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAEtD,OAAO,EACN,mBAAmB,EACnB,gCAAgC,GAChC,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,oBAAoB,EAA2B,MAAM,qBAAqB,CAAC;AAGpF;;GAEG;AACH,MAAM,OAAO,UAAU;IAKtB,YACC,MAA4B,EACX,aAEL,EACK,gBAAwB,EACxB,mBAA2B,EAC5C,MAAwB;QALP,kBAAa,GAAb,aAAa,CAElB;QACK,qBAAgB,GAAhB,gBAAgB,CAAQ;QACxB,wBAAmB,GAAnB,mBAAmB,CAAQ;QAG5C,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAmB,MAAM,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACxD,CAAC;IAED,IAAW,sBAAsB;QAChC,OAAO,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS,CAAC;IAC7F,CAAC;IAED,IAAW,MAAM;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC;IACtB,CAAC;IAEM,oBAAoB,CAAC,OAAkC;QAC7D,IAAI,OAAO,CAAC,IAAI,KAAK,oBAAoB,CAAC,SAAS,EAAE;YACpD,OAAO;gBACN,OAAO;gBACP,KAAK,EAAE,SAAS;aAChB,CAAC;SACF;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,MAAM,cAAc,GAAG,OAAO,CAAC,QAAsB,CAAC;QACtD,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QAEjD,IAAI,cAAc,CAAC,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE;YACxD,yDAAyD;YACzD,6DAA6D;YAC7D,OAAO;gBACN,OAAO;gBACP,KAAK,EAAE,UAAU;aACjB,CAAC;SACF;QAED,oEAAoE;QACpE,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAElC,MAAM,UAAU,qBAAQ,OAAO,CAAE,CAAC;QAClC,UAAU,CAAC,QAAQ,GAAG,iBAAiB,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAC3F,UAAU,CAAC,IAAI,GAAG,cAAc,CAAC,YAAY,CAAC;QAC9C,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC,gBAAgB,CAAC;QACtD,UAAU,CAAC,WAAW,GAAG,cAAc,CAAC,mBAAmB,CAAC;QAC5D,OAAO;YACN,OAAO,EAAE,UAAU;YACnB,KAAK,EAAE,WAAW;SAClB,CAAC;IACH,CAAC;IAEM,kBAAkB,CAAC,QAAgB;QACzC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAChC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;SAC/B;IACF,CAAC;IAEO,QAAQ,CACf,QAAgB,EAChB,cAA0B,EAC1B,eAA0C;QAE1C,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,GAAG,KAAK,SAAS,EAAE;YACtB,GAAG,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;SACjC;QAED,IAAI,cAAc,CAAC,OAAO,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;YAC9C,gGAAgG;YAChG,sGAAsG;YACtG,4DAA4D;YAC5D,MAAM,IAAI,mBAAmB,CAAC,mBAAmB,kCAC7C,gCAAgC,CAAC,eAAe,CAAC,KACpD,cAAc,EAAE,GAAG,CAAC,MAAM,EAC1B,OAAO,EAAE,cAAc,CAAC,OAAO,EAC/B,WAAW,EAAE,cAAc,CAAC,WAAW,IACtC,CAAC;SACH;QAED,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACI,oBAAoB,CAAC,KAAa;;QACxC,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QAC9E,MAAM,CACL,KAAK,CAAC,kBAAkB,GAAG,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EACxD,KAAK,CAAC,iCAAiC,CACvC,CAAC;QACF,MAAM,CACL,KAAK,CAAC,uBAAuB,KAAK,SAAS,EAC3C,KAAK,CAAC,8DAA8D,CACpE,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACjF,MAAM,CACL,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,mBAAmB,EAChD,KAAK,CAAC,4DAA4D,CAClE,CAAC;QAEF,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,wEAAwE;QAC/G,MAAM,CACL,CAAA,MAAA,YAAY,CAAC,QAAQ,0CAAE,UAAU,MAAK,IAAI,IAAI,YAAY,CAAC,WAAW,KAAK,SAAS,EACpF,KAAK,CAAC,kCAAkC,CACxC,CAAC;QACF,MAAM,CACL,CAAC,MAAA,MAAA,YAAY,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAC7D,KAAK,CAAC,sDAAsD,CAC5D,CAAC;QAEF,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,uEAAuE;QACtH,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAE5D,MAAM,CAAC,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACnF,wCAAwC;QACxC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;YACxC,IAAI,CAAC,aAAa,CACjB,CAAC,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC,uBAAuB,CAAC,CAAC,EAC3D,KAAK,CAAC,uBAAuB,CAC7B,CAAC;SACF;QAED,4DAA4D;QAC5D,oDAAoD;QACpD,MAAM,SAAS,GAAG,mBAAmB,CACpC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EACzB,KAAK,CAAC,uBAAuB,EAC7B,EAAE,KAAK,EAAE,MAAA,YAAY,CAAC,QAAQ,0CAAE,KAAK,EAAE,CACvC,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;YAChC,SAAS,EAAE,0BAA0B;YACrC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;YAC5B,WAAW,EAAE,KAAK,CAAC,kBAAkB;YACrC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACvC,CAAC,CAAC;QAEH,OAAO;YACN,OAAO,EAAE,CAAC,SAAS,EAAE,GAAG,cAAc,CAAC;YACvC,kBAAkB,EAAE,MAAA,MAAA,SAAS,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC;YACnD,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;SACtD,CAAC;IACH,CAAC;CACD;AAED,MAAM,mBAAmB,GAAG,CAC3B,KAAiB,EACjB,uBAA+B,EAC/B,WAAgD,SAAS,EAC1C,EAAE;IACjB,MAAM,OAAO,GAA4B;QACxC,IAAI,EAAE,oBAAoB,CAAC,SAAS;QACpC,QAAQ,EAAE,KAAK;KACf,CAAC;IACF,OAAO;QACN,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QACjC,mBAAmB,EAAE,OAAO;QAC5B,QAAQ;QACR,eAAe,EAAE,SAAS;QAC1B,uBAAuB;KACvB,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAgB,EAAE,gBAAwB,EAAgB,EAAE;IACnF,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,CACL,EAAE,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,CAAC,QAAQ,KAAK,IAAI,EACjD,KAAK,CAAC,uCAAuC,CAC7C,CAAC;IAEF,MAAM,aAAa,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAC1E,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE;QACpC,MAAM,KAAK,GAAe;YACzB,OAAO,EAAE,CAAC;YACV,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC;YACtD,YAAY,EAAE,EAAE,CAAC,mBAAmB,CAAC,IAAI;YACzC,WAAW,EAAE,UAAU;SACvB,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,IAAI,gBAAgB,CAAC;QAC3B,MAAM,CACL,CAAC,KAAK,UAAU,GAAG,CAAC,IAAI,MAAM,IAAI,aAAa,EAC/C,KAAK,CAAC,kCAAkC,CACxC,CAAC;KACF;IAED,MAAM,CAAC,MAAM,IAAI,aAAa,EAAE,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAChG,iEAAiE;IACjE,yEAAyE;IACzE,sCAAsC;IACtC,MAAM,CAAC,IAAI,CAAC;QACX,OAAO,EAAE,UAAU;QACnB,QAAQ,EAAE,EAAE;QACZ,YAAY,EAAE,EAAE,CAAC,mBAAmB,CAAC,IAAI;QACzC,WAAW,EAAE,UAAU;QACvB,gBAAgB,EAAE,EAAE,CAAC,QAAQ;QAC7B,mBAAmB,EAAE,EAAE,CAAC,WAAW;KACnC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AACf,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert } from \"@fluidframework/common-utils\";\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport {\n\tDataCorruptionError,\n\textractSafePropertiesFromMessage,\n} from \"@fluidframework/container-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { ContainerMessageType, ContainerRuntimeMessage } from \"../containerRuntime\";\nimport { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from \"./definitions\";\n\n/**\n * Responsible for creating and reconstructing chunked messages.\n */\nexport class OpSplitter {\n\t// Local copy of incomplete received chunks.\n\tprivate readonly chunkMap: Map<string, string[]>;\n\tprivate readonly logger;\n\n\tconstructor(\n\t\tchunks: [string, string[]][],\n\t\tprivate readonly submitBatchFn:\n\t\t\t| ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number)\n\t\t\t| undefined,\n\t\tprivate readonly chunkSizeInBytes: number,\n\t\tprivate readonly maxBatchSizeInBytes: number,\n\t\tlogger: ITelemetryLogger,\n\t) {\n\t\tthis.chunkMap = new Map<string, string[]>(chunks);\n\t\tthis.logger = ChildLogger.create(logger, \"OpSplitter\");\n\t}\n\n\tpublic get isBatchChunkingEnabled(): boolean {\n\t\treturn this.chunkSizeInBytes < Number.POSITIVE_INFINITY && this.submitBatchFn !== undefined;\n\t}\n\n\tpublic get chunks(): ReadonlyMap<string, string[]> {\n\t\treturn this.chunkMap;\n\t}\n\n\tpublic processRemoteMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {\n\t\tif (message.type !== ContainerMessageType.ChunkedOp) {\n\t\t\treturn {\n\t\t\t\tmessage,\n\t\t\t\tstate: \"Skipped\",\n\t\t\t};\n\t\t}\n\n\t\tconst clientId = message.clientId;\n\t\tconst chunkedContent = message.contents as IChunkedOp;\n\t\tthis.addChunk(clientId, chunkedContent, message);\n\n\t\tif (chunkedContent.chunkId < chunkedContent.totalChunks) {\n\t\t\t// We are processing the op in chunks but haven't reached\n\t\t\t// the last chunk yet in order to reconstruct the original op\n\t\t\treturn {\n\t\t\t\tmessage,\n\t\t\t\tstate: \"Accepted\",\n\t\t\t};\n\t\t}\n\n\t\t// eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n\t\tconst serializedContent = this.chunkMap.get(clientId)!.join(\"\");\n\t\tthis.clearPartialChunks(clientId);\n\n\t\tconst newMessage = { ...message };\n\t\tnewMessage.contents = serializedContent === \"\" ? undefined : JSON.parse(serializedContent);\n\t\tnewMessage.type = chunkedContent.originalType;\n\t\tnewMessage.metadata = chunkedContent.originalMetadata;\n\t\tnewMessage.compression = chunkedContent.originalCompression;\n\t\treturn {\n\t\t\tmessage: newMessage,\n\t\t\tstate: \"Processed\",\n\t\t};\n\t}\n\n\tpublic clearPartialChunks(clientId: string) {\n\t\tif (this.chunkMap.has(clientId)) {\n\t\t\tthis.chunkMap.delete(clientId);\n\t\t}\n\t}\n\n\tprivate addChunk(\n\t\tclientId: string,\n\t\tchunkedContent: IChunkedOp,\n\t\toriginalMessage: ISequencedDocumentMessage,\n\t) {\n\t\tlet map = this.chunkMap.get(clientId);\n\t\tif (map === undefined) {\n\t\t\tmap = [];\n\t\t\tthis.chunkMap.set(clientId, map);\n\t\t}\n\n\t\tif (chunkedContent.chunkId !== map.length + 1) {\n\t\t\t// We are expecting the chunks to be processed sequentially, in the same order as they are sent.\n\t\t\t// Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)\n\t\t\t// holding the existing chunks for that particular clientId.\n\t\t\tthrow new DataCorruptionError(\"Chunk Id mismatch\", {\n\t\t\t\t...extractSafePropertiesFromMessage(originalMessage),\n\t\t\t\tchunkMapLength: map.length,\n\t\t\t\tchunkId: chunkedContent.chunkId,\n\t\t\t\ttotalChunks: chunkedContent.totalChunks,\n\t\t\t});\n\t\t}\n\n\t\tmap.push(chunkedContent.contents);\n\t}\n\n\t/**\n\t * Splits the first op of a compressed batch in chunks, sends the chunks separately and\n\t * returns a new batch composed of the last chunk and the rest of the ops in the original batch.\n\t *\n\t * A compressed batch is formed by one large op at the first position, followed by a series of placeholder ops\n\t * which are used in order to reserve the sequence numbers for when the first op gets unrolled into the original\n\t * uncompressed ops at ingestion in the runtime.\n\t *\n\t * If the first op is too large, it can be chunked (split into smaller op) which can be sent individually over the wire\n\t * and accumulate at ingestion, until the last op in the chunk is processed, when the original op is unrolled.\n\t *\n\t * This method will send the first N - 1 chunks separately and use the last chunk as the first message in the result batch\n\t * and then appends the original placeholder ops. This will ensure that the batch semantics of the original (non-compressed) batch\n\t * are preserved, as the original chunked op will be unrolled by the runtime when the first message in the batch is processed\n\t * (as it is the last chunk).\n\t *\n\t * To illustrate, if the input is `[largeOp, emptyOp, emptyOp]`, `largeOp` will be split into `[chunk1, chunk2, chunk3, chunk4]`.\n\t * `chunk1`, `chunk2` and `chunk3` will be sent individually and `[chunk4, emptyOp, emptyOp]` will be returned.\n\t *\n\t * @param batch - the compressed batch which needs to be processed\n\t * @returns A new adjusted batch which can be sent over the wire\n\t */\n\tpublic splitCompressedBatch(batch: IBatch): IBatch {\n\t\tassert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);\n\t\tassert(\n\t\t\tbatch.contentSizeInBytes > 0 && batch.content.length > 0,\n\t\t\t0x514 /* Batch needs to be non-empty */,\n\t\t);\n\t\tassert(\n\t\t\tbatch.referenceSequenceNumber !== undefined,\n\t\t\t0x58a /* Batch must have a reference sequence number if non-empty */,\n\t\t);\n\t\tassert(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);\n\t\tassert(\n\t\t\tthis.chunkSizeInBytes < this.maxBatchSizeInBytes,\n\t\t\t0x516 /* Chunk size needs to be smaller than the max batch size */,\n\t\t);\n\n\t\tconst firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split\n\t\tassert(\n\t\t\tfirstMessage.metadata?.compressed === true || firstMessage.compression !== undefined,\n\t\t\t0x517 /* Batch needs to be compressed */,\n\t\t);\n\t\tassert(\n\t\t\t(firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,\n\t\t\t0x518 /* First message in the batch needs to be chunkable */,\n\t\t);\n\n\t\tconst restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers\n\t\tconst chunks = splitOp(firstMessage, this.chunkSizeInBytes);\n\n\t\tassert(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);\n\t\t// Send the first N-1 chunks immediately\n\t\tfor (const chunk of chunks.slice(0, -1)) {\n\t\t\tthis.submitBatchFn(\n\t\t\t\t[chunkToBatchMessage(chunk, batch.referenceSequenceNumber)],\n\t\t\t\tbatch.referenceSequenceNumber,\n\t\t\t);\n\t\t}\n\n\t\t// The last chunk will be part of the new batch and needs to\n\t\t// preserve the batch metadata of the original batch\n\t\tconst lastChunk = chunkToBatchMessage(\n\t\t\tchunks[chunks.length - 1],\n\t\t\tbatch.referenceSequenceNumber,\n\t\t\t{ batch: firstMessage.metadata?.batch },\n\t\t);\n\n\t\tthis.logger.sendPerformanceEvent({\n\t\t\teventName: \"Chunked compressed batch\",\n\t\t\tlength: batch.content.length,\n\t\t\tsizeInBytes: batch.contentSizeInBytes,\n\t\t\tchunks: chunks.length,\n\t\t\tchunkSizeInBytes: this.chunkSizeInBytes,\n\t\t});\n\n\t\treturn {\n\t\t\tcontent: [lastChunk, ...restOfMessages],\n\t\t\tcontentSizeInBytes: lastChunk.contents?.length ?? 0,\n\t\t\treferenceSequenceNumber: batch.referenceSequenceNumber,\n\t\t};\n\t}\n}\n\nconst chunkToBatchMessage = (\n\tchunk: IChunkedOp,\n\treferenceSequenceNumber: number,\n\tmetadata: Record<string, unknown> | undefined = undefined,\n): BatchMessage => {\n\tconst payload: ContainerRuntimeMessage = {\n\t\ttype: ContainerMessageType.ChunkedOp,\n\t\tcontents: chunk,\n\t};\n\treturn {\n\t\tcontents: JSON.stringify(payload),\n\t\tdeserializedContent: payload,\n\t\tmetadata,\n\t\tlocalOpMetadata: undefined,\n\t\treferenceSequenceNumber,\n\t};\n};\n\nexport const splitOp = (op: BatchMessage, chunkSizeInBytes: number): IChunkedOp[] => {\n\tconst chunks: IChunkedOp[] = [];\n\tassert(\n\t\top.contents !== undefined && op.contents !== null,\n\t\t0x51a /* We should have something to chunk */,\n\t);\n\n\tconst contentLength = op.contents.length;\n\tconst chunkCount = Math.floor((contentLength - 1) / chunkSizeInBytes) + 2;\n\tlet offset = 0;\n\tfor (let i = 1; i < chunkCount; i++) {\n\t\tconst chunk: IChunkedOp = {\n\t\t\tchunkId: i,\n\t\t\tcontents: op.contents.substr(offset, chunkSizeInBytes),\n\t\t\toriginalType: op.deserializedContent.type,\n\t\t\ttotalChunks: chunkCount,\n\t\t};\n\n\t\tchunks.push(chunk);\n\t\toffset += chunkSizeInBytes;\n\t\tassert(\n\t\t\ti === chunkCount - 1 || offset <= contentLength,\n\t\t\t0x58b /* Content offset within bounds */,\n\t\t);\n\t}\n\n\tassert(offset >= contentLength, 0x58c /* Content offset equal or larger than content length */);\n\t// The last chunk has empty contents, to minimize the risk of the\n\t// resulting payload exceeding 1MB due to the overhead from the empty ops\n\t// which will be bundled with this op.\n\tchunks.push({\n\t\tchunkId: chunkCount,\n\t\tcontents: \"\",\n\t\toriginalType: op.deserializedContent.type,\n\t\ttotalChunks: chunkCount,\n\t\toriginalMetadata: op.metadata,\n\t\toriginalCompression: op.compression,\n\t});\n\n\treturn chunks;\n};\n"]}

package/lib/opLifecycle/outbox.d.ts

@@ -12,7 +12,7 @@ import { OpSplitter } from "./opSplitter";
 export interface IOutboxConfig {
     readonly compressionOptions: ICompressionRuntimeOptions;
     readonly maxBatchSizeInBytes: number;
-    readonly enableOpReentryCheck?: boolean;
+    readonly disablePartialFlush: boolean;
 }
 export interface IOutboxParameters {
     readonly shouldSend: () => boolean;
@@ -25,11 +25,29 @@ export interface IOutboxParameters {
 }
 export declare class Outbox {
     private readonly params;
+    private readonly mc;
     private readonly attachFlowBatch;
     private readonly mainBatch;
     private readonly defaultAttachFlowSoftLimitInBytes;
+    /**
+     * Track the number of ops which were detected to have a mismatched
+     * reference sequence number, in order to self-throttle the telemetry events.
+     *
+     * This should be removed as part of ADO:2322
+     */
+    private readonly maxMismatchedOpsToReport;
+    private mismatchedOpsReported;
     constructor(params: IOutboxParameters);
     get isEmpty(): boolean;
+    /**
+     * If we detect that the reference sequence number of the incoming message does not match
+     * what was already in the batch managers, this means that batching has been interrupted so
+     * we will flush the accumulated messages to account for that and create a new batch with the new
+     * message as the first message.
+     *
+     * @param message - the incoming message
+     */
+    private maybeFlushPartialBatch;
     submit(message: BatchMessage): void;
     submitAttach(message: BatchMessage): void;
     flush(): void;
@@ -39,8 +57,6 @@ export declare class Outbox {
      * Sends the batch object to the container context to be sent over the wire.
      *
      * @param batch - batch to be sent
-     * @returns the client sequence number of the last batched op which was sent and
-     * -1 if there are no ops or the container cannot send ops.
      */
     private sendBatch;
     private persistBatch;

package/lib/opLifecycle/outbox.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"outbox.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/outbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAEtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,uCAAuC,CAAC;AAG1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,OAAO,EAAE,YAAY,EAAU,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,0BAA0B,CAAC;IAExD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,OAAO,CAAC;CACxC;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,OAAO,CAAC;IACnC,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;IAClD,QAAQ,CAAC,gBAAgB,EAAE,iBAAiB,CAAC;IAC7C,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,YAAY,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;CAClC;AAED,qBAAa,MAAM;IAKN,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJnC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAa;gBAElC,MAAM,EAAE,iBAAiB;IAyBtD,IAAW,OAAO,IAAI,OAAO,CAE5B;IAEM,MAAM,CAAC,OAAO,EAAE,YAAY;IAW5B,YAAY,CAAC,OAAO,EAAE,YAAY;IA6BlC,KAAK;IAKZ,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,aAAa;IA+BrB;;;;;;OAMG;IACH,OAAO,CAAC,SAAS;IA6CjB,OAAO,CAAC,YAAY;IAkBb,UAAU;;;;CAMjB"}
+{"version":3,"file":"outbox.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/outbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAEtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,uCAAuC,CAAC;AAQ1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,OAAO,EAAE,YAAY,EAAU,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,0BAA0B,CAAC;IAExD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,OAAO,CAAC;IACnC,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;IAClD,QAAQ,CAAC,gBAAgB,EAAE,iBAAiB,CAAC;IAC7C,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,YAAY,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;CAClC;AAED,qBAAa,MAAM;IAeN,OAAO,CAAC,QAAQ,CAAC,MAAM;IAdnC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAc;IAEhE;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAK;IAC9C,OAAO,CAAC,qBAAqB,CAAK;gBAEL,MAAM,EAAE,iBAAiB;IAatD,IAAW,OAAO,IAAI,OAAO,CAE5B;IAED;;;;;;;OAOG;IACH,OAAO,CAAC,sBAAsB;IAsCvB,MAAM,CAAC,OAAO,EAAE,YAAY;IAa5B,YAAY,CAAC,OAAO,EAAE,YAAY;IA+BlC,KAAK;IAKZ,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,aAAa;IA+BrB;;;;OAIG;IACH,OAAO,CAAC,SAAS;IA4CjB,OAAO,CAAC,YAAY;IAcb,UAAU;;;;CAMjB"}